Merge pull request #6686 from ethereum/extractedGasTests

Move gas cost tests from command line tests to interactive gas cost tests.

test/InteractiveTests.h

@@ -19,6 +19,7 @@
 
 #include <test/TestCase.h>
 #include <test/libsolidity/ASTJSONTest.h>
+#include <test/libsolidity/GasTest.h>
 #include <test/libsolidity/SyntaxTest.h>
 #include <test/libsolidity/SemanticTest.h>
 #include <test/libsolidity/SMTCheckerJSONTest.h>
@@ -58,7 +59,8 @@ Testsuite const g_interactiveTestsuites[] = {
 	{"Semantic",            "libsolidity", "semanticTests",       false, true,  &SemanticTest::create},
 	{"JSON AST",            "libsolidity", "ASTJSON",             false, false, &ASTJSONTest::create},
 	{"SMT Checker",         "libsolidity", "smtCheckerTests",     true,  false, &SyntaxTest::create},
-	{"SMT Checker JSON",    "libsolidity", "smtCheckerTestsJSON", true,  false, &SMTCheckerTest::create}
+	{"SMT Checker JSON",    "libsolidity", "smtCheckerTestsJSON", true,  false, &SMTCheckerTest::create},
+	{"Gas Estimates",       "libsolidity", "gasTests",            false, false, &GasTest::create}
 };
 
 }

test/TestCase.h

@@ -56,6 +56,8 @@ public:
 		langutil::EVMVersion evmVersion;
 	};
 
+	enum class TestResult { Success, Failure, FatalError };
+
 	using TestCaseCreator = std::unique_ptr<TestCase>(*)(Config const&);
 
 	virtual ~TestCase() = default;
@@ -64,7 +66,7 @@ public:
 	/// Outputs error messages to @arg _stream. Each line of output is prefixed with @arg _linePrefix.
 	/// Optionally, color-coding can be enabled (if @arg _formatted is set to true).
 	/// @returns true, if the test case succeeds, false otherwise
-	virtual bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) = 0;
+	virtual TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) = 0;
 
 	/// Outputs the test contract to @arg _stream.
 	/// Each line of output is prefixed with @arg _linePrefix.

test/boostTest.cpp

@@ -106,8 +106,17 @@ int registerTests(
 						stringstream errorStream;
 						auto testCase = _testCaseCreator(config);
 						if (testCase->validateSettings(dev::test::Options::get().evmVersion()))
-							if (!testCase->run(errorStream))
+							switch (testCase->run(errorStream))
+							{
+								case TestCase::TestResult::Success:
+									break;
+								case TestCase::TestResult::Failure:
 									BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str());
+									break;
+								case TestCase::TestResult::FatalError:
+									BOOST_ERROR("Fatal error during test.\n" + errorStream.str());
+									break;
+							}
 					}
 					catch (boost::exception const& _e)
 					{

test/cmdlineTests/data_storage/args

@@ -1 +0,0 @@
---gas

test/cmdlineTests/data_storage/input.sol

@@ -1,15 +0,0 @@
-pragma solidity >=0.0;
-
-contract C {
-  function f() pure public {
-    require(false, "1234567890123456789012345678901");
-    require(false, "12345678901234567890123456789012");
-    require(false, "123456789012345678901234567890123");
-    require(false, "1234567890123456789012345678901234");
-    require(false, "12345678901234567890123456789012345");
-    require(false, "123456789012345678901234567890123456");
-    require(false, "123456789012345678901234567890121234567890123456789012345678901");
-    require(false, "1234567890123456789012345678901212345678901234567890123456789012");
-    require(false, "12345678901234567890123456789012123456789012345678901234567890123");
-  }
-}

test/cmdlineTests/data_storage/output

@@ -1,7 +0,0 @@
-
-======= data_storage/input.sol:C =======
-Gas estimation:
-construction:
-   294 + 255000 = 255294
-external:
-   f():	252

test/cmdlineTests/gas_test_abiv2/args

@@ -1 +0,0 @@
---gas

test/cmdlineTests/gas_test_abiv2/err

@@ -1,3 +0,0 @@
-gas_test_abiv2/input.sol:2:1: Warning: Experimental features are turned on. Do not use experimental features on live deployments.
-pragma experimental ABIEncoderV2;
-^-------------------------------^

test/cmdlineTests/gas_test_abiv2/output

@@ -1,16 +0,0 @@
-
-======= gas_test_abiv2/input.sol:C =======
-Gas estimation:
-construction:
-   1160 + 1119000 = 1120160
-external:
-   a():	530
-   b(uint256):	infinite
-   f1(uint256):	infinite
-   f2(uint256[],string[],uint16,address):	infinite
-   f3(uint16[],string[],uint16,address):	infinite
-   f4(uint32[],string[12],bytes[2][],address):	infinite
-   f5(address[],string[],bytes,address):	infinite
-   f6(uint256[30],string[],uint16,address):	infinite
-   f7(uint256[31],string[20],address,address):	infinite
-   f8(uint256[32],string[],uint32,address):	infinite

test/cmdlineTests/gas_test_abiv2_optimize_yul/args

@@ -1 +0,0 @@
---gas --optimize --optimize-yul

test/cmdlineTests/gas_test_abiv2_optimize_yul/err

@@ -1,4 +0,0 @@
-Warning: The Yul optimiser is still experimental. Do not use it in production unless correctness of generated code is verified with extensive tests.
-gas_test_abiv2_optimize_yul/input.sol:2:1: Warning: Experimental features are turned on. Do not use experimental features on live deployments.
-pragma experimental ABIEncoderV2;
-^-------------------------------^

test/cmdlineTests/gas_test_abiv2_optimize_yul/output

@@ -1,16 +0,0 @@
-
-======= gas_test_abiv2_optimize_yul/input.sol:C =======
-Gas estimation:
-construction:
-   645 + 608800 = 609445
-external:
-   a():	429
-   b(uint256):	884
-   f1(uint256):	351
-   f2(uint256[],string[],uint16,address):	infinite
-   f3(uint16[],string[],uint16,address):	infinite
-   f4(uint32[],string[12],bytes[2][],address):	infinite
-   f5(address[],string[],bytes,address):	infinite
-   f6(uint256[30],string[],uint16,address):	infinite
-   f7(uint256[31],string[20],address,address):	infinite
-   f8(uint256[32],string[],uint32,address):	infinite

test/cmdlineTests/gas_test_dispatch/args

@@ -1 +0,0 @@
---gas

test/cmdlineTests/gas_test_dispatch/output

@@ -1,53 +0,0 @@
-
-======= gas_test_dispatch/input.sol:Large =======
-Gas estimation:
-construction:
-   670 + 635000 = 635670
-external:
-   a():	451
-   b(uint256):	846
-   f0(uint256):	427
-   f1(uint256):	40752
-   f2(uint256):	20693
-   f3(uint256):	20781
-   f4(uint256):	20759
-   f5(uint256):	20737
-   f6(uint256):	20760
-   f7(uint256):	20672
-   f8(uint256):	20672
-   f9(uint256):	20694
-   g0(uint256):	313
-   g1(uint256):	40707
-   g2(uint256):	20670
-   g3(uint256):	20758
-   g4(uint256):	20736
-   g5(uint256):	20692
-   g6(uint256):	20715
-   g7(uint256):	20714
-   g8(uint256):	20692
-   g9(uint256):	20649
-
-======= gas_test_dispatch/input.sol:Medium =======
-Gas estimation:
-construction:
-   294 + 251200 = 251494
-external:
-   a():	428
-   b(uint256):	846
-   f1(uint256):	40663
-   f2(uint256):	20693
-   f3(uint256):	20737
-   g0(uint256):	313
-   g7(uint256):	20692
-   g8(uint256):	20670
-   g9(uint256):	20626
-
-======= gas_test_dispatch/input.sol:Small =======
-Gas estimation:
-construction:
-   129 + 81800 = 81929
-external:
-   fallback:	118
-   a():	383
-   b(uint256):	802
-   f1(uint256):	40663

test/cmdlineTests/gas_test_dispatch_optimize/args

@@ -1 +0,0 @@
---optimize --optimize-runs 2 --gas

test/cmdlineTests/gas_test_dispatch_optimize/output

@@ -1,53 +0,0 @@
-
-======= gas_test_dispatch_optimize/input.sol:Large =======
-Gas estimation:
-construction:
-   300 + 260000 = 260300
-external:
-   a():	398
-   b(uint256):	1105
-   f0(uint256):	334
-   f1(uint256):	40886
-   f2(uint256):	20952
-   f3(uint256):	21040
-   f4(uint256):	21018
-   f5(uint256):	20996
-   f6(uint256):	20908
-   f7(uint256):	20688
-   f8(uint256):	20820
-   f9(uint256):	20842
-   g0(uint256):	574
-   g1(uint256):	40598
-   g2(uint256):	20686
-   g3(uint256):	20774
-   g4(uint256):	20752
-   g5(uint256):	20840
-   g6(uint256):	20620
-   g7(uint256):	20730
-   g8(uint256):	20708
-   g9(uint256):	20554
-
-======= gas_test_dispatch_optimize/input.sol:Medium =======
-Gas estimation:
-construction:
-   183 + 140400 = 140583
-external:
-   a():	398
-   b(uint256):	863
-   f1(uint256):	40666
-   f2(uint256):	20710
-   f3(uint256):	20754
-   g0(uint256):	332
-   g7(uint256):	20620
-   g8(uint256):	20598
-   g9(uint256):	20554
-
-======= gas_test_dispatch_optimize/input.sol:Small =======
-Gas estimation:
-construction:
-   111 + 63600 = 63711
-external:
-   fallback:	118
-   a():	376
-   b(uint256):	753
-   f1(uint256):	40600

test/libsolidity/ASTJSONTest.cpp

@@ -18,6 +18,7 @@
 #include <test/libsolidity/ASTJSONTest.h>
 #include <test/Options.h>
 #include <libdevcore/AnsiColorized.h>
+#include <liblangutil/SourceReferenceFormatterHuman.h>
 #include <libsolidity/ast/ASTJsonConverter.h>
 #include <libsolidity/interface/CompilerStack.h>
 #include <boost/algorithm/string.hpp>
@@ -27,6 +28,7 @@
 #include <memory>
 #include <stdexcept>
 
+using namespace langutil;
 using namespace dev::solidity;
 using namespace dev::solidity::test;
 using namespace dev::formatting;
@@ -88,7 +90,7 @@ ASTJSONTest::ASTJSONTest(string const& _filename)
 	}
 }
 
-bool ASTJSONTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
+TestCase::TestResult ASTJSONTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
 {
 	CompilerStack c;
 
@@ -101,7 +103,15 @@ bool ASTJSONTest::run(ostream& _stream, string const& _linePrefix, bool const _f
 	}
 	c.setSources(sources);
 	c.setEVMVersion(dev::test::Options::get().evmVersion());
-	c.parseAndAnalyze();
+	if (c.parse())
+		c.analyze();
+	else
+	{
+		SourceReferenceFormatterHuman formatter(_stream, _formatted);
+		for (auto const& error: c.errors())
+			formatter.printErrorInformation(*error);
+		return TestResult::FatalError;
+	}
 
 	for (size_t i = 0; i < m_sources.size(); i++)
 	{
@@ -169,7 +179,7 @@ bool ASTJSONTest::run(ostream& _stream, string const& _linePrefix, bool const _f
 		resultsMatch = false;
 	}
 
-	return resultsMatch;
+	return resultsMatch ? TestResult::Success : TestResult::Failure;
 }
 
 void ASTJSONTest::printSource(ostream& _stream, string const& _linePrefix, bool const) const

test/libsolidity/ASTJSONTest.h

@@ -39,7 +39,7 @@ public:
 	{ return std::unique_ptr<TestCase>(new ASTJSONTest(_config.filename)); }
 	ASTJSONTest(std::string const& _filename);
 
-	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
+	TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
 
 	void printSource(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) const override;
 	void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override;

test/libsolidity/GasTest.cpp

@@ -0,0 +1,238 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <test/libsolidity/GasTest.h>
+#include <test/Options.h>
+#include <libdevcore/CommonIO.h>
+#include <libdevcore/JSON.h>
+#include <liblangutil/SourceReferenceFormatterHuman.h>
+#include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/throw_exception.hpp>
+#include <fstream>
+#include <stdexcept>
+
+using namespace langutil;
+using namespace dev::solidity;
+using namespace dev::solidity::test;
+using namespace dev;
+using namespace std;
+namespace fs = boost::filesystem;
+using namespace boost::unit_test;
+
+
+namespace
+{
+
+u256 parseGasCost(string::iterator& _it, string::iterator _end)
+{
+	if (_it == _end || !isdigit(*_it))
+		throw runtime_error("Invalid test expectation: expected gas cost.");
+	auto begin = _it;
+	while (_it != _end && isdigit(*_it))
+		++_it;
+	return u256(std::string(begin, _it));
+}
+
+}
+
+GasTest::GasTest(string const& _filename)
+{
+	ifstream file(_filename);
+	if (!file)
+		BOOST_THROW_EXCEPTION(runtime_error("Cannot open test contract: \"" + _filename + "\"."));
+	file.exceptions(ios::badbit);
+
+	m_source = parseSourceAndSettings(file);
+
+	if (m_settings.count("optimize"))
+	{
+		m_optimise = true;
+		m_validatedSettings["optimize"] = "true";
+		m_settings.erase("optimize");
+	}
+	if (m_settings.count("optimize-yul"))
+	{
+		m_optimiseYul = true;
+		m_validatedSettings["optimize-yul"] = "true";
+		m_settings.erase("optimize-yul");
+	}
+	if (m_settings.count("optimize-runs"))
+	{
+		m_optimiseRuns = stoul(m_settings["optimize-runs"]);
+		m_validatedSettings["optimize-runs"] = m_settings["optimize-runs"];
+		m_settings.erase("optimize-runs");
+	}
+
+	parseExpectations(file);
+}
+
+void GasTest::parseExpectations(std::istream& _stream)
+{
+	std::map<std::string, std::string>* currentKind = nullptr;
+	std::string line;
+
+	while (getline(_stream, line))
+		if (boost::starts_with(line, "// creation:"))
+		{
+			auto it = line.begin() + 12;
+			skipWhitespace(it, line.end());
+			m_creationCost.executionCost = parseGasCost(it, line.end());
+			skipWhitespace(it, line.end());
+			if (*it++ != '+')
+				BOOST_THROW_EXCEPTION(runtime_error("Invalid expectation: expected \"+\"-"));
+			skipWhitespace(it, line.end());
+			m_creationCost.codeDepositCost = parseGasCost(it, line.end());
+			skipWhitespace(it, line.end());
+			if (*it++ != '=')
+				BOOST_THROW_EXCEPTION(runtime_error("Invalid expectation: expected \"+\"-"));
+			skipWhitespace(it, line.end());
+			m_creationCost.totalCost = parseGasCost(it, line.end());
+		}
+		else if (line == "// external:")
+			currentKind = &m_externalFunctionCosts;
+		else if (line == "// internal:")
+			currentKind = &m_internalFunctionCosts;
+		else if (!currentKind)
+			BOOST_THROW_EXCEPTION(runtime_error("No function kind specified. Expected \"external:\" or \"internal:\"."));
+		else
+		{
+			if (!boost::starts_with(line, "// "))
+				BOOST_THROW_EXCEPTION(runtime_error("Invalid expectation: expected \"// \"."));
+			auto it = line.begin() + 3;
+			skipWhitespace(it, line.end());
+			auto functionNameBegin = it;
+			while (it != line.end() && *it != ':')
+				++it;
+			std::string functionName(functionNameBegin, it);
+			if (functionName == "fallback")
+				functionName.clear();
+			expect(it, line.end(), ':');
+			skipWhitespace(it, line.end());
+			if (it == line.end())
+				BOOST_THROW_EXCEPTION(runtime_error("Invalid expectation: expected gas cost."));
+			(*currentKind)[functionName] = std::string(it, line.end());
+		}
+}
+
+void GasTest::printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const
+{
+	Json::Value estimates = compiler().gasEstimates(compiler().lastContractName());
+	_stream << _linePrefix
+		<< "creation: "
+		<< estimates["creation"]["executionCost"].asString()
+		<< " + "
+		<< estimates["creation"]["codeDepositCost"].asString()
+		<< " = "
+		<< estimates["creation"]["totalCost"].asString()
+		<< std::endl;
+
+	for (auto kind: {"external", "internal"})
+		if (estimates[kind])
+		{
+			_stream << _linePrefix << kind << ":" << std::endl;
+			for (auto it = estimates[kind].begin(); it != estimates[kind].end(); ++it)
+			{
+				_stream << _linePrefix << "  ";
+				if (it.key().asString().empty())
+					_stream << "fallback";
+				else
+					_stream << it.key().asString();
+				_stream << ": " << it->asString() << std::endl;
+			}
+		}
+}
+
+
+TestCase::TestResult GasTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
+{
+	string const versionPragma = "pragma solidity >=0.0;\n";
+	compiler().reset();
+	OptimiserSettings settings = m_optimise ? OptimiserSettings::standard() : OptimiserSettings::minimal();
+	if (m_optimiseYul)
+	{
+		settings.runYulOptimiser = m_optimise;
+		settings.optimizeStackAllocation = m_optimise;
+	}
+	settings.expectedExecutionsPerDeployment = m_optimiseRuns;
+	compiler().setOptimiserSettings(settings);
+	compiler().setSources({{"", versionPragma + m_source}});
+
+	if (!compiler().parseAndAnalyze() || !compiler().compile())
+	{
+		SourceReferenceFormatterHuman formatter(_stream, _formatted);
+		for (auto const& error: compiler().errors())
+			formatter.printErrorInformation(*error);
+		return TestResult::FatalError;
+	}
+
+	Json::Value estimates = compiler().gasEstimates(compiler().lastContractName());
+
+
+	auto creation = estimates["creation"];
+	bool success =
+		(creation["codeDepositCost"].asString() == toString(m_creationCost.codeDepositCost)) &&
+		(creation["executionCost"].asString() == toString(m_creationCost.executionCost)) &&
+		(creation["totalCost"].asString() == toString(m_creationCost.totalCost));
+
+	auto check = [&](map<string, string> const& _a, Json::Value const& _b) {
+		for (auto& entry: _a)
+			success &= _b[entry.first].asString() == entry.second;
+	};
+	check(m_internalFunctionCosts, estimates["internal"]);
+	check(m_externalFunctionCosts, estimates["external"]);
+
+	if (!success)
+	{
+		_stream << _linePrefix << "Expected:" << std::endl;
+		_stream << _linePrefix
+			<< "  creation: "
+			<< toString(m_creationCost.executionCost)
+			<< " + "
+			<< toString(m_creationCost.codeDepositCost)
+			<< " = "
+			<< toString(m_creationCost.totalCost)
+			<< std::endl;
+		auto printExpected = [&](std::string const& _kind, auto const& _expectations)
+		{
+			_stream << _linePrefix << "  " << _kind << ":" << std::endl;
+			for (auto const& entry: _expectations)
+				_stream << _linePrefix
+					<< "    "
+					<< (entry.first.empty() ? "fallback" : entry.first)
+					<< ": "
+					<< entry.second
+					<< std::endl;
+		};
+		if (!m_externalFunctionCosts.empty())
+			printExpected("external", m_externalFunctionCosts);
+		if (!m_internalFunctionCosts.empty())
+			printExpected("internal", m_internalFunctionCosts);
+		_stream << _linePrefix << "Obtained:" << std::endl;
+		printUpdatedExpectations(_stream, _linePrefix + "  ");
+	}
+
+	return success ? TestResult::Success : TestResult::Failure;
+}
+
+void GasTest::printSource(ostream& _stream, string const& _linePrefix, bool) const
+{
+	std::string line;
+	std::istringstream input(m_source);
+	while (getline(input, line))
+		_stream << _linePrefix << line << std::endl;
+}

test/libsolidity/GasTest.h

@@ -0,0 +1,69 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <test/libsolidity/AnalysisFramework.h>
+#include <test/TestCase.h>
+#include <liblangutil/Exceptions.h>
+
+#include <iosfwd>
+#include <string>
+#include <vector>
+#include <utility>
+
+namespace dev
+{
+namespace solidity
+{
+namespace test
+{
+
+class GasTest: AnalysisFramework, public TestCase
+{
+public:
+	static std::unique_ptr<TestCase> create(Config const& _config)
+	{ return std::make_unique<GasTest>(_config.filename); }
+	GasTest(std::string const& _filename);
+
+	TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
+
+	void printSource(std::ostream &_stream, std::string const &_linePrefix = "", bool _formatted = false) const override;
+	void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override;
+
+private:
+	void parseExpectations(std::istream& _stream);
+
+	struct CreationCost
+	{
+		u256 executionCost{0};
+		u256 codeDepositCost{0};
+		u256 totalCost{0};
+	};
+
+	bool m_optimise = false;
+	bool m_optimiseYul = false;
+	size_t m_optimiseRuns = 200;
+	std::string m_source;
+	CreationCost m_creationCost;
+	std::map<std::string, std::string> m_externalFunctionCosts;
+	std::map<std::string, std::string> m_internalFunctionCosts;
+};
+
+}
+}
+}

test/libsolidity/SMTCheckerJSONTest.cpp

@@ -49,7 +49,7 @@ SMTCheckerTest::SMTCheckerTest(string const& _filename, langutil::EVMVersion _ev
 		BOOST_THROW_EXCEPTION(runtime_error("Invalid JSON file."));
 }
 
-bool SMTCheckerTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
+TestCase::TestResult SMTCheckerTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
 {
 	StandardCompiler compiler;
 
@@ -121,7 +121,7 @@ bool SMTCheckerTest::run(ostream& _stream, string const& _linePrefix, bool _form
 		}
 	}
 
-	return printExpectationAndError(_stream, _linePrefix, _formatted);
+	return printExpectationAndError(_stream, _linePrefix, _formatted) ? TestResult::Success : TestResult::Failure;
 }
 
 vector<string> SMTCheckerTest::hashesFromJson(Json::Value const& _jsonObj, string const& _auxInput, string const& _smtlib)

test/libsolidity/SMTCheckerJSONTest.h

@@ -39,7 +39,7 @@ public:
 	}
 	SMTCheckerTest(std::string const& _filename, langutil::EVMVersion _evmVersion);
 
-	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
+	TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
 
 private:
 	std::vector<std::string> hashesFromJson(Json::Value const& _jsonObj, std::string const& _auxInput, std::string const& _smtlib);

test/libsolidity/SemanticTest.cpp

@@ -53,7 +53,7 @@ SemanticTest::SemanticTest(string const& _filename, string const& _ipcPath, lang
 	parseExpectations(file);
 }
 
-bool SemanticTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
+TestCase::TestResult SemanticTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
 {
 	soltestAssert(deploy("", 0, bytes()), "Failed to deploy contract.");
 
@@ -96,9 +96,9 @@ bool SemanticTest::run(ostream& _stream, string const& _linePrefix, bool _format
 		}
 		AnsiColorized(_stream, _formatted, {BOLD, RED}) << _linePrefix << endl << _linePrefix
 			<< "Attention: Updates on the test will apply the detected format displayed." << endl;
-		return false;
+		return TestResult::Failure;
 	}
-	return true;
+	return TestResult::Success;
 }
 
 void SemanticTest::printSource(ostream& _stream, string const& _linePrefix, bool) const

test/libsolidity/SemanticTest.h

@@ -48,7 +48,7 @@ public:
 
 	explicit SemanticTest(std::string const& _filename, std::string const& _ipcPath, langutil::EVMVersion _evmVersion);
 
-	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
+	TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
 	void printSource(std::ostream &_stream, std::string const& _linePrefix = "", bool _formatted = false) const override;
 	void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix = "") const override;
 

test/libsolidity/SyntaxTest.cpp

@@ -63,7 +63,7 @@ SyntaxTest::SyntaxTest(string const& _filename, langutil::EVMVersion _evmVersion
 	m_expectations = parseExpectations(file);
 }
 
-bool SyntaxTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
+TestCase::TestResult SyntaxTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
 {
 	string const versionPragma = "pragma solidity >=0.0;\n";
 	compiler().reset();
@@ -92,7 +92,7 @@ bool SyntaxTest::run(ostream& _stream, string const& _linePrefix, bool _formatte
 		});
 	}
 
-	return printExpectationAndError(_stream, _linePrefix, _formatted);
+	return printExpectationAndError(_stream, _linePrefix, _formatted) ? TestResult::Success : TestResult::Failure;
 }
 
 bool SyntaxTest::printExpectationAndError(ostream& _stream, string const& _linePrefix, bool _formatted)

test/libsolidity/SyntaxTest.h

@@ -57,7 +57,7 @@ public:
 	{ return std::make_unique<SyntaxTest>(_config.filename, _config.evmVersion); }
 	SyntaxTest(std::string const& _filename, langutil::EVMVersion _evmVersion);
 
-	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
+	TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
 
 	void printSource(std::ostream &_stream, std::string const &_linePrefix = "", bool _formatted = false) const override;
 	void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override

test/libsolidity/gasTests/abiv2.sol

@@ -1,4 +1,3 @@
-pragma solidity >=0.0;
 pragma experimental ABIEncoderV2;
 
 contract C {
@@ -13,3 +12,16 @@ contract C {
     function f7(uint[31] memory, string[20] memory, C, address) public returns (bytes[] memory, uint16[] memory) {}
     function f8(uint[32] memory, string[] memory, uint32, address) public returns (uint[] memory, uint16[] memory) {}
 }
+// ----
+// creation: 1160 + 1119000 = 1120160
+// external:
+//   a(): 530
+//   b(uint256): infinite
+//   f1(uint256): infinite
+//   f2(uint256[],string[],uint16,address): infinite
+//   f3(uint16[],string[],uint16,address): infinite
+//   f4(uint32[],string[12],bytes[2][],address): infinite
+//   f5(address[],string[],bytes,address): infinite
+//   f6(uint256[30],string[],uint16,address): infinite
+//   f7(uint256[31],string[20],address,address): infinite
+//   f8(uint256[32],string[],uint32,address): infinite

test/libsolidity/gasTests/abiv2_optimised.sol

@@ -1,4 +1,3 @@
-pragma solidity >=0.0;
 pragma experimental ABIEncoderV2;
 
 contract C {
@@ -13,3 +12,19 @@ contract C {
     function f7(uint[31] memory, string[20] memory, C, address) public returns (bytes[] memory, uint16[] memory) {}
     function f8(uint[32] memory, string[] memory, uint32, address) public returns (uint[] memory, uint16[] memory) {}
 }
+// ====
+// optimize: true
+// optimize-yul: true
+// ----
+// creation: 645 + 608800 = 609445
+// external:
+//   a(): 429
+//   b(uint256): 884
+//   f1(uint256): 351
+//   f2(uint256[],string[],uint16,address): infinite
+//   f3(uint16[],string[],uint16,address): infinite
+//   f4(uint32[],string[12],bytes[2][],address): infinite
+//   f5(address[],string[],bytes,address): infinite
+//   f6(uint256[30],string[],uint16,address): infinite
+//   f7(uint256[31],string[20],address,address): infinite
+//   f8(uint256[32],string[],uint32,address): infinite

test/libsolidity/gasTests/data_storage.sol

@@ -0,0 +1,17 @@
+contract C {
+    function f() pure public {
+        require(false, "1234567890123456789012345678901");
+        require(false, "12345678901234567890123456789012");
+        require(false, "123456789012345678901234567890123");
+        require(false, "1234567890123456789012345678901234");
+        require(false, "12345678901234567890123456789012345");
+        require(false, "123456789012345678901234567890123456");
+        require(false, "123456789012345678901234567890121234567890123456789012345678901");
+        require(false, "1234567890123456789012345678901212345678901234567890123456789012");
+        require(false, "12345678901234567890123456789012123456789012345678901234567890123");
+    }
+}
+// ----
+// creation: 294 + 255000 = 255294
+// external:
+//   f(): 252

test/libsolidity/gasTests/dispatch_large.sol

@@ -1,5 +1,3 @@
-pragma solidity >=0.0;
-
 contract Large {
     uint public a;
     uint[] public b;
@@ -24,20 +22,28 @@ contract Large {
     function g9(uint x) public payable returns (uint) { b[uint8(msg.data[8])] = x; }
     function g0(uint x) public payable returns (uint) { require(x > 10); }
 }
-contract Medium {
+// ----
-    uint public a;
+// creation: 670 + 635000 = 635670
-    uint[] public b;
+// external:
-    function f1(uint x) public returns (uint) { a = x; b[uint8(msg.data[0])] = x; }
+//   a(): 451
-    function f2(uint x) public returns (uint) { b[uint8(msg.data[1])] = x; }
+//   b(uint256): 846
-    function f3(uint x) public returns (uint) { b[uint8(msg.data[2])] = x; }
+//   f0(uint256): 427
-    function g7(uint x) public payable returns (uint) { b[uint8(msg.data[6])] = x; }
+//   f1(uint256): 40752
-    function g8(uint x) public payable returns (uint) { b[uint8(msg.data[7])] = x; }
+//   f2(uint256): 20693
-    function g9(uint x) public payable returns (uint) { b[uint8(msg.data[8])] = x; }
+//   f3(uint256): 20781
-    function g0(uint x) public payable returns (uint) { require(x > 10); }
+//   f4(uint256): 20759
-}
+//   f5(uint256): 20737
-contract Small {
+//   f6(uint256): 20760
-    uint public a;
+//   f7(uint256): 20672
-    uint[] public b;
+//   f8(uint256): 20672
-    function f1(uint x) public returns (uint) { a = x; b[uint8(msg.data[0])] = x; }
+//   f9(uint256): 20694
-    function () external payable {}
+//   g0(uint256): 313
-}
+//   g1(uint256): 40707
+//   g2(uint256): 20670
+//   g3(uint256): 20758
+//   g4(uint256): 20736
+//   g5(uint256): 20692
+//   g6(uint256): 20715
+//   g7(uint256): 20714
+//   g8(uint256): 20692
+//   g9(uint256): 20649

test/libsolidity/gasTests/dispatch_large_optimised.sol

@@ -1,5 +1,3 @@
-pragma solidity >=0.0;
-
 contract Large {
     uint public a;
     uint[] public b;
@@ -24,20 +22,31 @@ contract Large {
     function g9(uint x) public payable returns (uint) { b[uint8(msg.data[8])] = x; }
     function g0(uint x) public payable returns (uint) { require(x > 10); }
 }
-contract Medium {
+// ====
-    uint public a;
+// optimize: true
-    uint[] public b;
+// optimize-runs: 2
-    function f1(uint x) public returns (uint) { a = x; b[uint8(msg.data[0])] = x; }
+// ----
-    function f2(uint x) public returns (uint) { b[uint8(msg.data[1])] = x; }
+// creation: 300 + 260000 = 260300
-    function f3(uint x) public returns (uint) { b[uint8(msg.data[2])] = x; }
+// external:
-    function g7(uint x) public payable returns (uint) { b[uint8(msg.data[6])] = x; }
+//   a(): 398
-    function g8(uint x) public payable returns (uint) { b[uint8(msg.data[7])] = x; }
+//   b(uint256): 1105
-    function g9(uint x) public payable returns (uint) { b[uint8(msg.data[8])] = x; }
+//   f0(uint256): 334
-    function g0(uint x) public payable returns (uint) { require(x > 10); }
+//   f1(uint256): 40886
-}
+//   f2(uint256): 20952
-contract Small {
+//   f3(uint256): 21040
-    uint public a;
+//   f4(uint256): 21018
-    uint[] public b;
+//   f5(uint256): 20996
-    function f1(uint x) public returns (uint) { a = x; b[uint8(msg.data[0])] = x; }
+//   f6(uint256): 20908
-    function () external payable {}
+//   f7(uint256): 20688
-}
+//   f8(uint256): 20820
+//   f9(uint256): 20842
+//   g0(uint256): 574
+//   g1(uint256): 40598
+//   g2(uint256): 20686
+//   g3(uint256): 20774
+//   g4(uint256): 20752
+//   g5(uint256): 20840
+//   g6(uint256): 20620
+//   g7(uint256): 20730
+//   g8(uint256): 20708
+//   g9(uint256): 20554

test/libsolidity/gasTests/dispatch_medium.sol

@@ -0,0 +1,23 @@
+contract Medium {
+    uint public a;
+    uint[] public b;
+    function f1(uint x) public returns (uint) { a = x; b[uint8(msg.data[0])] = x; }
+    function f2(uint x) public returns (uint) { b[uint8(msg.data[1])] = x; }
+    function f3(uint x) public returns (uint) { b[uint8(msg.data[2])] = x; }
+    function g7(uint x) public payable returns (uint) { b[uint8(msg.data[6])] = x; }
+    function g8(uint x) public payable returns (uint) { b[uint8(msg.data[7])] = x; }
+    function g9(uint x) public payable returns (uint) { b[uint8(msg.data[8])] = x; }
+    function g0(uint x) public payable returns (uint) { require(x > 10); }
+}
+// ----
+// creation: 294 + 251200 = 251494
+// external:
+//   a(): 428
+//   b(uint256): 846
+//   f1(uint256): 40663
+//   f2(uint256): 20693
+//   f3(uint256): 20737
+//   g0(uint256): 313
+//   g7(uint256): 20692
+//   g8(uint256): 20670
+//   g9(uint256): 20626

test/libsolidity/gasTests/dispatch_medium_optimised.sol

@@ -0,0 +1,26 @@
+contract Medium {
+    uint public a;
+    uint[] public b;
+    function f1(uint x) public returns (uint) { a = x; b[uint8(msg.data[0])] = x; }
+    function f2(uint x) public returns (uint) { b[uint8(msg.data[1])] = x; }
+    function f3(uint x) public returns (uint) { b[uint8(msg.data[2])] = x; }
+    function g7(uint x) public payable returns (uint) { b[uint8(msg.data[6])] = x; }
+    function g8(uint x) public payable returns (uint) { b[uint8(msg.data[7])] = x; }
+    function g9(uint x) public payable returns (uint) { b[uint8(msg.data[8])] = x; }
+    function g0(uint x) public payable returns (uint) { require(x > 10); }
+}
+// ====
+// optimize: true
+// optimize-runs: 2
+// ----
+// creation: 183 + 140400 = 140583
+// external:
+//   a(): 398
+//   b(uint256): 863
+//   f1(uint256): 40666
+//   f2(uint256): 20710
+//   f3(uint256): 20754
+//   g0(uint256): 332
+//   g7(uint256): 20620
+//   g8(uint256): 20598
+//   g9(uint256): 20554

test/libsolidity/gasTests/dispatch_small.sol

@@ -0,0 +1,13 @@
+contract Small {
+    uint public a;
+    uint[] public b;
+    function f1(uint x) public returns (uint) { a = x; b[uint8(msg.data[0])] = x; }
+    function () external payable {}
+}
+// ----
+// creation: 129 + 81800 = 81929
+// external:
+//   fallback: 118
+//   a(): 383
+//   b(uint256): 802
+//   f1(uint256): 40663

test/libsolidity/gasTests/dispatch_small_optimised.sol

@@ -0,0 +1,16 @@
+contract Small {
+    uint public a;
+    uint[] public b;
+    function f1(uint x) public returns (uint) { a = x; b[uint8(msg.data[0])] = x; }
+    function () external payable {}
+}
+// ====
+// optimize: true
+// optimize-runs: 2
+// ----
+// creation: 111 + 63600 = 63711
+// external:
+//   fallback: 118
+//   a(): 376
+//   b(uint256): 753
+//   f1(uint256): 40600

test/libyul/ObjectCompilerTest.cpp

@@ -62,7 +62,7 @@ ObjectCompilerTest::ObjectCompilerTest(string const& _filename)
 			m_expectation += line + "\n";
 }
 
-bool ObjectCompilerTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
+TestCase::TestResult ObjectCompilerTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
 {
 	AssemblyStack stack(
 		EVMVersion(),
@@ -73,7 +73,7 @@ bool ObjectCompilerTest::run(ostream& _stream, string const& _linePrefix, bool c
 	{
 		AnsiColorized(_stream, _formatted, {formatting::BOLD, formatting::RED}) << _linePrefix << "Error parsing source." << endl;
 		printErrors(_stream, stack.errors());
-		return false;
+		return TestResult::FatalError;
 	}
 	stack.optimize();
 
@@ -98,9 +98,9 @@ bool ObjectCompilerTest::run(ostream& _stream, string const& _linePrefix, bool c
 		printIndented(_stream, m_expectation, nextIndentLevel);
 		AnsiColorized(_stream, _formatted, {formatting::BOLD, formatting::CYAN}) << _linePrefix << "Obtained result:" << endl;
 		printIndented(_stream, m_obtainedResult, nextIndentLevel);
-		return false;
+		return TestResult::Failure;
 	}
-	return true;
+	return TestResult::Success;
 }
 
 void ObjectCompilerTest::printSource(ostream& _stream, string const& _linePrefix, bool const) const

test/libyul/ObjectCompilerTest.h

@@ -47,7 +47,7 @@ public:
 
 	explicit ObjectCompilerTest(std::string const& _filename);
 
-	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
+	TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
 
 	void printSource(std::ostream& _stream, std::string const &_linePrefix = "", bool const _formatted = false) const override;
 	void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override;

test/libyul/YulInterpreterTest.cpp

@@ -67,10 +67,10 @@ YulInterpreterTest::YulInterpreterTest(string const& _filename)
 			m_expectation += line + "\n";
 }
 
-bool YulInterpreterTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
+TestCase::TestResult YulInterpreterTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
 {
 	if (!parse(_stream, _linePrefix, _formatted))
-		return false;
+		return TestResult::FatalError;
 
 	m_obtainedResult = interpret();
 
@@ -82,9 +82,9 @@ bool YulInterpreterTest::run(ostream& _stream, string const& _linePrefix, bool c
 		printIndented(_stream, m_expectation, nextIndentLevel);
 		AnsiColorized(_stream, _formatted, {formatting::BOLD, formatting::CYAN}) << _linePrefix << "Obtained result:" << endl;
 		printIndented(_stream, m_obtainedResult, nextIndentLevel);
-		return false;
+		return TestResult::Failure;
 	}
-	return true;
+	return TestResult::Success;
 }
 
 void YulInterpreterTest::printSource(ostream& _stream, string const& _linePrefix, bool const) const

test/libyul/YulInterpreterTest.h

@@ -47,7 +47,7 @@ public:
 
 	explicit YulInterpreterTest(std::string const& _filename);
 
-	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
+	TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
 
 	void printSource(std::ostream& _stream, std::string const &_linePrefix = "", bool const _formatted = false) const override;
 	void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override;

test/libyul/YulOptimizerTest.cpp

@@ -103,10 +103,10 @@ YulOptimizerTest::YulOptimizerTest(string const& _filename)
 			m_expectation += line + "\n";
 }
 
-bool YulOptimizerTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
+TestCase::TestResult YulOptimizerTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
 {
 	if (!parse(_stream, _linePrefix, _formatted))
-		return false;
+		return TestResult::FatalError;
 
 	if (m_optimizerStep == "disambiguator")
 		disambiguate();
@@ -277,12 +277,11 @@ bool YulOptimizerTest::run(ostream& _stream, string const& _linePrefix, bool con
 	else
 	{
 		AnsiColorized(_stream, _formatted, {formatting::BOLD, formatting::RED}) << _linePrefix << "Invalid optimizer step: " << m_optimizerStep << endl;
-		return false;
+		return TestResult::FatalError;
 	}
 
 	m_obtainedResult = AsmPrinter{m_yul}(*m_ast) + "\n";
 
-	bool success = true;
 	if (m_optimizerStep != m_validatedSettings["step"])
 	{
 		string nextIndentLevel = _linePrefix + "  ";
@@ -294,7 +293,7 @@ bool YulOptimizerTest::run(ostream& _stream, string const& _linePrefix, bool con
 			m_optimizerStep <<
 			"\"." <<
 			endl;
-		success = false;
+		return TestResult::FatalError;
 	}
 	if (m_expectation != m_obtainedResult)
 	{
@@ -304,9 +303,9 @@ bool YulOptimizerTest::run(ostream& _stream, string const& _linePrefix, bool con
 		printIndented(_stream, m_expectation, nextIndentLevel);
 		AnsiColorized(_stream, _formatted, {formatting::BOLD, formatting::CYAN}) << _linePrefix << "Obtained result:" << endl;
 		printIndented(_stream, m_obtainedResult, nextIndentLevel);
-		success = false;
+		return TestResult::Failure;
 	}
-	return success;
+	return TestResult::Success;
 }
 
 void YulOptimizerTest::printSource(ostream& _stream, string const& _linePrefix, bool const) const

test/libyul/YulOptimizerTest.h

@@ -48,7 +48,7 @@ public:
 
 	explicit YulOptimizerTest(std::string const& _filename);
 
-	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
+	TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
 
 	void printSource(std::ostream& _stream, std::string const &_linePrefix = "", bool const _formatted = false) const override;
 	void printUpdatedSettings(std::ostream &_stream, std::string const &_linePrefix = "", bool const _formatted = false) override;

test/tools/CMakeLists.txt

@@ -20,6 +20,7 @@ add_executable(isoltest
 	../TestCase.cpp
 	../libsolidity/util/TestFileParser.cpp
 	../libsolidity/util/TestFunctionCall.cpp
+	../libsolidity/GasTest.cpp
 	../libsolidity/SyntaxTest.cpp
 	../libsolidity/SemanticTest.cpp
 	../libsolidity/AnalysisFramework.cpp

test/tools/isoltest.cpp

@@ -147,7 +147,6 @@ bool TestTool::m_exitRequested = false;
 
 TestTool::Result TestTool::process()
 {
-	bool success;
 	bool formatted{!m_options.noColor};
 	std::stringstream outputMessages;
 
@@ -159,7 +158,21 @@ TestTool::Result TestTool::process()
 
 			m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_options.ipcPath.string(), m_options.evmVersion()});
 			if (m_test->validateSettings(m_options.evmVersion()))
-				success = m_test->run(outputMessages, "  ", formatted);
+				switch (TestCase::TestResult result = m_test->run(outputMessages, "  ", formatted))
+				{
+					case TestCase::TestResult::Success:
+						AnsiColorized(cout, formatted, {BOLD, GREEN}) << "OK" << endl;
+						return Result::Success;
+					default:
+						AnsiColorized(cout, formatted, {BOLD, RED}) << "FAIL" << endl;
+
+						AnsiColorized(cout, formatted, {BOLD, CYAN}) << "  Contract:" << endl;
+						m_test->printSource(cout, "    ", formatted);
+						m_test->printUpdatedSettings(cout, "    ", formatted);
+
+						cout << endl << outputMessages.str() << endl;
+						return result == TestCase::TestResult::FatalError ? Result::Exception : Result::Failure;
+				}
 			else
 			{
 				AnsiColorized(cout, formatted, {BOLD, YELLOW}) << "NOT RUN" << endl;
@@ -187,23 +200,6 @@ TestTool::Result TestTool::process()
 			"Unknown exception during test." << endl;
 		return Result::Exception;
 	}
-
-	if (success)
-	{
-		AnsiColorized(cout, formatted, {BOLD, GREEN}) << "OK" << endl;
-		return Result::Success;
-	}
-	else
-	{
-		AnsiColorized(cout, formatted, {BOLD, RED}) << "FAIL" << endl;
-
-		AnsiColorized(cout, formatted, {BOLD, CYAN}) << "  Contract:" << endl;
-		m_test->printSource(cout, "    ", formatted);
-		m_test->printUpdatedSettings(cout, "    ", formatted);
-
-		cout << endl << outputMessages.str() << endl;
-		return Result::Failure;
-	}
 }
 
 TestTool::Request TestTool::handleResponse(bool _exception)