Implementing enforcing gas expectations in isoltest.

Co-authored-by: Daniel Kirchner <daniel@ekpyron.org>

test/Common.cpp

@@ -103,6 +103,8 @@ CommonOptions::CommonOptions(std::string _caption):
 		("no-smt", po::bool_switch(&disableSMT), "disable SMT checker")
 		("optimize", po::bool_switch(&optimize), "enables optimization")
 		("enforce-via-yul", po::bool_switch(&enforceViaYul), "Enforce compiling all tests via yul to see if additional tests can be activated.")
+		("enforce-gas-cost", po::bool_switch(&enforceGasTest), "Enforce gas cost expectations in semantic tests.")
+		("enforce-gas-cost-min-value", po::value(&enforceGasTestMinValue), "Threshold value when enforcing gas cost expectations.")
 		("abiencoderv1", po::bool_switch(&useABIEncoderV1), "enables abi encoder v1")
 		("show-messages", po::bool_switch(&showMessages), "enables message output")
 		("show-metadata", po::bool_switch(&showMetadata), "enables metadata output");
@@ -120,6 +122,11 @@ void CommonOptions::validate() const
 		ConfigException,
 		"Invalid test path specified."
 	);
+	assertThrow(
+		!enforceGasTest || evmVersion() == langutil::EVMVersion{},
+		ConfigException,
+		"Gas costs can only be enforced on latest evm version."
+	);
 
 }
 

test/Common.h

@@ -56,6 +56,8 @@ struct CommonOptions: boost::noncopyable
 	bool ewasm = false;
 	bool optimize = false;
 	bool enforceViaYul = false;
+	bool enforceGasTest = false;
+	u256 enforceGasTestMinValue = 100000;
 	bool disableSMT = false;
 	bool useABIEncoderV1 = false;
 	bool showMessages = false;

test/TestCase.h

@@ -40,7 +40,9 @@ public:
 		std::string filename;
 		langutil::EVMVersion evmVersion;
 		std::vector<boost::filesystem::path> vmPaths;
-		bool enforceCompileViaYul;
+		bool enforceCompileViaYul = false;
+		bool enforceGasCost = false;
+		u256 enforceGasCostMinValue;
 	};
 
 	enum class TestResult { Success, Failure, FatalError };

test/boostTest.cpp

@@ -71,7 +71,14 @@ int registerTests(
 {
 	int numTestsAdded = 0;
 	fs::path fullpath = _basepath / _path;
-	TestCase::Config config{fullpath.string(), solidity::test::CommonOptions::get().evmVersion(), solidity::test::CommonOptions::get().vmPaths, _enforceViaYul};
+	TestCase::Config config{
+		fullpath.string(),
+		solidity::test::CommonOptions::get().evmVersion(),
+		solidity::test::CommonOptions::get().vmPaths,
+		_enforceViaYul,
+		/* enforceGasCost */ false,
+		0
+	};
 	if (fs::is_directory(fullpath))
 	{
 		test_suite* sub_suite = BOOST_TEST_SUITE(_path.filename().string());

test/libsolidity/SemanticTest.cpp

@@ -49,13 +49,17 @@ SemanticTest::SemanticTest(
 	string const& _filename,
 	langutil::EVMVersion _evmVersion,
 	vector<boost::filesystem::path> const& _vmPaths,
-	bool enforceViaYul
+	bool _enforceViaYul,
+	bool _enforceGasCost,
+	u256 _enforceGasCostMinValue
 ):
 	SolidityExecutionFramework(_evmVersion, _vmPaths),
 	EVMVersionRestrictedTestCase(_filename),
 	m_sources(m_reader.sources()),
 	m_lineOffset(m_reader.lineNumber()),
-	m_enforceViaYul(enforceViaYul)
+	m_enforceViaYul(_enforceViaYul),
+	m_enforceGasCost(_enforceGasCost),
+	m_enforceGasCostMinValue(_enforceGasCostMinValue)
 {
 	string choice = m_reader.stringSetting("compileViaYul", "default");
 	if (choice == "also")
@@ -326,15 +330,23 @@ TestCase::TestResult SemanticTest::runTest(
 
 bool SemanticTest::checkGasCostExpectation(TestFunctionCall& io_test, bool _compileViaYul) const
 {
+	if (m_evmVersion != EVMVersion{})
+		return true;
+
 	string setting =
 		(_compileViaYul ? "ir"s : "legacy"s) +
 		(m_optimiserSettings == OptimiserSettings::full() ? "Optimized" : "");
 
-	if (io_test.call().expectations.gasUsed.count(setting) == 0)
+	if (
+		(!m_enforceGasCost || m_gasUsed < m_enforceGasCostMinValue) &&
+		io_test.call().expectations.gasUsed.count(setting) == 0
+	)
 		return true;
 
 	io_test.setGasCost(setting, m_gasUsed);
-	return m_gasUsed == io_test.call().expectations.gasUsed.at(setting);
+	return
+		io_test.call().expectations.gasUsed.count(setting) > 0 &&
+		m_gasUsed == io_test.call().expectations.gasUsed.at(setting);
 }
 
 void SemanticTest::printSource(ostream& _stream, string const& _linePrefix, bool _formatted) const

test/libsolidity/SemanticTest.h

@@ -40,13 +40,24 @@ class SemanticTest: public SolidityExecutionFramework, public EVMVersionRestrict
 {
 public:
 	static std::unique_ptr<TestCase> create(Config const& _options)
-	{ return std::make_unique<SemanticTest>(_options.filename, _options.evmVersion, _options.vmPaths, _options.enforceCompileViaYul); }
+	{
+		return std::make_unique<SemanticTest>(
+			_options.filename,
+			_options.evmVersion,
+			_options.vmPaths,
+			_options.enforceCompileViaYul,
+			_options.enforceGasCost,
+			_options.enforceGasCostMinValue
+		);
+	}
 
 	explicit SemanticTest(
 		std::string const& _filename,
 		langutil::EVMVersion _evmVersion,
 		std::vector<boost::filesystem::path> const& _vmPaths,
-		bool _enforceViaYul = false
+		bool _enforceViaYul = false,
+		bool _enforceGasCost = false,
+		u256 _enforceGasCostMinValue = 100000
 	);
 
 	TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
@@ -80,6 +91,8 @@ private:
 	std::map<std::string, Builtin> m_builtins{};
 
 	bool m_gasCostFailure = false;
+	bool m_enforceGasCost = false;
+	u256 m_enforceGasCostMinValue;
 };
 
 }

test/tools/IsolTestOptions.cpp

@@ -76,6 +76,8 @@ bool IsolTestOptions::parse(int _argc, char const* const* _argv)
 		return false;
 	}
 	enforceViaYul = true;
+	enforceGasTest = (evmVersion() == langutil::EVMVersion{});
+	enforceGasTestMinValue = 100000;
 
 	return res;
 }

test/tools/isoltest.cpp

@@ -162,7 +162,9 @@ TestTool::Result TestTool::process()
 				m_path.string(),
 				m_options.evmVersion(),
 				m_options.vmPaths,
-				m_options.enforceViaYul
+				m_options.enforceViaYul,
+				m_options.enforceGasTest,
+				m_options.enforceGasTestMinValue
 			});
 			if (m_test->shouldRun())
 				switch (TestCase::TestResult result = m_test->run(outputMessages, "  ", formatted))