Fuzzer: Add test harness for solidity inheritance protobuf fuzzer

Co-Authored-By: Leonardo <leo@ethereum.org>

.circleci/config.yml

@@ -50,6 +50,7 @@ defaults:
         cd build
         protoc --proto_path=../test/tools/ossfuzz yulProto.proto --cpp_out=../test/tools/ossfuzz
         protoc --proto_path=../test/tools/ossfuzz abiV2Proto.proto --cpp_out=../test/tools/ossfuzz
+        protoc --proto_path=../test/tools/ossfuzz solProto.proto --cpp_out=../test/tools/ossfuzz
         cmake .. -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} $CMAKE_OPTIONS
         make ossfuzz ossfuzz_proto ossfuzz_abiv2 -j4
 
@@ -97,6 +98,7 @@ defaults:
         - test/tools/ossfuzz/strictasm_opt_ossfuzz
         - test/tools/ossfuzz/yul_proto_diff_ossfuzz
         - test/tools/ossfuzz/yul_proto_ossfuzz
+        - test/tools/ossfuzz/sol_proto_ossfuzz
 
   # test result output directory
   - artifacts_test_results: &artifacts_test_results

test/tools/ossfuzz/CMakeLists.txt

@@ -10,7 +10,7 @@ add_dependencies(ossfuzz
 
 if (OSSFUZZ)
     add_custom_target(ossfuzz_proto)
-    add_dependencies(ossfuzz_proto yul_proto_ossfuzz yul_proto_diff_ossfuzz)
+    add_dependencies(ossfuzz_proto yul_proto_ossfuzz yul_proto_diff_ossfuzz sol_proto_ossfuzz)
 
     add_custom_target(ossfuzz_abiv2)
     add_dependencies(ossfuzz_abiv2 abiv2_proto_ossfuzz)
@@ -78,6 +78,25 @@ if (OSSFUZZ)
             protobuf.a
     )
     set_target_properties(abiv2_proto_ossfuzz PROPERTIES LINK_FLAGS ${LIB_FUZZING_ENGINE})
+
+    add_executable(sol_proto_ossfuzz
+            solProtoFuzzer.cpp
+            protoToSol.cpp
+            solProto.pb.cc
+            abiV2FuzzerCommon.cpp
+            ../../EVMHost.cpp
+    )
+    target_include_directories(sol_proto_ossfuzz PRIVATE
+            /usr/include/libprotobuf-mutator
+    )
+    target_link_libraries(sol_proto_ossfuzz PRIVATE solidity libsolc
+            evmc
+            evmone-standalone
+            protobuf-mutator-libfuzzer.a
+            protobuf-mutator.a
+            protobuf.a
+    )
+    set_target_properties(sol_proto_ossfuzz PROPERTIES LINK_FLAGS ${LIB_FUZZING_ENGINE})
 else()
     add_library(solc_opt_ossfuzz
             solc_opt_ossfuzz.cpp

test/tools/ossfuzz/protoToSol.cpp

@@ -46,17 +46,22 @@ pair<string, string> ProtoConverter::generateTestCase(TestContract const& _testC
 		m_libraryTest = true;
 		auto testTuple = pseudoRandomLibraryTest();
 		m_libraryName = get<0>(testTuple);
-		usingLibDecl = Whiskers(R"(
-	using <libraryName> for uint;)")
-			("libraryName", get<0>(testTuple))
-			.render();
-		testCode << Whiskers(R"(
-		uint x;
-		if (x.<testFunction>() != <expectedOutput>)
-			return 1;)")
-			("testFunction", get<1>(testTuple))
-			("expectedOutput", get<2>(testTuple))
-			.render();
+		Whiskers u(R"(<ind>using <libraryName> for uint;)");
+		u("ind", "\t");
+		u("libraryName", get<0>(testTuple));
+		usingLibDecl = u.render();
+		Whiskers test(R"(<endl><ind><varDecl><endl><ind><ifStmt>)");
+		test("endl", "\n");
+		test("ind", "\t\t");
+		test("varDecl", "uint x;");
+		Whiskers ifStmt(R"(if (<cond>)<endl><ind>return 1;)");
+		Whiskers ifCond(R"(x.<testFunction>() != <expectedOutput>)");
+		ifCond("testFunction", get<1>(testTuple));
+		ifCond("expectedOutput", get<2>(testTuple));
+		ifStmt("cond", ifCond.render());
+		ifStmt("endl", "\n");
+		ifStmt("ind", "\t\t\t");
+		test("ifStmt", ifStmt.render());
 		break;
 	}
 	case TestContract::CONTRACT:
@@ -73,21 +78,28 @@ pair<string, string> ProtoConverter::generateTestCase(TestContract const& _testC
 				break;
 			string contractName = testTuple.first;
 			string contractVarName = "tc" + to_string(contractVarIndex);
-			testCode << Whiskers(R"(
-		<contractName> <contractVarName> = new <contractName>();)")
-				("contractName", contractName)
-				("contractVarName", contractVarName)
-				.render();
+			Whiskers init(R"(<endl><ind><contractName> <contractVarName> = new <contractName>();)");
+			init("endl", "\n");
+			init("ind", "\t\t");
+			init("contractName", contractName);
+			init("contractVarName", contractVarName);
+			testCode << init.render();
 			for (auto const& t: testTuple.second)
 			{
-				testCode << Whiskers(R"(
-		if (<contractVarName>.<testFunction>() != <expectedOutput>)
-			return <errorCode>;)")
-					("contractVarName", contractVarName)
-					("testFunction", t.first)
-					("expectedOutput", t.second)
-					("errorCode", to_string(errorCode))
-					.render();
+				Whiskers tc(R"(<endl><ind><ifStmt>)");
+				tc("endl", "\n");
+				tc("ind", "\t\t");
+				Whiskers ifStmt(R"(if (<cond>)<endl><ind>return <errorCode>;)");
+				Whiskers ifCond(R"(<contractVarName>.<testFunction>() != <expectedOutput>)");
+				ifCond("contractVarName", contractVarName);
+				ifCond("testFunction", t.first);
+				ifCond("expectedOutput", t.second);
+				ifStmt("endl", "\n");
+				ifStmt("cond", ifCond.render());
+				ifStmt("ind", "\t\t\t");
+				ifStmt("errorCode", to_string(errorCode));
+				tc("ifStmt", ifStmt.render());
+				testCode << tc.render();
 				errorCode++;
 			}
 			contractVarIndex++;
@@ -96,10 +108,8 @@ pair<string, string> ProtoConverter::generateTestCase(TestContract const& _testC
 	}
 	}
 	// Expected return value when all tests pass
-	testCode << Whiskers(R"(
-		return 0;)")
-		.render();
-	return pair(usingLibDecl, testCode.str());
+	testCode << Whiskers(R"(<endl><ind>return 0;)")("endl", "\n")("ind", "\t\t").render();
+	return {usingLibDecl, testCode.str()};
 }
 
 string ProtoConverter::visit(TestContract const& _testContract)
@@ -111,23 +121,20 @@ string ProtoConverter::visit(TestContract const& _testContract)
 	// Simply return valid uint (zero) if there are
 	// no tests.
 	if (emptyLibraryTests() || emptyContractTests())
-		testCode = Whiskers(R"(
-		return 0;)")
-			.render();
+		testCode = Whiskers(R"(<endl><ind>return 0;)")("endl", "\n")("ind", "\t\t").render();
 	else
 		tie(usingLibDecl, testCode) = generateTestCase(_testContract);
 
-	return Whiskers(R"(
-contract C {<?isLibrary><usingDecl></isLibrary>
-	function test() public returns (uint)
-	{<testCode>
-	}
-}
-)")
-		("isLibrary", m_libraryTest)
-		("usingDecl", usingLibDecl)
-		("testCode", testCode)
-		.render();
+	Whiskers c(R"(<endl>contract C {<?isLibrary><usingDecl></isLibrary><endl><function><endl>})");
+	c("endl", "\n");
+	c("isLibrary", m_libraryTest);
+	c("usingDecl", usingLibDecl);
+	Whiskers f("<ind>function test() public returns (uint)<endl><ind>{<testCode><endl><ind>}");
+	f("ind", "\t");
+	f("endl", "\n");
+	f("testCode", testCode);
+	c("function", f.render());
+	return c.render();
 }
 
 bool ProtoConverter::libraryTest() const
@@ -148,17 +155,11 @@ string ProtoConverter::visit(Program const& _p)
 	for (auto &contract: _p.contracts())
 		contracts << visit(contract);
 
-	program << Whiskers(R"(
-pragma solidity >=0.0;
-
-<contracts>
-
-<testContract>
-)")
-	("contracts", contracts.str())
-	("testContract", visit(_p.test()))
-	.render();
-	return program.str();
+	Whiskers p(R"(<endl>pragma solidity >=0.0;<endl><contracts><endl><test>)");
+	p("endl", "\n");
+	p("contracts", contracts.str());
+	p("test", visit(_p.test()));
+	return p.render();
 }
 
 string ProtoConverter::visit(ContractType const& _contractType)

test/tools/ossfuzz/protoToSol.h

@@ -156,13 +156,13 @@ private:
 	/// otherwise.
 	bool emptyLibraryTests()
 	{
-		return m_libraryTests.size() == 0;
+		return m_libraryTests.empty();
 	}
 	/// @returns true if there are no valid contract test cases, false
 	/// otherwise.
 	bool emptyContractTests()
 	{
-		return m_contractTests.size() == 0;
+		return m_contractTests.empty();
 	}
 	/// Numeric suffix that is part of program names e.g., "0" in "C0"
 	unsigned m_programNumericSuffix = 0;

test/tools/ossfuzz/solProto.proto

@@ -108,4 +108,4 @@ message Program {
     required uint64 seed = 3;
 }
 
-package solidity.test.solprotofuzzer;
+package solidity.test.solprotofuzzer;

test/tools/ossfuzz/solProtoFuzzer.cpp

@@ -0,0 +1,234 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <test/tools/ossfuzz/protoToSol.h>
+#include <test/tools/ossfuzz/solProto.pb.h>
+#include <test/tools/ossfuzz/abiV2FuzzerCommon.h>
+#include <test/EVMHost.h>
+
+#include <evmone/evmone.h>
+#include <src/libfuzzer/libfuzzer_macro.h>
+
+#include <fstream>
+
+static evmc::VM evmone = evmc::VM{evmc_create_evmone()};
+
+using namespace solidity::test::abiv2fuzzer;
+using namespace solidity::test::solprotofuzzer;
+using namespace solidity;
+using namespace solidity::test;
+using namespace solidity::util;
+using namespace std;
+
+namespace
+{
+/// Test function returns a uint256 value
+static size_t const expectedOutputLength = 32;
+/// Expected output value is decimal 0
+static uint8_t const expectedOutput[expectedOutputLength] = {
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+};
+
+/// Compares the contents of the memory address pointed to
+/// by `_result` of `_length` bytes to the expected output.
+/// Returns true if `_result` matches expected output, false
+/// otherwise.
+bool isOutputExpected(evmc::result const& _run)
+{
+	if (_run.output_size != expectedOutputLength)
+		return false;
+
+	return memcmp(_run.output_data, expectedOutput, expectedOutputLength) == 0;
+}
+
+/// Accepts a reference to a user-specified input and returns an
+/// evmc_message with all of its fields zero initialized except
+/// gas and input fields.
+/// The gas field is set to the maximum permissible value so that we
+/// don't run into out of gas errors. The input field is copied from
+/// user input.
+evmc_message initializeMessage(bytes const& _input)
+{
+	// Zero initialize all message fields
+	evmc_message msg = {};
+	// Gas available (value of type int64_t) is set to its maximum
+	// value.
+	msg.gas = std::numeric_limits<int64_t>::max();
+	msg.input_data = _input.data();
+	msg.input_size = _input.size();
+	return msg;
+}
+
+/// Accepts host context implementation, and keccak256 hash of the function
+/// to be called at a specified address in the simulated blockchain as
+/// input and returns the result of the execution of the called function.
+evmc::result executeContract(
+	EVMHost& _hostContext,
+	bytes const& _functionHash,
+	evmc_address _deployedAddress
+)
+{
+	evmc_message message = initializeMessage(_functionHash);
+	message.destination = _deployedAddress;
+	message.kind = EVMC_CALL;
+	return _hostContext.call(message);
+}
+
+/// Accepts a reference to host context implementation and byte code
+/// as input and deploys it on the simulated blockchain. Returns the
+/// result of deployment.
+evmc::result deployContract(EVMHost& _hostContext, bytes const& _code)
+{
+	evmc_message message = initializeMessage(_code);
+	message.kind = EVMC_CREATE;
+	return _hostContext.call(message);
+}
+
+std::pair<bytes, Json::Value> compileContract(
+	std::string _sourceCode,
+	std::string _contractName,
+	std::map<std::string, solidity::util::h160> const& _libraryAddresses = {},
+	frontend::OptimiserSettings _optimization = frontend::OptimiserSettings::minimal()
+)
+{
+	try
+	{
+		// Compile contract generated by the proto fuzzer
+		SolidityCompilationFramework solCompilationFramework;
+		return std::make_pair(
+			solCompilationFramework.compileContract(_sourceCode, _contractName, _libraryAddresses, _optimization),
+			solCompilationFramework.getMethodIdentifiers()
+		);
+	}
+	// Ignore stack too deep errors during compilation
+	catch (evmasm::StackTooDeepException const&)
+	{
+		return std::make_pair(bytes{}, Json::Value(0));
+	}
+}
+
+evmc::result deployAndExecute(EVMHost& _hostContext, bytes _byteCode, std::string _hexEncodedInput)
+{
+	// Deploy contract and signal failure if deploy failed
+	evmc::result createResult = deployContract(_hostContext, _byteCode);
+	solAssert(
+		createResult.status_code == EVMC_SUCCESS,
+		"Proto solc fuzzer: Contract creation failed"
+	);
+
+	// Execute test function and signal failure if EVM reverted or
+	// did not return expected output on successful execution.
+	evmc::result callResult = executeContract(
+		_hostContext,
+		fromHex(_hexEncodedInput),
+		createResult.create_address
+	);
+
+	// We don't care about EVM One failures other than EVMC_REVERT
+	solAssert(callResult.status_code != EVMC_REVERT, "Proto solc fuzzer: EVM One reverted");
+	return callResult;
+}
+
+evmc::result compileDeployAndExecute(
+	std::string _sourceCode,
+	std::string _contractName,
+	std::string _methodName,
+	frontend::OptimiserSettings _optimization,
+	std::string _libraryName = {}
+)
+{
+	bytes libraryBytecode;
+	Json::Value libIds;
+	// We target the default EVM which is the latest
+	langutil::EVMVersion version = {};
+	EVMHost hostContext(version, evmone);
+	std::map<std::string, solidity::util::h160> _libraryAddressMap;
+
+	// First deploy library
+	if (!_libraryName.empty())
+	{
+		tie(libraryBytecode, libIds) = compileContract(
+			_sourceCode,
+			_libraryName,
+			{},
+			_optimization
+		);
+		// Deploy contract and signal failure if deploy failed
+		evmc::result createResult = deployContract(hostContext, libraryBytecode);
+		solAssert(
+			createResult.status_code == EVMC_SUCCESS,
+			"Proto solc fuzzer: Library deployment failed"
+		);
+		_libraryAddressMap[_libraryName] = EVMHost::convertFromEVMC(createResult.create_address);
+	}
+
+	auto [bytecode, ids] = compileContract(
+		_sourceCode,
+		_contractName,
+		_libraryAddressMap,
+		_optimization
+	);
+
+	return deployAndExecute(
+		hostContext,
+		bytecode,
+		ids[_methodName].asString()
+	);
+}
+}
+
+DEFINE_PROTO_FUZZER(Program const& _input)
+{
+	ProtoConverter converter;
+	string sol_source = converter.protoToSolidity(_input);
+
+	if (char const* dump_path = getenv("PROTO_FUZZER_DUMP_PATH"))
+	{
+		// With libFuzzer binary run this to generate a YUL source file x.yul:
+		// PROTO_FUZZER_DUMP_PATH=x.yul ./a.out proto-input
+		ofstream of(dump_path);
+		of.write(sol_source.data(), sol_source.size());
+	}
+
+	if (char const* dump_path = getenv("SOL_DEBUG_FILE"))
+	{
+		sol_source.clear();
+		// With libFuzzer binary run this to generate a YUL source file x.yul:
+		// PROTO_FUZZER_LOAD_PATH=x.yul ./a.out proto-input
+		ifstream ifstr(dump_path);
+		sol_source = {
+			std::istreambuf_iterator<char>(ifstr),
+			std::istreambuf_iterator<char>()
+		};
+		std::cout << sol_source << std::endl;
+	}
+
+	auto minimalResult = compileDeployAndExecute(
+		sol_source,
+		":C",
+		"test()",
+		frontend::OptimiserSettings::minimal(),
+		converter.libraryTest() ? converter.libraryName() : ""
+	);
+	bool successState = minimalResult.status_code == EVMC_SUCCESS;
+	if (successState)
+		solAssert(
+			isOutputExpected(minimalResult),
+			"Proto solc fuzzer: Output incorrect"
+		);
+}