Merge pull request #9722 from ethereum/fix-9676

Add Solidity test case generator framework

.circleci/config.yml

@@ -91,7 +91,9 @@ defaults:
       paths:
         - test/tools/ossfuzz/abiv2_proto_ossfuzz
         - test/tools/ossfuzz/const_opt_ossfuzz
+        - test/tools/ossfuzz/solc_noopt_mutator_ossfuzz
         - test/tools/ossfuzz/solc_noopt_ossfuzz
+        - test/tools/ossfuzz/solc_opt_mutator_ossfuzz
         - test/tools/ossfuzz/solc_opt_ossfuzz
         - test/tools/ossfuzz/strictasm_assembly_ossfuzz
         - test/tools/ossfuzz/strictasm_diff_ossfuzz

test/tools/ossfuzz/CMakeLists.txt

@@ -1,12 +1,14 @@
 add_custom_target(ossfuzz)
 add_dependencies(ossfuzz
         solc_opt_ossfuzz
+        solc_opt_mutator_ossfuzz
         solc_noopt_ossfuzz
+        solc_noopt_mutator_ossfuzz
         const_opt_ossfuzz
         strictasm_diff_ossfuzz
         strictasm_opt_ossfuzz
         strictasm_assembly_ossfuzz
-        )
+)
 
 
 if (OSSFUZZ)
@@ -31,6 +33,16 @@ if (OSSFUZZ)
     target_link_libraries(solc_opt_ossfuzz PRIVATE libsolc evmasm)
     set_target_properties(solc_opt_ossfuzz PROPERTIES LINK_FLAGS ${LIB_FUZZING_ENGINE})
 
+    add_executable(solc_opt_mutator_ossfuzz
+            solc_opt_ossfuzz.cpp
+            ../fuzzer_common.cpp
+            ../../TestCaseReader.cpp
+            SolidityGenerator.cpp
+            SolidityCustomMutatorInterface.cpp
+    )
+    target_link_libraries(solc_opt_mutator_ossfuzz PRIVATE libsolc evmasm)
+    set_target_properties(solc_opt_mutator_ossfuzz PROPERTIES LINK_FLAGS ${LIB_FUZZING_ENGINE})
+
     add_executable(solc_noopt_ossfuzz
             solc_noopt_ossfuzz.cpp
             ../fuzzer_common.cpp
@@ -39,6 +51,16 @@ if (OSSFUZZ)
     target_link_libraries(solc_noopt_ossfuzz PRIVATE libsolc evmasm)
     set_target_properties(solc_noopt_ossfuzz PROPERTIES LINK_FLAGS ${LIB_FUZZING_ENGINE})
 
+    add_executable(solc_noopt_mutator_ossfuzz
+            solc_noopt_ossfuzz.cpp
+            ../fuzzer_common.cpp
+            ../../TestCaseReader.cpp
+            SolidityGenerator.cpp
+            SolidityCustomMutatorInterface.cpp
+    )
+    target_link_libraries(solc_noopt_mutator_ossfuzz PRIVATE libsolc evmasm)
+    set_target_properties(solc_noopt_mutator_ossfuzz PROPERTIES LINK_FLAGS ${LIB_FUZZING_ENGINE})
+
     add_executable(const_opt_ossfuzz const_opt_ossfuzz.cpp ../fuzzer_common.cpp)
     target_link_libraries(const_opt_ossfuzz PRIVATE libsolc evmasm)
     set_target_properties(const_opt_ossfuzz PROPERTIES LINK_FLAGS ${LIB_FUZZING_ENGINE})
@@ -82,14 +104,14 @@ if (OSSFUZZ)
             protoToYul.cpp
             yulProto.pb.cc
             protomutators/YulProtoMutator.cpp
-)
+    )
     target_include_directories(yul_proto_diff_custom_mutate_ossfuzz PRIVATE /usr/include/libprotobuf-mutator)
     target_link_libraries(yul_proto_diff_custom_mutate_ossfuzz PRIVATE yul
             yulInterpreter
             protobuf-mutator-libfuzzer.a
             protobuf-mutator.a
             protobuf.a
-)
+    )
     set_target_properties(yul_proto_diff_custom_mutate_ossfuzz PROPERTIES LINK_FLAGS ${LIB_FUZZING_ENGINE})
     target_compile_options(yul_proto_diff_custom_mutate_ossfuzz PUBLIC ${COMPILE_OPTIONS} -Wno-sign-conversion)
 
@@ -139,12 +161,24 @@ else()
             )
     target_link_libraries(solc_opt_ossfuzz PRIVATE libsolc evmasm)
 
+    add_library(solc_opt_mutator_ossfuzz
+            solc_opt_ossfuzz.cpp
+            ../fuzzer_common.cpp
+            )
+    target_link_libraries(solc_opt_mutator_ossfuzz PRIVATE libsolc evmasm)
+
     add_library(solc_noopt_ossfuzz
             solc_noopt_ossfuzz.cpp
             ../fuzzer_common.cpp
             )
     target_link_libraries(solc_noopt_ossfuzz PRIVATE libsolc evmasm)
 
+    add_library(solc_noopt_mutator_ossfuzz
+            solc_noopt_ossfuzz.cpp
+            ../fuzzer_common.cpp
+            )
+    target_link_libraries(solc_noopt_mutator_ossfuzz PRIVATE libsolc evmasm)
+
     add_library(const_opt_ossfuzz
             const_opt_ossfuzz.cpp
             ../fuzzer_common.cpp)

test/tools/ossfuzz/SolidityCustomMutatorInterface.cpp

@@ -0,0 +1,69 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+// SPDX-License-Identifier: GPL-3.0
+
+#include <test/tools/ossfuzz/SolidityCustomMutatorInterface.h>
+#include <test/tools/ossfuzz/SolidityGenerator.h>
+
+#include <liblangutil/Exceptions.h>
+
+using namespace std;
+using namespace solidity::test::fuzzer;
+
+namespace
+{
+/// Forward declare libFuzzer's default mutator definition
+extern "C" size_t LLVMFuzzerMutate(uint8_t* _data, size_t _size, size_t _maxSize);
+
+/// Define Solidity's custom mutator by implementing libFuzzer's
+/// custom mutator external interface.
+extern "C" size_t LLVMFuzzerCustomMutator(
+	uint8_t* _data,
+	size_t _size,
+	size_t _maxSize,
+	unsigned int _seed
+)
+{
+	if (_maxSize <= _size || _size == 0)
+		return LLVMFuzzerMutate(_data, _size, _maxSize);
+	return SolidityCustomMutatorInterface{_data, _size, _maxSize, _seed}.generate();
+}
+}
+
+SolidityCustomMutatorInterface::SolidityCustomMutatorInterface(
+	uint8_t* _data,
+	size_t _size,
+	size_t _maxSize,
+	unsigned int _seed
+	):
+	data(_data),
+	size(_size),
+	maxMutantSize(_maxSize),
+	generator(make_shared<SolidityGenerator>(_seed))
+{}
+
+size_t SolidityCustomMutatorInterface::generate()
+{
+	string testCase = generator->generateTestProgram();
+	solAssert(
+		!testCase.empty() && data,
+		"Solc custom mutator: Invalid mutant or memory pointer"
+	);
+	size_t mutantSize = min(testCase.size(), maxMutantSize - 1);
+	mempcpy(data, testCase.data(), mutantSize);
+	return mutantSize;
+}

test/tools/ossfuzz/SolidityCustomMutatorInterface.h

@@ -0,0 +1,46 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+// SPDX-License-Identifier: GPL-3.0
+/**
+ * Implements libFuzzer's custom mutator interface.
+ */
+
+#pragma once
+
+#include <test/tools/ossfuzz/SolidityGenerator.h>
+
+#include <memory>
+
+namespace solidity::test::fuzzer
+{
+struct SolidityCustomMutatorInterface
+{
+	SolidityCustomMutatorInterface(uint8_t* _data, size_t _size, size_t _maxSize, unsigned _seed);
+	/// Generates Solidity test program, copies it into buffer
+	/// provided by libFuzzer and @returns size of the test program.
+	size_t generate();
+
+	/// Raw pointer to libFuzzer provided input
+	uint8_t* data;
+	/// Size of libFuzzer provided input
+	size_t size;
+	/// Maximum length of mutant specified by libFuzzer
+	size_t maxMutantSize;
+	/// Solidity generator handle
+	std::shared_ptr<SolidityGenerator> generator;
+};
+}

test/tools/ossfuzz/SolidityGenerator.cpp

@@ -0,0 +1,35 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+// SPDX-License-Identifier: GPL-3.0
+
+#include <test/tools/ossfuzz/SolidityGenerator.h>
+
+#include <libsolutil/Whiskers.h>
+
+using namespace solidity::test::fuzzer;
+using namespace solidity::util;
+using namespace std;
+
+string SolidityGenerator::generateTestProgram()
+{
+	// TODO: Add generators for grammar elements of
+	// Solidity antlr4 grammar. Currently, the generated
+	// test program consists of a version pragma only.
+	return Whiskers(R"(pragma <directive>;)")
+		("directive", "solidity >= 0.0.0")
+		.render();
+}

test/tools/ossfuzz/SolidityGenerator.h

@@ -0,0 +1,42 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+// SPDX-License-Identifier: GPL-3.0
+/**
+ * Implements generators for synthesizing mostly syntactically valid
+ * Solidity test programs.
+ */
+
+#pragma once
+
+#include <random>
+
+namespace solidity::test::fuzzer
+{
+using RandomEngine = std::mt19937_64;
+
+class SolidityGenerator
+{
+public:
+	SolidityGenerator(uint64_t _seed): m_rand(_seed)
+	{}
+	/// @returns a pseudo randomly generated test program
+	std::string generateTestProgram();
+private:
+	/// Random number generator
+	RandomEngine const m_rand;
+};
+}