Merge pull request #1703 from ethereum/fuzzer

Executable for use with AFL

test/CMakeLists.txt

@@ -7,23 +7,9 @@ aux_source_directory(libsolidity SRC_LIST)
 aux_source_directory(contracts SRC_LIST)
 aux_source_directory(liblll SRC_LIST)
 
-get_filename_component(TESTS_DIR "${CMAKE_CURRENT_SOURCE_DIR}" ABSOLUTE)
+list(REMOVE_ITEM SRC_LIST "./fuzzer.cpp")
 
-# search for test names and create ctest tests
+get_filename_component(TESTS_DIR "${CMAKE_CURRENT_SOURCE_DIR}" ABSOLUTE)
-enable_testing()
-foreach(file ${SRC_LIST})
-	file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/${file} test_list_raw REGEX "BOOST_.*TEST_(SUITE|CASE)")
-	set(TestSuite "DEFAULT")
-	foreach(test_raw ${test_list_raw})
-		string(REGEX REPLACE ".*TEST_(SUITE|CASE)\\(([^ ,\\)]*).*" "\\1 \\2" test ${test_raw})
-		if(test MATCHES "^SUITE .*")
-			string(SUBSTRING ${test} 6 -1 TestSuite)
-		elseif(test MATCHES "^CASE .*")
-			string(SUBSTRING ${test} 5 -1 TestCase)
-			add_test(NAME ${TestSuite}/${TestCase} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test COMMAND test -t ${TestSuite}/${TestCase})
-		endif(test MATCHES "^SUITE .*")
-	endforeach(test_raw)
-endforeach(file)
 
 file(GLOB HEADERS "*.h" "*/*.h")
 set(EXECUTABLE soltest)
@@ -34,5 +20,5 @@ eth_use(${EXECUTABLE} REQUIRED Solidity::solidity Solidity::lll)
 include_directories(BEFORE ..)
 target_link_libraries(${EXECUTABLE} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARIES})
 
-enable_testing()
+add_executable(solfuzzer fuzzer.cpp)
-set(CTEST_OUTPUT_ON_FAILURE TRUE)
+target_link_libraries(solfuzzer soljson)

test/cmdlineTests.sh

@@ -46,6 +46,21 @@ do
     test -z "$output" -a "$failed" -eq 0
 done
 
-# Test library checksum
+echo "Testing library checksum..."
-echo 'contact C {}' | "$SOLC" --link --libraries a:0x90f20564390eAe531E810af625A22f51385Cd222
+echo '' | "$SOLC" --link --libraries a:0x90f20564390eAe531E810af625A22f51385Cd222
-! echo 'contract C {}' | "$SOLC" --link --libraries a:0x80f20564390eAe531E810af625A22f51385Cd222 2>/dev/null
+! echo '' | "$SOLC" --link --libraries a:0x80f20564390eAe531E810af625A22f51385Cd222 2>/dev/null
+
+echo "Testing soljson via the fuzzer..."
+TMPDIR=$(mktemp -d)
+(
+    cd "$REPO_ROOT"
+    REPO_ROOT=$(pwd) # make it absolute
+    cd "$TMPDIR"
+    "$REPO_ROOT"/scripts/isolate_tests.py "$REPO_ROOT"/test/contracts/* "$REPO_ROOT"/test/libsolidity/*EndToEnd*
+    for f in *.sol
+    do
+        "$REPO_ROOT"/build/test/solfuzzer < "$f"
+    done
+)
+rm -rf "$TMPDIR"
+echo "Done."

test/fuzzer.cpp

@@ -0,0 +1,92 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * Executable for use with AFL <http://lcamtuf.coredump.cx/afl>.
+ * Reads a single source from stdin and signals a failure for internal errors.
+ */
+
+#include <json/json.h>
+
+#include <string>
+#include <iostream>
+
+using namespace std;
+
+extern "C"
+{
+extern char const* compileJSON(char const* _input, bool _optimize);
+}
+
+string contains(string const& _haystack, vector<string> const& _needles)
+{
+	for (string const& needle: _needles)
+		if (_haystack.find(needle) != string::npos)
+			return needle;
+	return "";
+}
+
+int main()
+{
+	string input;
+	while (!cin.eof())
+	{
+		string s;
+		getline(cin, s);
+		input += s + '\n';
+	}
+
+	bool optimize = true;
+	string outputString(compileJSON(input.c_str(), optimize));
+	Json::Value outputJson;
+	if (!Json::Reader().parse(outputString, outputJson))
+	{
+		cout << "Compiler produced invalid JSON output." << endl;
+		abort();
+	}
+	if (outputJson.isMember("errors"))
+	{
+		if (!outputJson["errors"].isArray())
+		{
+			cout << "Output JSON has \"errors\" but it is not an array." << endl;
+			abort();
+		}
+		for (Json::Value const& error: outputJson["errors"])
+		{
+			string invalid = contains(error.asString(), vector<string>{
+				"Compiler error",
+				"Internal compiler error",
+				"Exception during compilation",
+				"Unknown exception during compilation",
+				"Unknown exception while generating contract data output",
+				"Unknown exception while generating formal method output",
+				"Unknown exception while generating source name output",
+				"Unknown error while generating JSON"
+			});
+			if (!invalid.empty())
+			{
+				cout << "Invalid error: \"" << invalid << "\"" << endl;
+				abort();
+			}
+		}
+	}
+	else if (!outputJson.isMember("contracts"))
+	{
+		cout << "Output JSON has neither \"errors\" nor \"contracts\"." << endl;
+		abort();
+	}
+	return 0;
+}