From f93f9fa3a0dc1fada5688c0433a85cad1231a881 Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 16 Feb 2017 16:59:19 +0100 Subject: [PATCH 1/3] Add executable for use with AFL. --- test/CMakeLists.txt | 22 ++--------- test/fuzzer.cpp | 92 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 18 deletions(-) create mode 100644 test/fuzzer.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 609aaab36..4d56ec9d6 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -7,23 +7,9 @@ aux_source_directory(libsolidity SRC_LIST) aux_source_directory(contracts SRC_LIST) aux_source_directory(liblll SRC_LIST) -get_filename_component(TESTS_DIR "${CMAKE_CURRENT_SOURCE_DIR}" ABSOLUTE) +list(REMOVE_ITEM SRC_LIST "./fuzzer.cpp") -# search for test names and create ctest tests -enable_testing() -foreach(file ${SRC_LIST}) - file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/${file} test_list_raw REGEX "BOOST_.*TEST_(SUITE|CASE)") - set(TestSuite "DEFAULT") - foreach(test_raw ${test_list_raw}) - string(REGEX REPLACE ".*TEST_(SUITE|CASE)\\(([^ ,\\)]*).*" "\\1 \\2" test ${test_raw}) - if(test MATCHES "^SUITE .*") - string(SUBSTRING ${test} 6 -1 TestSuite) - elseif(test MATCHES "^CASE .*") - string(SUBSTRING ${test} 5 -1 TestCase) - add_test(NAME ${TestSuite}/${TestCase} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test COMMAND test -t ${TestSuite}/${TestCase}) - endif(test MATCHES "^SUITE .*") - endforeach(test_raw) -endforeach(file) +get_filename_component(TESTS_DIR "${CMAKE_CURRENT_SOURCE_DIR}" ABSOLUTE) file(GLOB HEADERS "*.h" "*/*.h") set(EXECUTABLE soltest) @@ -34,5 +20,5 @@ eth_use(${EXECUTABLE} REQUIRED Solidity::solidity Solidity::lll) include_directories(BEFORE ..) target_link_libraries(${EXECUTABLE} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARIES}) -enable_testing() -set(CTEST_OUTPUT_ON_FAILURE TRUE) +add_executable(solfuzzer fuzzer.cpp) +target_link_libraries(solfuzzer soljson) diff --git a/test/fuzzer.cpp b/test/fuzzer.cpp new file mode 100644 index 000000000..5e46662e5 --- /dev/null +++ b/test/fuzzer.cpp @@ -0,0 +1,92 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ +/** + * Executable for use with AFL . + * Reads a single source from stdin and signals a failure for internal errors. + */ + +#include + +#include +#include + +using namespace std; + +extern "C" +{ +extern char const* compileJSON(char const* _input, bool _optimize); +} + +string contains(string const& _haystack, vector const& _needles) +{ + for (string const& needle: _needles) + if (_haystack.find(needle) != string::npos) + return needle; + return ""; +} + +int main() +{ + string input; + while (!cin.eof()) + { + string s; + getline(cin, s); + input += s + '\n'; + } + + bool optimize = true; + string outputString(compileJSON(input.c_str(), optimize)); + Json::Value outputJson; + if (!Json::Reader().parse(outputString, outputJson)) + { + cout << "Compiler produced invalid JSON output." << endl; + return -1; + } + if (outputJson.isMember("errors")) + { + if (!outputJson["errors"].isArray()) + { + cout << "Output JSON has \"errors\" but it is not an array." << endl; + return -1; + } + for (Json::Value const& error: outputJson["errors"]) + { + string invalid = contains(error.asString(), vector{ + "Compiler error", + "Internal compiler error", + "Exception during compilation", + "Unknown exception during compilation", + "Unknown exception while generating contract data output", + "Unknown exception while generating formal method output", + "Unknown exception while generating source name output", + "Unknown error while generating JSON" + }); + if (!invalid.empty()) + { + cout << "Invalid error: \"" << invalid << "\"" << endl; + return -1; + } + } + } + else if (!outputJson.isMember("contracts")) + { + cout << "Output JSON has neither \"errors\" nor \"contracts\"." << endl; + return -1; + } + return 0; +} From 8be318e75bbed68520d3a3ef34f777804b6b60e0 Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 16 Feb 2017 17:13:55 +0100 Subject: [PATCH 2/3] Include non-fuzzing fuzzer tests in commandline run. --- test/cmdlineTests.sh | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/test/cmdlineTests.sh b/test/cmdlineTests.sh index fc48654a0..cb714efe2 100755 --- a/test/cmdlineTests.sh +++ b/test/cmdlineTests.sh @@ -31,7 +31,7 @@ set -e REPO_ROOT="$(dirname "$0")"/.. SOLC="$REPO_ROOT/build/solc/solc" - # Compile all files in std and examples. +# Compile all files in std and examples. for f in "$REPO_ROOT"/std/*.sol do @@ -46,6 +46,21 @@ do test -z "$output" -a "$failed" -eq 0 done -# Test library checksum -echo 'contact C {}' | "$SOLC" --link --libraries a:0x90f20564390eAe531E810af625A22f51385Cd222 -! echo 'contract C {}' | "$SOLC" --link --libraries a:0x80f20564390eAe531E810af625A22f51385Cd222 2>/dev/null +echo "Testing library checksum..." +echo '' | "$SOLC" --link --libraries a:0x90f20564390eAe531E810af625A22f51385Cd222 +! echo '' | "$SOLC" --link --libraries a:0x80f20564390eAe531E810af625A22f51385Cd222 2>/dev/null + +echo "Testing soljson via the fuzzer..." +TMPDIR=$(mktemp -d) +( + cd "$REPO_ROOT" + REPO_ROOT=$(pwd) # make it absolute + cd "$TMPDIR" + "$REPO_ROOT"/scripts/isolate_tests.py "$REPO_ROOT"/test/contracts/* "$REPO_ROOT"/test/libsolidity/*EndToEnd* + for f in *.sol + do + "$REPO_ROOT"/build/test/solfuzzer < "$f" + done +) +rm -rf "$TMPDIR" +echo "Done." From f66ebbc8e2e07bc5e1b75b6708f2f209229f6bec Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 16 Feb 2017 18:05:11 +0100 Subject: [PATCH 3/3] Report failures correctly to AFL. --- test/fuzzer.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/fuzzer.cpp b/test/fuzzer.cpp index 5e46662e5..85a8fe99b 100644 --- a/test/fuzzer.cpp +++ b/test/fuzzer.cpp @@ -55,14 +55,14 @@ int main() if (!Json::Reader().parse(outputString, outputJson)) { cout << "Compiler produced invalid JSON output." << endl; - return -1; + abort(); } if (outputJson.isMember("errors")) { if (!outputJson["errors"].isArray()) { cout << "Output JSON has \"errors\" but it is not an array." << endl; - return -1; + abort(); } for (Json::Value const& error: outputJson["errors"]) { @@ -79,14 +79,14 @@ int main() if (!invalid.empty()) { cout << "Invalid error: \"" << invalid << "\"" << endl; - return -1; + abort(); } } } else if (!outputJson.isMember("contracts")) { cout << "Output JSON has neither \"errors\" nor \"contracts\"." << endl; - return -1; + abort(); } return 0; }