Merge pull request #6475 from ethereum/ossfuzz-regression-ci

ossfuzz regression ci

.circleci/config.yml

@@ -28,6 +28,9 @@ defaults:
   - run_tests: &run_tests
       name: Tests
       command: scripts/tests.sh --junit_report test_results
+  - run_regressions: &run_regressions
+      name: Regression tests
+      command: scripts/regressions.py -o test_results
   - solc_artifact: &solc_artifact
       path: build/solc/solc
       destination: solc
@@ -37,6 +40,15 @@ defaults:
         - solc/solc
         - test/soltest
         - test/tools/solfuzzer
+  - ossfuzz_artifacts: &ossfuzz_artifacts
+      root: build
+      paths:
+        - test/tools/ossfuzz/solc_opt_ossfuzz
+        - test/tools/ossfuzz/solc_noopt_ossfuzz
+        - test/tools/ossfuzz/const_opt_ossfuzz
+        - test/tools/ossfuzz/strictasm_diff_ossfuzz
+        - test/tools/ossfuzz/yul_proto_ossfuzz
+        - test/tools/ossfuzz/yul_proto_diff_ossfuzz
 
 version: 2
 jobs:
@@ -421,6 +433,31 @@ jobs:
             ./scripts/install_libfuzzer.sh
       - run: *setup_prerelease_commit_hash
       - run: *run_build_ossfuzz
+      - persist_to_workspace: *ossfuzz_artifacts
+
+  test_x86_ossfuzz_regression:
+    docker:
+      - image: buildpack-deps:cosmic
+    environment:
+      TERM: xterm
+    steps:
+      - checkout
+      - attach_workspace:
+          at: build
+      - run:
+          name: Install dependencies
+          command: |
+            apt-get -qq update
+            apt-get -qy install libcvc4-dev llvm-7-dev
+            ./scripts/download_ossfuzz_corpus.sh
+            update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-7 1
+      - run: mkdir -p test_results
+      - run: *run_regressions
+      - store_test_results:
+          path: test_results/
+      - store_artifacts:
+          path: test_results/
+          destination: test_results/
 
 workflows:
   version: 2
@@ -484,3 +521,7 @@ workflows:
           <<: *build_on_tags
           requires:
             - build_emscripten
+      - test_x86_ossfuzz_regression:
+          <<: *build_on_tags
+          requires:
+            - build_x86_linux_ossfuzz

scripts/download_ossfuzz_corpus.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+set -e
+cd /tmp
+git clone --depth 1 https://github.com/ethereum/solidity-fuzzing-corpus.git

scripts/regressions.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+
+from argparse import ArgumentParser
+import sys
+import shutil
+import os
+import subprocess
+import re
+import glob
+
+DESCRIPTION = """Regressor is a tool to run regression tests in a CI env."""
+
+
+class regressor():
+    _re_sanitizer_log = re.compile(r"""ERROR: (?P<sanitizer>\w+).*""")
+
+    def __init__(self, description, args):
+        self._description = description
+        self._args = self.parseCmdLine(description, args)
+        self._repo_root = os.path.dirname(sys.path[0])
+        self._fuzzer_path = os.path.join(self._repo_root,
+                                         "build/test/tools/ossfuzz")
+        self._logpath = os.path.join(self._repo_root, "test_results")
+
+    def parseCmdLine(self, description, args):
+        argParser = ArgumentParser(description)
+        argParser.add_argument('-o', '--out-dir', required=True, type=str,
+                       help="""Directory where test results will be written""")
+        return argParser.parse_args(args)
+
+    @staticmethod
+    def run_cmd(command, logfile=None, env=None):
+        if not logfile:
+            logfile = os.devnull
+
+        if not env:
+            env = os.environ.copy()
+
+        logfh = open(logfile, 'w')
+        proc = subprocess.Popen(command, shell=True, executable='/bin/bash',
+                                env=env, stdout=logfh,
+                                stderr=subprocess.STDOUT)
+        ret = proc.wait()
+        logfh.close()
+
+        if ret != 0:
+            return False
+        return True
+
+    def process_log(self, logfile):
+        list = re.findall(self._re_sanitizer_log, open(logfile, 'r').read())
+        numSuppressedLeaks = list.count("LeakSanitizer")
+        return "AddressSanitizer" not in list, numSuppressedLeaks
+
+    def run(self):
+        for fuzzer in glob.iglob("{}/*_ossfuzz".format(self._fuzzer_path)):
+            basename = os.path.basename(fuzzer)
+            logfile = os.path.join(self._logpath, "{}.log".format(basename))
+            corpus_dir = "/tmp/solidity-fuzzing-corpus/{0}_seed_corpus" \
+                .format(basename)
+            cmd = "find {0} -type f | xargs {1}".format(corpus_dir, fuzzer)
+            if not self.run_cmd(cmd, logfile=logfile):
+                ret, numLeaks = self.process_log(logfile)
+                if not ret:
+                    print(
+                        "\t[-] AddressSanitizer reported failure for {0}. "
+                        "Failure logged to test_results".format(
+                            basename))
+                    return False
+                else:
+                    print("\t[+] {0} passed regression tests but leaked "
+                          "memory.".format(basename))
+                    print("\t\t[+] Suppressed {0} memory leak reports".format(
+                            numLeaks))
+            else:
+                print("\t[+] {0} passed regression tests.".format(basename))
+        return True
+
+
+if __name__ == '__main__':
+    tool = regressor(DESCRIPTION, sys.argv[1:])
+    tool.run()