Merge pull request #8336 from ethereum/bugfix-regressions

Fix script used for fuzzer nightly test

.circleci/config.yml

@@ -413,6 +413,7 @@ jobs:
       - run:
           name: Regression tests
           command: |
+            git clone https://github.com/ethereum/solidity-fuzzing-corpus /tmp/solidity-fuzzing-corpus
             mkdir -p test_results
             scripts/regressions.py -o test_results
       - run: *gitter_notify_failure

scripts/regressions.py

@@ -31,7 +31,7 @@ class PrintDotsThread(object):
             time.sleep(self.interval)
 
 class regressor():
-    _re_sanitizer_log = re.compile(r"""(.*runtime error: (?P<sanitizer>\w+).*|std::exception::what: (?P<description>\w+).*)""")
+    _re_sanitizer_log = re.compile(r"""ERROR: (libFuzzer|UndefinedBehaviorSanitizer)""")
 
     def __init__(self, description, args):
         self._description = description
@@ -44,7 +44,7 @@ class regressor():
     def parseCmdLine(self, description, args):
         argParser = ArgumentParser(description)
         argParser.add_argument('-o', '--out-dir', required=True, type=str,
-                       help="""Directory where test results will be written""")
+                               help="""Directory where test results will be written""")
         return argParser.parse_args(args)
 
     @staticmethod
@@ -89,8 +89,7 @@ class regressor():
         ## Log may contain non ASCII characters, so we simply stringify them
         ## since they don't matter for regular expression matching
         rawtext = str(open(logfile, 'rb').read())
-        list = re.findall(self._re_sanitizer_log, rawtext)
-        return len(list) == 0
+        return not re.search(self._re_sanitizer_log, rawtext)
 
     def run(self):
         """
@@ -106,29 +105,22 @@ class regressor():
             logfile = os.path.join(self._logpath, "{}.log".format(basename))
             corpus_dir = "/tmp/solidity-fuzzing-corpus/{0}_seed_corpus" \
                 .format(basename)
-            cmd = "find {0} -type f | xargs -n1 {1}".format(corpus_dir, fuzzer)
-            cmd_status = self.run_cmd(cmd, logfile=logfile)
-            if cmd_status:
-                ret, numLeaks = self.process_log(logfile)
-                if not ret:
-                    print(
-                        "\t[-] libFuzzer reported failure for {0}. "
-                        "Failure logged to test_results".format(
-                            basename))
-                    testStatus.append(cmd_status)
-                else:
-                    print("\t[+] {0} passed regression tests but leaked "
-                          "memory.".format(basename))
-                    print("\t\t[+] Suppressed {0} memory leak reports".format(
-                            numLeaks))
-                    testStatus.append(0)
+            cmd = "find {0} -type f | xargs -n1 sh -c '{1} $0 || exit 255'".format(corpus_dir, fuzzer)
+            self.run_cmd(cmd, logfile=logfile)
+            ret = self.process_log(logfile)
+            if not ret:
+                print(
+                    "\t[-] libFuzzer reported failure for {0}. "
+                    "Failure logged to test_results".format(
+                        basename))
+                testStatus.append(False)
             else:
                 print("\t[+] {0} passed regression tests.".format(basename))
-                testStatus.append(cmd_status)
-        return any(testStatus)
+                testStatus.append(True)
+        return all(testStatus)
 
 
 if __name__ == '__main__':
     dotprinter = PrintDotsThread()
     tool = regressor(DESCRIPTION, sys.argv[1:])
-    sys.exit(tool.run())
+    sys.exit(not tool.run())