Add a nightly run for ossfuzz regression tests

This commit is contained in:
Bhargava Shastry 2019-04-04 23:04:49 +02:00
parent dd15c24320
commit 33ccc19619
3 changed files with 127 additions and 0 deletions

View File

@ -28,6 +28,9 @@ defaults:
- run_tests: &run_tests
name: Tests
command: scripts/tests.sh --junit_report test_results
- run_regressions: &run_regressions
name: Regression tests
command: scripts/regressions.py -o test_results
- solc_artifact: &solc_artifact
path: build/solc/solc
destination: solc
@ -37,6 +40,15 @@ defaults:
- solc/solc
- test/soltest
- test/tools/solfuzzer
- ossfuzz_artifacts: &ossfuzz_artifacts
root: build
paths:
- test/tools/ossfuzz/solc_opt_ossfuzz
- test/tools/ossfuzz/solc_noopt_ossfuzz
- test/tools/ossfuzz/const_opt_ossfuzz
- test/tools/ossfuzz/strictasm_diff_ossfuzz
- test/tools/ossfuzz/yul_proto_ossfuzz
- test/tools/ossfuzz/yul_proto_diff_ossfuzz
version: 2
jobs:
@ -414,6 +426,31 @@ jobs:
./scripts/install_libfuzzer.sh
- run: *setup_prerelease_commit_hash
- run: *run_build_ossfuzz
- persist_to_workspace: *ossfuzz_artifacts
test_x86_ossfuzz_regression:
docker:
- image: buildpack-deps:cosmic
environment:
TERM: xterm
steps:
- checkout
- attach_workspace:
at: build
- run:
name: Install dependencies
command: |
apt-get -qq update
apt-get -qy install libcvc4-dev llvm-7-dev
./scripts/download_ossfuzz_corpus.sh
update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-7 1
- run: mkdir -p test_results
- run: *run_regressions
- store_test_results:
path: test_results/
- store_artifacts:
path: test_results/
destination: test_results/
workflows:
version: 2
@ -479,3 +516,7 @@ workflows:
<<: *build_on_tags
requires:
- build_emscripten
- test_x86_ossfuzz_regression:
<<: *build_on_tags
requires:
- build_x86_linux_ossfuzz

View File

@ -0,0 +1,4 @@
#!/usr/bin/env sh
set -e
cd /tmp
git clone --depth 1 https://github.com/ethereum/solidity-fuzzing-corpus.git

82
scripts/regressions.py Executable file
View File

@ -0,0 +1,82 @@
#!/usr/bin/env python3
from argparse import ArgumentParser
import sys
import shutil
import os
import subprocess
import re
import glob
DESCRIPTION = """Regressor is a tool to run regression tests in a CI env."""
class regressor():
_re_sanitizer_log = re.compile(r"""ERROR: (?P<sanitizer>\w+).*""")
def __init__(self, description, args):
self._description = description
self._args = self.parseCmdLine(description, args)
self._repo_root = os.path.dirname(sys.path[0])
self._fuzzer_path = os.path.join(self._repo_root,
"build/test/tools/ossfuzz")
self._logpath = os.path.join(self._repo_root, "test_results")
def parseCmdLine(self, description, args):
argParser = ArgumentParser(description)
argParser.add_argument('-o', '--out-dir', required=True, type=str,
help="""Directory where test results will be written""")
return argParser.parse_args(args)
@staticmethod
def run_cmd(command, logfile=None, env=None):
if not logfile:
logfile = os.devnull
if not env:
env = os.environ.copy()
logfh = open(logfile, 'w')
proc = subprocess.Popen(command, shell=True, executable='/bin/bash',
env=env, stdout=logfh,
stderr=subprocess.STDOUT)
ret = proc.wait()
logfh.close()
if ret != 0:
return False
return True
def process_log(self, logfile):
list = re.findall(self._re_sanitizer_log, open(logfile, 'r').read())
numSuppressedLeaks = list.count("LeakSanitizer")
return "AddressSanitizer" not in list, numSuppressedLeaks
def run(self):
for fuzzer in glob.iglob("{}/*_ossfuzz".format(self._fuzzer_path)):
basename = os.path.basename(fuzzer)
logfile = os.path.join(self._logpath, "{}.log".format(basename))
corpus_dir = "/tmp/solidity-fuzzing-corpus/{0}_seed_corpus" \
.format(basename)
cmd = "find {0} -type f | xargs {1}".format(corpus_dir, fuzzer)
if not self.run_cmd(cmd, logfile=logfile):
ret, numLeaks = self.process_log(logfile)
if not ret:
print(
"\t[-] AddressSanitizer reported failure for {0}. "
"Failure logged to test_results".format(
basename))
return False
else:
print("\t[+] {0} passed regression tests but leaked "
"memory.".format(basename))
print("\t\t[+] Suppressed {0} memory leak reports".format(
numLeaks))
else:
print("\t[+] {0} passed regression tests.".format(basename))
return True
if __name__ == '__main__':
tool = regressor(DESCRIPTION, sys.argv[1:])
tool.run()