From 45a3d8228d9b0d8d796d02f268de69f66b587dff Mon Sep 17 00:00:00 2001 From: Zachinquarantine Date: Wed, 25 Jan 2023 14:54:42 -0500 Subject: [PATCH] Removed folder scripts/endToEndExtraction/ --- scripts/endToEndExtraction/create_traces.sh | 25 -- .../endToEndExtraction/remove-testcases.py | 183 --------------- .../endToEndExtraction/verify-testcases.py | 214 ------------------ 3 files changed, 422 deletions(-) delete mode 100755 scripts/endToEndExtraction/create_traces.sh delete mode 100755 scripts/endToEndExtraction/remove-testcases.py delete mode 100755 scripts/endToEndExtraction/verify-testcases.py diff --git a/scripts/endToEndExtraction/create_traces.sh b/scripts/endToEndExtraction/create_traces.sh deleted file mode 100755 index 3e1de4677..000000000 --- a/scripts/endToEndExtraction/create_traces.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -set -eu - -BASE_PATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 && pwd -P )" - -mkdir -p build -cd build -cmake ../../../ -make soltest -cd test/ -echo "running soltest on 'semanticTests/extracted'..." -./soltest --color_output=false --log_level=test_suite -t semanticTests/extracted/ -- --testpath "${BASE_PATH}/../../test" --no-smt --evmonepath /Users/alex/evmone/lib/libevmone.dylib --show-messages --show-metadata > "${BASE_PATH}/extracted-tests.trace" -echo "running soltest on 'semanticTests/extracted'... done" - -cd "$BASE_PATH" -git clone git@github.com:ethereum/solidity.git solidity-develop -cd solidity-develop -mkdir -p build -cd build -cmake .. -make soltest -cd test/ -echo "running soltest on 'SolidityEndToEndTest'..." -./soltest --color_output=false --log_level=test_suite -t SolidityEndToEndTest/ -- --testpath "${BASE_PATH}/solidity-develop/test" --no-smt --evmonepath /Users/alex/evmone/lib/libevmone.dylib --show-messages --show-metadata > "${BASE_PATH}/endToEndExtraction-tests.trace" -echo "running soltest on 'SolidityEndToEndTest'... done" diff --git a/scripts/endToEndExtraction/remove-testcases.py b/scripts/endToEndExtraction/remove-testcases.py deleted file mode 100755 index ca9a63599..000000000 --- a/scripts/endToEndExtraction/remove-testcases.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env python3 -# pylint: disable=consider-using-enumerate, import-error - -import re -import os -import sys -import getopt -import tempfile -from getkey import getkey - - -def parse_call(call): - function = '' - arguments = "" - results = "" - search = re.search(r'// (.*):(.*)\s->\s(.*)', call, re.MULTILINE | re.DOTALL) - if search: - function = search.group(1) - arguments = search.group(2) - results = search.group(3) - if results.find("#") != -1: - results = results[:results.find("#")] - else: - search = re.search(r'// (.*)(.*)\s->\s(.*)', call, re.MULTILINE | re.DOTALL) - if search: - function = search.group(1) - arguments = search.group(2) - results = search.group(3) - if results.find("#") != -1: - results = results[:results.find("#")] - if function.find("wei") >= 0: - function = function[:function.find(",")] - return function.strip(), arguments.strip(), results.strip() - - -def colorize(left, right, index): - red = "\x1b[31m" - yellow = "\x1b[33m" - reset = "\x1b[0m" - colors = [red, yellow] - color = colors[index % len(colors)] - function, _arguments, _results = parse_call(right) - left = left.replace("compileAndRun", color + "compileAndRun" + reset) - right = right.replace("constructor", color + "constructor" + reset) - if function: - left = left.replace(function, color + function + reset) - right = right.replace(function, color + function + reset) - if left.find(function): - bottom = " " * (left.find(function) - 4) + right - else: - bottom = " " + right - return " " + left + "\n" + bottom # " {:<90} {:<90}\n{}".format(left, right, bottom) - - -def get_checks(content, sol_file_path): - constructors = [] - checks = [] - for line in content.split("\n"): - line = line.strip() - if line.startswith("compileAndRun"): - constructors.append(line) - if line.startswith("ABI_CHECK") or line.startswith("BOOST_REQUIRE"): - checks.append(line) - with open(sol_file_path, "r", encoding='utf8') as sol_file: - sol_constructors = [] - sol_checks = [] - inside_expectations = False - for line in sol_file.readlines(): - if line.startswith("// constructor()"): - sol_constructors.append(line) - elif inside_expectations and line.startswith("// "): - sol_checks.append(line) - if line.startswith("// ----"): - inside_expectations = True - sol_file.close() - if len(constructors) == len(sol_constructors) == 1: - checks.insert(0, constructors[0]) - sol_checks.insert(0, sol_constructors[0]) - return checks, sol_checks - - -def show_test(name, content, sol_file_path, current_test, test_count): - with tempfile.NamedTemporaryFile(delete=False) as cpp_file: - cpp_file.write(content.encode()) - cpp_file.close() - - os.system("clear") - print(str(current_test) + " / " + str(test_count) + " - " + name + "\n") - diff_env = os.getenv('DIFF', "/usr/local/bin/colordiff -a -d -w -y -W 200 ") - os.system(diff_env + " " + cpp_file.name + " " + sol_file_path) - os.unlink(cpp_file.name) - print("\n") - - checks, sol_checks = get_checks(content, sol_file_path) - - if len(checks) == len(sol_checks): - for i in range(0, len(checks)): - print(colorize(checks[i].strip(), sol_checks[i].strip(), i)) - else: - print("warning: check count not matching. this should not happen!") - - what = "" - print("\nContinue? (ENTER) Abort? (ANY OTHER KEY)") - while what != '\n': - what = getkey() - if what != '\n': - sys.exit(0) - print() - - -def get_tests(e2e_path): - tests = [] - for f in os.listdir(e2e_path): - if f.endswith(".sol"): - tests.append(f.replace(".sol", "")) - return tests - - -def process_input_file(e2e_path, input_file, interactive): - tests = get_tests(e2e_path) - with open(input_file, "r", encoding='utf8') as cpp_file: - inside_test = False - test_name = "" - inside_extracted_test = False - new_lines = 0 - count = 0 - test_content = "" - for line in cpp_file.readlines(): - test = re.search(r'BOOST_AUTO_TEST_CASE\((.*)\)', line, re.M | re.I) - if test: - test_name = test.group(1) - inside_test = True - inside_extracted_test = inside_test & (test_name in tests) - if inside_extracted_test: - count = count + 1 - - if interactive and inside_extracted_test: - test_content = test_content + line - - if not inside_extracted_test: - if line == "\n": - new_lines = new_lines + 1 - else: - new_lines = 0 - if not interactive and new_lines <= 1: - sys.stdout.write(line) - - if line == "}\n": - if interactive and inside_extracted_test: - show_test(test_name, test_content.strip(), e2e_path + "/" + test_name + ".sol", count, len(tests)) - test_content = "" - inside_test = False - cpp_file.close() - sys.stdout.flush() - - -def main(argv): - interactive = False - input_file = None - try: - opts, _args = getopt.getopt(argv, "if:") - except getopt.GetoptError: - print("./remove-testcases.py [-i] [-f ]") - sys.exit(1) - - for opt, arg in opts: - if opt == '-i': - interactive = True - elif opt in '-f': - input_file = arg - - base_path = os.path.dirname(__file__) - - if not input_file: - input_file = base_path + "/../../test/libsolidity/SolidityEndToEndTest.cpp" - - e2e_path = base_path + "/../../test/libsolidity/semanticTests/extracted" - - process_input_file(e2e_path, input_file, interactive) - - -if __name__ == "__main__": - main(sys.argv[1:]) diff --git a/scripts/endToEndExtraction/verify-testcases.py b/scripts/endToEndExtraction/verify-testcases.py deleted file mode 100755 index b01187042..000000000 --- a/scripts/endToEndExtraction/verify-testcases.py +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env python3 -# -# - SolidityEndToEndTest.trace was created with soltest with the following command on -# ./soltest --color_output=false --log_level=test_suite -t SolidityEndToEndTest/ -- --no-smt -# --evmonepath /Users/alex/evmone/lib/libevmone.dylib --show-messages > SolidityEndToEndTest.trace -# - a trace of the semantic tests can be created by using -# ./soltest --color_output=false --log_level=test_suite -t semanticTests/extracted/ -- --no-smt -# --evmonepath /Users/alex/evmone/lib/libevmone.dylib --show-messages > semanticTests.trace -# -# verify-testcases.py will compare both traces. If these traces are identical, the extracted tests were -# identical with the tests specified in SolidityEndToEndTest.cpp. - -import re -import os -import sys -import getopt -import json - - -class Trace: - def __init__(self, kind, parameter): - self.kind = kind - self.parameter = parameter - self._input = "" - self._output = "" - self.value = "" - self.result = "" - self.gas = "" - - def get_input(self): - return self._input - - def set_input(self, bytecode): - if self.kind == "create": - # remove cbor encoded metadata from bytecode - length = int(bytecode[-4:], 16) * 2 - self._input = bytecode[:len(bytecode) - length - 4] - - def get_output(self): - return self._output - - def set_output(self, output): - if self.kind == "create": - # remove cbor encoded metadata from bytecode - length = int(output[-4:], 16) * 2 - self._output = output[:len(output) - length - 4] - - def __str__(self): - # we ignore the used gas - result = str( - "kind='" + self.kind + "' parameter='" + self.parameter + "' input='" + self._input + - "' output='" + self._output + "' value='" + self.value + "' result='" + self.result + "'" - ) - return result - - -class TestCase: - def __init__(self, name): - self.name = name - self.metadata = None - self.traces = [] - - def add_trace(self, kind, parameter): - trace = Trace(kind, parameter) - self.traces.append(trace) - return trace - - -class TraceAnalyser: - def __init__(self, file): - self.file = file - self.tests = {} - self.ready = False - - def analyse(self): - with open(self.file, "r", encoding='utf8') as trace_file: - trace = None - test_case = None - for line in trace_file.readlines(): - test = re.search(r'Entering test case "(.*)"', line, re.M | re.I) - if test: - test_name = test.group(1) - test_case = TestCase(test_name) - self.tests[test_name] = test_case - - metadata = re.search(r'\s*metadata:\s*(.*)$', line, re.M | re.I) - if metadata: - test_case.metadata = json.loads(metadata.group(1)) - del test_case.metadata["sources"] - del test_case.metadata["compiler"]["version"] - - create = re.search(r'CREATE\s*([a-fA-F0-9]*):', line, re.M | re.I) - if create: - trace = test_case.add_trace("create", create.group(1)) - - call = re.search(r'CALL\s*([a-fA-F0-9]*)\s*->\s*([a-fA-F0-9]*):', line, re.M | re.I) - if call: - trace = test_case.add_trace("call", call.group(1)) # + "->" + call.group(2)) - - if not create and not call: - self.parse_parameters(line, trace) - - trace_file.close() - - print(self.file + ":", len(self.tests), "test-cases.") - - self.ready = True - - @staticmethod - def parse_parameters(line, trace): - input_match = re.search(r'\s*in:\s*([a-fA-F0-9]*)', line, re.M | re.I) - if input_match: - trace.input = input_match.group(1) - output_match = re.search(r'\s*out:\s*([a-fA-F0-9]*)', line, re.M | re.I) - if output_match: - trace.output = output_match.group(1) - result_match = re.search(r'\s*result:\s*([a-fA-F0-9]*)', line, re.M | re.I) - if result_match: - trace.result = result_match.group(1) - gas_used_match = re.search(r'\s*gas\sused:\s*([a-fA-F0-9]*)', line, re.M | re.I) - if gas_used_match: - trace.gas = gas_used_match.group(1) - value_match = re.search(r'\s*value:\s*([a-fA-F0-9]*)', line, re.M | re.I) - if value_match: - trace.value = value_match.group(1) - - def diff(self, analyser): - if not self.ready: - self.analyse() - if not analyser.ready: - analyser.analyse() - - intersection = set(self.tests.keys()) & set(analyser.tests.keys()) - mismatches = set() - - for test_name in intersection: - left = self.tests[test_name] - right = analyser.tests[test_name] - if json.dumps(left.metadata) != json.dumps(right.metadata): - mismatches.add( - (test_name, "metadata where different: " + json.dumps(left.metadata) + " != " + json.dumps( - right.metadata))) - if len(left.traces) != len(right.traces): - mismatches.add((test_name, "trace count are different: " + str(len(left.traces)) + - " != " + str(len(right.traces)))) - else: - self.check_traces(test_name, left, right, mismatches) - - for mismatch in mismatches: - print(mismatch[0]) - print(mismatch[1]) - - print(len(intersection), "test-cases - ", len(mismatches), " mismatche(s)") - - @classmethod - def check_traces(cls, test_name, left, right, mismatches): - for trace_id, trace in enumerate(left.traces): - left_trace = trace - right_trace = right.traces[trace_id] - assert left_trace.kind == right_trace.kind - if str(left_trace) != str(right_trace): - mismatch_info = " " + str(left_trace) + "\n" - mismatch_info += " " + str(right_trace) + "\n" - mismatch_info += " " - for ch in range(0, len(str(left_trace))): - if ch < len(str(left_trace)) and ch < len(str(right_trace)): - if str(left_trace)[ch] != str(right_trace)[ch]: - mismatch_info += "|" - else: - mismatch_info += " " - else: - mismatch_info += "|" - mismatch_info += "\n" - mismatches.add((test_name, mismatch_info)) - - -def main(argv): - extracted_tests_trace_file = None - end_to_end_trace_file = None - try: - opts, _args = getopt.getopt(argv, "s:e:") - except getopt.GetoptError: - print("verify-testcases.py [-s ] [-e ]") - sys.exit(2) - - for opt, arg in opts: - if opt in '-s': - extracted_tests_trace_file = arg - elif opt in '-e': - end_to_end_trace_file = arg - - base_path = os.path.dirname(__file__) - if not extracted_tests_trace_file: - extracted_tests_trace_file = base_path + "/extracted-tests.trace" - if not end_to_end_trace_file: - end_to_end_trace_file = base_path + "/endToEndExtraction-tests.trace" - - for f in [extracted_tests_trace_file, end_to_end_trace_file]: - if not os.path.isfile(f): - print("trace file '" + f + "' not found. aborting.") - sys.exit(1) - - if not os.path.isfile(extracted_tests_trace_file): - print("semantic trace file '" + extracted_tests_trace_file + "' not found. aborting.") - sys.exit(1) - - semantic_trace = TraceAnalyser(extracted_tests_trace_file) - end_to_end_trace = TraceAnalyser(end_to_end_trace_file) - - semantic_trace.diff(end_to_end_trace) - - -if __name__ == "__main__": - main(sys.argv[1:])