Add tools: create_traces.sh, remove-testcases.py & verify-testcases.py

scripts/endToEndExtraction/create_traces.sh

@@ -0,0 +1,22 @@
+BASE_PATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 || exit ; pwd -P )"
+
+mkdir -p build
+cd build || exit
+cmake ../../../
+make soltest
+cd test/ || exit
+echo "running soltest on 'semanticTests/extracted'..."
+./soltest --color_output=false --log_level=test_suite -t semanticTests/extracted/ -- --testpath ${BASE_PATH}/../../test --no-smt --evmonepath /Users/alex/evmone/lib/libevmone.dylib --show-messages --show-metadata > ${BASE_PATH}/extracted-tests.trace
+echo "running soltest on 'semanticTests/extracted'... done"
+
+cd $BASE_PATH || exit
+git clone git@github.com:ethereum/solidity.git solidity-develop
+cd solidity-develop || exit
+mkdir -p build
+cd build || exit
+cmake ..
+make soltest
+cd test/ || exit
+echo "running soltest on 'SolidityEndToEndTest'..."
+./soltest --color_output=false --log_level=test_suite -t SolidityEndToEndTest/ -- --testpath ${BASE_PATH}/solidity-develop/test --no-smt --evmonepath /Users/alex/evmone/lib/libevmone.dylib --show-messages --show-metadata > ${BASE_PATH}/endToEndExtraction-tests.trace
+echo "running soltest on 'SolidityEndToEndTest'... done"

scripts/endToEndExtraction/remove-testcases.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+# pylint: disable=consider-using-enumerate, import-error
+
+import re
+import os
+import sys
+import getopt
+import tempfile
+from getkey import getkey
+
+
+def parse_call(call):
+    function = ''
+    arguments = ""
+    results = ""
+    search = re.search(r'// (.*):(.*)\s->\s(.*)', call, re.MULTILINE | re.DOTALL)
+    if search:
+        function = search.group(1)
+        arguments = search.group(2)
+        results = search.group(3)
+        if results.find("#") != -1:
+            results = results[:results.find("#")]
+    else:
+        search = re.search(r'// (.*)(.*)\s->\s(.*)', call, re.MULTILINE | re.DOTALL)
+        if search:
+            function = search.group(1)
+            arguments = search.group(2)
+            results = search.group(3)
+            if results.find("#") != -1:
+                results = results[:results.find("#")]
+    if function.find("wei") >= 0:
+        function = function[:function.find(",")]
+    return function.strip(), arguments.strip(), results.strip()
+
+
+def colorize(left, right, id):
+    red = "\x1b[31m"
+    yellow = "\x1b[33m"
+    reset = "\x1b[0m"
+    colors = [red, yellow]
+    color = colors[id % len(colors)]
+    function, arguments, results = parse_call(right)
+    left = left.replace("compileAndRun", color + "compileAndRun" + reset)
+    right = right.replace("constructor", color + "constructor" + reset)
+    if function:
+        left = left.replace(function, color + function + reset)
+        right = right.replace(function, color + function + reset)
+    if left.find(function):
+        bottom = " " * (left.find(function) - 4) + right
+    else:
+        bottom = " " + right
+    return "    " + left + "\n" + bottom  # " {:<90} {:<90}\n{}".format(left, right, bottom)
+
+
+def get_checks(content, sol_file_path):
+    constructors = []
+    checks = []
+    for line in content.split("\n"):
+        line = line.strip()
+        if line.startswith("compileAndRun"):
+            constructors.append(line)
+        if line.startswith("ABI_CHECK") or line.startswith("BOOST_REQUIRE"):
+            checks.append(line)
+    sol_file = open(sol_file_path, "r")
+    sol_constructors = []
+    sol_checks = []
+    inside_expectations = False
+    for line in sol_file.readlines():
+        if line.startswith("// constructor()"):
+            sol_constructors.append(line)
+        elif inside_expectations and line.startswith("// "):
+            sol_checks.append(line)
+        if line.startswith("// ----"):
+            inside_expectations = True
+    sol_file.close()
+    if len(constructors) == len(sol_constructors) == 1:
+        checks.insert(0, constructors[0])
+        sol_checks.insert(0, sol_constructors[0])
+    return checks, sol_checks
+
+
+def show_test(name, content, sol_file_path, current_test, test_count):
+    cpp_file = tempfile.NamedTemporaryFile(delete=False)
+    cpp_file.write(content.encode())
+    cpp_file.close()
+
+    os.system("clear")
+    print(str(current_test) + " / " + str(test_count) + " - " + name + "\n")
+    diff_env = os.getenv('DIFF', "/usr/local/bin/colordiff -a -d -w -y -W 200 ")
+    os.system(diff_env + " " + cpp_file.name + " " + sol_file_path)
+    os.unlink(cpp_file.name)
+    print("\n")
+
+    checks, sol_checks = get_checks(content, sol_file_path)
+
+    if len(checks) == len(sol_checks):
+        for i in range(0, len(checks)):
+            print(colorize(checks[i].strip(), sol_checks[i].strip(), i))
+    else:
+        print("warning: check count not matching. this should not happen!")
+
+    what = ""
+    print("\nContinue? (ENTER) Abort? (ANY OTHER KEY)")
+    while what != '\n':
+        what = getkey()
+        if what != '\n':
+            sys.exit(0)
+    print()
+
+
+def get_tests(e2e_path):
+    tests = []
+    for f in os.listdir(e2e_path):
+        if f.endswith(".sol"):
+            tests.append(f.replace(".sol", ""))
+    return tests
+
+
+def process_input_file(e2e_path, input_file, interactive):
+    tests = get_tests(e2e_path)
+    cpp_file = open(input_file, "r")
+    inside_test = False
+    test_name = ""
+    inside_extracted_test = False
+    new_lines = 0
+    count = 0
+    test_content = ""
+    for line in cpp_file.readlines():
+        test = re.search(r'BOOST_AUTO_TEST_CASE\((.*)\)', line, re.M | re.I)
+        if test:
+            test_name = test.group(1)
+            inside_test = True
+            inside_extracted_test = inside_test & (test_name in tests)
+            if inside_extracted_test:
+                count = count + 1
+
+        if interactive and inside_extracted_test:
+            test_content = test_content + line
+
+        if not inside_extracted_test:
+            if line == "\n":
+                new_lines = new_lines + 1
+            else:
+                new_lines = 0
+            if not interactive and new_lines <= 1:
+                sys.stdout.write(line)
+
+        if line == "}\n":
+            if interactive and inside_extracted_test:
+                show_test(test_name, test_content.strip(), e2e_path + "/" + test_name + ".sol", count, len(tests))
+                test_content = ""
+            inside_test = False
+    cpp_file.close()
+    sys.stdout.flush()
+
+
+def main(argv):
+    interactive = False
+    input_file = None
+    try:
+        opts, args = getopt.getopt(argv, "if:")
+    except getopt.GetoptError:
+        print("./remove-testcases.py [-i] [-f <full path to SolidityEndToEndTest.cpp>]")
+        sys.exit(1)
+
+    for opt, arg in opts:
+        if opt == '-i':
+            interactive = True
+        elif opt in '-f':
+            input_file = arg
+
+    base_path = os.path.dirname(__file__)
+
+    if not input_file:
+        input_file = base_path + "/../../test/libsolidity/SolidityEndToEndTest.cpp"
+
+    e2e_path = base_path + "/../../test/libsolidity/semanticTests/extracted"
+
+    process_input_file(e2e_path, input_file, interactive)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

scripts/endToEndExtraction/verify-testcases.py

@@ -0,0 +1,215 @@
+#!/usr/bin/env python3
+#
+# - SolidityEndToEndTest.trace was created with soltest with the following command on
+#     ./soltest --color_output=false --log_level=test_suite -t SolidityEndToEndTest/ -- --no-smt
+#         --evmonepath /Users/alex/evmone/lib/libevmone.dylib --show-messages > SolidityEndToEndTest.trace
+# - a trace of the semantic tests can be created by using
+#     ./soltest --color_output=false --log_level=test_suite -t semanticTests/extracted/ -- --no-smt
+#         --evmonepath /Users/alex/evmone/lib/libevmone.dylib --show-messages > semanticTests.trace
+#
+# verify-testcases.py will compare both traces. If these traces are identical, the extracted tests where
+# identical with the tests specified in SolidityEndToEndTest.cpp.
+#
+# pylint: disable=too-many-instance-attributes
+
+import re
+import os
+import sys
+import getopt
+import json
+
+
+class Trace:
+    def __init__(self, kind, parameter):
+        self.kind = kind
+        self.parameter = parameter
+        self._input = ""
+        self._output = ""
+        self.value = ""
+        self.result = ""
+        self.gas = ""
+
+    def get_input(self):
+        return self._input
+
+    def set_input(self, input):
+        if self.kind == "create":
+            # remove cbor encoded metadata from bytecode
+            length = int(input[-4:], 16) * 2
+            self._input = input[:len(input) - length - 4]
+
+    def get_output(self):
+        return self._output
+
+    def set_output(self, output):
+        if self.kind == "create":
+            # remove cbor encoded metadata from bytecode
+            length = int(output[-4:], 16) * 2
+            self._output = output[:len(output) - length - 4]
+
+    def __str__(self):
+        # we ignore the used gas
+        result = str(
+            "kind='" + self.kind + "' parameter='" + self.parameter + "' input='" + self._input +
+            "' output='" + self._output + "' value='" + self.value + "' result='" + self.result + "'"
+        )
+        return result
+
+
+class TestCase:
+    def __init__(self, name):
+        self.name = name
+        self.metadata = None
+        self.traces = []
+
+    def add_trace(self, kind, parameter):
+        trace = Trace(kind, parameter)
+        self.traces.append(trace)
+        return trace
+
+
+class TraceAnalyser:
+    def __init__(self, file):
+        self.file = file
+        self.tests = {}
+        self.ready = False
+
+    def analyse(self):
+        trace_file = open(self.file, "r")
+        trace = None
+        test_case = None
+        for line in trace_file.readlines():
+            test = re.search(r'Entering test case "(.*)"', line, re.M | re.I)
+            if test:
+                test_name = test.group(1)
+                test_case = TestCase(test_name)
+                self.tests[test_name] = test_case
+
+            metadata = re.search(r'\s*metadata:\s*(.*)$', line, re.M | re.I)
+            if metadata:
+                test_case.metadata = json.loads(metadata.group(1))
+                del test_case.metadata["sources"]
+                del test_case.metadata["compiler"]["version"]
+
+            create = re.search(r'CREATE\s*([a-fA-F0-9]*):', line, re.M | re.I)
+            if create:
+                trace = test_case.add_trace("create", create.group(1))
+
+            call = re.search(r'CALL\s*([a-fA-F0-9]*)\s*->\s*([a-fA-F0-9]*):', line, re.M | re.I)
+            if call:
+                trace = test_case.add_trace("call", call.group(1))  # + "->" + call.group(2))
+
+            if not create and not call:
+                self.parse_parameters(line, trace)
+
+        trace_file.close()
+
+        print(self.file + ":", len(self.tests), "test-cases.")
+
+        self.ready = True
+
+    @staticmethod
+    def parse_parameters(line, trace):
+        input = re.search(r'\s*in:\s*([a-fA-F0-9]*)', line, re.M | re.I)
+        if input:
+            trace.input = input.group(1)
+        output = re.search(r'\s*out:\s*([a-fA-F0-9]*)', line, re.M | re.I)
+        if output:
+            trace.output = output.group(1)
+        result = re.search(r'\s*result:\s*([a-fA-F0-9]*)', line, re.M | re.I)
+        if result:
+            trace.result = result.group(1)
+        gas_used = re.search(r'\s*gas\sused:\s*([a-fA-F0-9]*)', line, re.M | re.I)
+        if gas_used:
+            trace.gas = gas_used.group(1)
+        value = re.search(r'\s*value:\s*([a-fA-F0-9]*)', line, re.M | re.I)
+        if value:
+            trace.value = value.group(1)
+
+    def diff(self, analyser):
+        if not self.ready:
+            self.analyse()
+        if not analyser.ready:
+            analyser.analyse()
+
+        intersection = set(self.tests.keys()) & set(analyser.tests.keys())
+        mismatches = set()
+
+        for test_name in intersection:
+            left = self.tests[test_name]
+            right = analyser.tests[test_name]
+            if json.dumps(left.metadata) != json.dumps(right.metadata):
+                mismatches.add(
+                    (test_name, "metadata where different: " + json.dumps(left.metadata) + " != " + json.dumps(
+                        right.metadata)))
+            if len(left.traces) != len(right.traces):
+                mismatches.add((test_name, "trace count are different: " + str(len(left.traces)) +
+                                " != " + str(len(right.traces))))
+            else:
+                self.check_traces(test_name, left, right, mismatches)
+
+        for mismatch in mismatches:
+            print(mismatch[0])
+            print(mismatch[1])
+
+        print(len(intersection), "test-cases - ", len(mismatches), " mismatche(s)")
+
+    def check_traces(self, test_name, left, right, mismatches):
+        for trace_id in range(0, len(left.traces)):
+            left_trace = left.traces[trace_id]
+            right_trace = right.traces[trace_id]
+            assert (left_trace.kind == right_trace.kind)
+            if str(left_trace) != str(right_trace):
+                mismatch_info = "    " + str(left_trace) + "\n"
+                mismatch_info += "    " + str(right_trace) + "\n"
+                mismatch_info += "    "
+                for ch in range(0, len(str(left_trace))):
+                    if ch < len(str(left_trace)) and ch < len(str(right_trace)):
+                        if str(left_trace)[ch] != str(right_trace)[ch]:
+                            mismatch_info += "|"
+                        else:
+                            mismatch_info += " "
+                    else:
+                        mismatch_info += "|"
+                mismatch_info += "\n"
+                mismatches.add((test_name, mismatch_info))
+
+
+def main(argv):
+    extracted_tests_trace_file = None
+    end_to_end_trace_file = None
+    try:
+        opts, args = getopt.getopt(argv, "s:e:")
+    except getopt.GetoptError:
+        print("verify-testcases.py [-s <path to semantic-trace>] [-e <path to endToEndExtraction-trace>]")
+        sys.exit(2)
+
+    for opt, arg in opts:
+        if opt in '-s':
+            extracted_tests_trace_file = arg
+        elif opt in '-e':
+            end_to_end_trace_file = arg
+
+    base_path = os.path.dirname(__file__)
+    if not extracted_tests_trace_file:
+        extracted_tests_trace_file = base_path + "/extracted-tests.trace"
+    if not end_to_end_trace_file:
+        end_to_end_trace_file = base_path + "/endToEndExtraction-tests.trace"
+
+    for f in [extracted_tests_trace_file, end_to_end_trace_file]:
+        if not os.path.isfile(f):
+            print("trace file '" + f + "' not found. aborting.")
+            sys.exit(1)
+
+    if not os.path.isfile(extracted_tests_trace_file):
+        print("semantic trace file '" + extracted_tests_trace_file + "' not found. aborting.")
+        sys.exit(1)
+
+    semantic_trace = TraceAnalyser(extracted_tests_trace_file)
+    end_to_end_trace = TraceAnalyser(end_to_end_trace_file)
+
+    semantic_trace.diff(end_to_end_trace)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])