mirror of
https://github.com/ethereum/solidity
synced 2023-10-03 13:03:40 +00:00
Merge pull request #13888 from Zachinquarantine/remove-verifytestcases
Remove obsolete endToEndExtraction script
This commit is contained in:
Kamil Śliwak
GitHub
commit
13c0d93f81
@ -1,25 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
|
||||
BASE_PATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 && pwd -P )"
|
||||
|
||||
mkdir -p build
|
||||
cd build
|
||||
cmake ../../../
|
||||
make soltest
|
||||
cd test/
|
||||
echo "running soltest on 'semanticTests/extracted'..."
|
||||
./soltest --color_output=false --log_level=test_suite -t semanticTests/extracted/ -- --testpath "${BASE_PATH}/../../test" --no-smt --evmonepath /Users/alex/evmone/lib/libevmone.dylib --show-messages --show-metadata > "${BASE_PATH}/extracted-tests.trace"
|
||||
echo "running soltest on 'semanticTests/extracted'... done"
|
||||
|
||||
cd "$BASE_PATH"
|
||||
git clone git@github.com:ethereum/solidity.git solidity-develop
|
||||
cd solidity-develop
|
||||
mkdir -p build
|
||||
cd build
|
||||
cmake ..
|
||||
make soltest
|
||||
cd test/
|
||||
echo "running soltest on 'SolidityEndToEndTest'..."
|
||||
./soltest --color_output=false --log_level=test_suite -t SolidityEndToEndTest/ -- --testpath "${BASE_PATH}/solidity-develop/test" --no-smt --evmonepath /Users/alex/evmone/lib/libevmone.dylib --show-messages --show-metadata > "${BASE_PATH}/endToEndExtraction-tests.trace"
|
||||
echo "running soltest on 'SolidityEndToEndTest'... done"
|
||||
@ -1,183 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# pylint: disable=consider-using-enumerate, import-error
|
||||
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
import getopt
|
||||
import tempfile
|
||||
from getkey import getkey
|
||||
|
||||
|
||||
def parse_call(call):
|
||||
function = ''
|
||||
arguments = ""
|
||||
results = ""
|
||||
search = re.search(r'// (.*):(.*)\s->\s(.*)', call, re.MULTILINE | re.DOTALL)
|
||||
if search:
|
||||
function = search.group(1)
|
||||
arguments = search.group(2)
|
||||
results = search.group(3)
|
||||
if results.find("#") != -1:
|
||||
results = results[:results.find("#")]
|
||||
else:
|
||||
search = re.search(r'// (.*)(.*)\s->\s(.*)', call, re.MULTILINE | re.DOTALL)
|
||||
if search:
|
||||
function = search.group(1)
|
||||
arguments = search.group(2)
|
||||
results = search.group(3)
|
||||
if results.find("#") != -1:
|
||||
results = results[:results.find("#")]
|
||||
if function.find("wei") >= 0:
|
||||
function = function[:function.find(",")]
|
||||
return function.strip(), arguments.strip(), results.strip()
|
||||
|
||||
|
||||
def colorize(left, right, index):
|
||||
red = "\x1b[31m"
|
||||
yellow = "\x1b[33m"
|
||||
reset = "\x1b[0m"
|
||||
colors = [red, yellow]
|
||||
color = colors[index % len(colors)]
|
||||
function, _arguments, _results = parse_call(right)
|
||||
left = left.replace("compileAndRun", color + "compileAndRun" + reset)
|
||||
right = right.replace("constructor", color + "constructor" + reset)
|
||||
if function:
|
||||
left = left.replace(function, color + function + reset)
|
||||
right = right.replace(function, color + function + reset)
|
||||
if left.find(function):
|
||||
bottom = " " * (left.find(function) - 4) + right
|
||||
else:
|
||||
bottom = " " + right
|
||||
return " " + left + "\n" + bottom # " {:<90} {:<90}\n{}".format(left, right, bottom)
|
||||
|
||||
|
||||
def get_checks(content, sol_file_path):
|
||||
constructors = []
|
||||
checks = []
|
||||
for line in content.split("\n"):
|
||||
line = line.strip()
|
||||
if line.startswith("compileAndRun"):
|
||||
constructors.append(line)
|
||||
if line.startswith("ABI_CHECK") or line.startswith("BOOST_REQUIRE"):
|
||||
checks.append(line)
|
||||
with open(sol_file_path, "r", encoding='utf8') as sol_file:
|
||||
sol_constructors = []
|
||||
sol_checks = []
|
||||
inside_expectations = False
|
||||
for line in sol_file.readlines():
|
||||
if line.startswith("// constructor()"):
|
||||
sol_constructors.append(line)
|
||||
elif inside_expectations and line.startswith("// "):
|
||||
sol_checks.append(line)
|
||||
if line.startswith("// ----"):
|
||||
inside_expectations = True
|
||||
sol_file.close()
|
||||
if len(constructors) == len(sol_constructors) == 1:
|
||||
checks.insert(0, constructors[0])
|
||||
sol_checks.insert(0, sol_constructors[0])
|
||||
return checks, sol_checks
|
||||
|
||||
|
||||
def show_test(name, content, sol_file_path, current_test, test_count):
|
||||
with tempfile.NamedTemporaryFile(delete=False) as cpp_file:
|
||||
cpp_file.write(content.encode())
|
||||
cpp_file.close()
|
||||
|
||||
os.system("clear")
|
||||
print(str(current_test) + " / " + str(test_count) + " - " + name + "\n")
|
||||
diff_env = os.getenv('DIFF', "/usr/local/bin/colordiff -a -d -w -y -W 200 ")
|
||||
os.system(diff_env + " " + cpp_file.name + " " + sol_file_path)
|
||||
os.unlink(cpp_file.name)
|
||||
print("\n")
|
||||
|
||||
checks, sol_checks = get_checks(content, sol_file_path)
|
||||
|
||||
if len(checks) == len(sol_checks):
|
||||
for i in range(0, len(checks)):
|
||||
print(colorize(checks[i].strip(), sol_checks[i].strip(), i))
|
||||
else:
|
||||
print("warning: check count not matching. this should not happen!")
|
||||
|
||||
what = ""
|
||||
print("\nContinue? (ENTER) Abort? (ANY OTHER KEY)")
|
||||
while what != '\n':
|
||||
what = getkey()
|
||||
if what != '\n':
|
||||
sys.exit(0)
|
||||
print()
|
||||
|
||||
|
||||
def get_tests(e2e_path):
|
||||
tests = []
|
||||
for f in os.listdir(e2e_path):
|
||||
if f.endswith(".sol"):
|
||||
tests.append(f.replace(".sol", ""))
|
||||
return tests
|
||||
|
||||
|
||||
def process_input_file(e2e_path, input_file, interactive):
|
||||
tests = get_tests(e2e_path)
|
||||
with open(input_file, "r", encoding='utf8') as cpp_file:
|
||||
inside_test = False
|
||||
test_name = ""
|
||||
inside_extracted_test = False
|
||||
new_lines = 0
|
||||
count = 0
|
||||
test_content = ""
|
||||
for line in cpp_file.readlines():
|
||||
test = re.search(r'BOOST_AUTO_TEST_CASE\((.*)\)', line, re.M | re.I)
|
||||
if test:
|
||||
test_name = test.group(1)
|
||||
inside_test = True
|
||||
inside_extracted_test = inside_test & (test_name in tests)
|
||||
if inside_extracted_test:
|
||||
count = count + 1
|
||||
|
||||
if interactive and inside_extracted_test:
|
||||
test_content = test_content + line
|
||||
|
||||
if not inside_extracted_test:
|
||||
if line == "\n":
|
||||
new_lines = new_lines + 1
|
||||
else:
|
||||
new_lines = 0
|
||||
if not interactive and new_lines <= 1:
|
||||
sys.stdout.write(line)
|
||||
|
||||
if line == "}\n":
|
||||
if interactive and inside_extracted_test:
|
||||
show_test(test_name, test_content.strip(), e2e_path + "/" + test_name + ".sol", count, len(tests))
|
||||
test_content = ""
|
||||
inside_test = False
|
||||
cpp_file.close()
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def main(argv):
|
||||
interactive = False
|
||||
input_file = None
|
||||
try:
|
||||
opts, _args = getopt.getopt(argv, "if:")
|
||||
except getopt.GetoptError:
|
||||
print("./remove-testcases.py [-i] [-f <full path to SolidityEndToEndTest.cpp>]")
|
||||
sys.exit(1)
|
||||
|
||||
for opt, arg in opts:
|
||||
if opt == '-i':
|
||||
interactive = True
|
||||
elif opt in '-f':
|
||||
input_file = arg
|
||||
|
||||
base_path = os.path.dirname(__file__)
|
||||
|
||||
if not input_file:
|
||||
input_file = base_path + "/../../test/libsolidity/SolidityEndToEndTest.cpp"
|
||||
|
||||
e2e_path = base_path + "/../../test/libsolidity/semanticTests/extracted"
|
||||
|
||||
process_input_file(e2e_path, input_file, interactive)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1:])
|
||||
@ -1,214 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# - SolidityEndToEndTest.trace was created with soltest with the following command on
|
||||
# ./soltest --color_output=false --log_level=test_suite -t SolidityEndToEndTest/ -- --no-smt
|
||||
# --evmonepath /Users/alex/evmone/lib/libevmone.dylib --show-messages > SolidityEndToEndTest.trace
|
||||
# - a trace of the semantic tests can be created by using
|
||||
# ./soltest --color_output=false --log_level=test_suite -t semanticTests/extracted/ -- --no-smt
|
||||
# --evmonepath /Users/alex/evmone/lib/libevmone.dylib --show-messages > semanticTests.trace
|
||||
#
|
||||
# verify-testcases.py will compare both traces. If these traces are identical, the extracted tests were
|
||||
# identical with the tests specified in SolidityEndToEndTest.cpp.
|
||||
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
import getopt
|
||||
import json
|
||||
|
||||
|
||||
class Trace:
|
||||
def __init__(self, kind, parameter):
|
||||
self.kind = kind
|
||||
self.parameter = parameter
|
||||
self._input = ""
|
||||
self._output = ""
|
||||
self.value = ""
|
||||
self.result = ""
|
||||
self.gas = ""
|
||||
|
||||
def get_input(self):
|
||||
return self._input
|
||||
|
||||
def set_input(self, bytecode):
|
||||
if self.kind == "create":
|
||||
# remove cbor encoded metadata from bytecode
|
||||
length = int(bytecode[-4:], 16) * 2
|
||||
self._input = bytecode[:len(bytecode) - length - 4]
|
||||
|
||||
def get_output(self):
|
||||
return self._output
|
||||
|
||||
def set_output(self, output):
|
||||
if self.kind == "create":
|
||||
# remove cbor encoded metadata from bytecode
|
||||
length = int(output[-4:], 16) * 2
|
||||
self._output = output[:len(output) - length - 4]
|
||||
|
||||
def __str__(self):
|
||||
# we ignore the used gas
|
||||
result = str(
|
||||
"kind='" + self.kind + "' parameter='" + self.parameter + "' input='" + self._input +
|
||||
"' output='" + self._output + "' value='" + self.value + "' result='" + self.result + "'"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
class TestCase:
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
self.metadata = None
|
||||
self.traces = []
|
||||
|
||||
def add_trace(self, kind, parameter):
|
||||
trace = Trace(kind, parameter)
|
||||
self.traces.append(trace)
|
||||
return trace
|
||||
|
||||
|
||||
class TraceAnalyser:
|
||||
def __init__(self, file):
|
||||
self.file = file
|
||||
self.tests = {}
|
||||
self.ready = False
|
||||
|
||||
def analyse(self):
|
||||
with open(self.file, "r", encoding='utf8') as trace_file:
|
||||
trace = None
|
||||
test_case = None
|
||||
for line in trace_file.readlines():
|
||||
test = re.search(r'Entering test case "(.*)"', line, re.M | re.I)
|
||||
if test:
|
||||
test_name = test.group(1)
|
||||
test_case = TestCase(test_name)
|
||||
self.tests[test_name] = test_case
|
||||
|
||||
metadata = re.search(r'\s*metadata:\s*(.*)$', line, re.M | re.I)
|
||||
if metadata:
|
||||
test_case.metadata = json.loads(metadata.group(1))
|
||||
del test_case.metadata["sources"]
|
||||
del test_case.metadata["compiler"]["version"]
|
||||
|
||||
create = re.search(r'CREATE\s*([a-fA-F0-9]*):', line, re.M | re.I)
|
||||
if create:
|
||||
trace = test_case.add_trace("create", create.group(1))
|
||||
|
||||
call = re.search(r'CALL\s*([a-fA-F0-9]*)\s*->\s*([a-fA-F0-9]*):', line, re.M | re.I)
|
||||
if call:
|
||||
trace = test_case.add_trace("call", call.group(1)) # + "->" + call.group(2))
|
||||
|
||||
if not create and not call:
|
||||
self.parse_parameters(line, trace)
|
||||
|
||||
trace_file.close()
|
||||
|
||||
print(self.file + ":", len(self.tests), "test-cases.")
|
||||
|
||||
self.ready = True
|
||||
|
||||
@staticmethod
|
||||
def parse_parameters(line, trace):
|
||||
input_match = re.search(r'\s*in:\s*([a-fA-F0-9]*)', line, re.M | re.I)
|
||||
if input_match:
|
||||
trace.input = input_match.group(1)
|
||||
output_match = re.search(r'\s*out:\s*([a-fA-F0-9]*)', line, re.M | re.I)
|
||||
if output_match:
|
||||
trace.output = output_match.group(1)
|
||||
result_match = re.search(r'\s*result:\s*([a-fA-F0-9]*)', line, re.M | re.I)
|
||||
if result_match:
|
||||
trace.result = result_match.group(1)
|
||||
gas_used_match = re.search(r'\s*gas\sused:\s*([a-fA-F0-9]*)', line, re.M | re.I)
|
||||
if gas_used_match:
|
||||
trace.gas = gas_used_match.group(1)
|
||||
value_match = re.search(r'\s*value:\s*([a-fA-F0-9]*)', line, re.M | re.I)
|
||||
if value_match:
|
||||
trace.value = value_match.group(1)
|
||||
|
||||
def diff(self, analyser):
|
||||
if not self.ready:
|
||||
self.analyse()
|
||||
if not analyser.ready:
|
||||
analyser.analyse()
|
||||
|
||||
intersection = set(self.tests.keys()) & set(analyser.tests.keys())
|
||||
mismatches = set()
|
||||
|
||||
for test_name in intersection:
|
||||
left = self.tests[test_name]
|
||||
right = analyser.tests[test_name]
|
||||
if json.dumps(left.metadata) != json.dumps(right.metadata):
|
||||
mismatches.add(
|
||||
(test_name, "metadata where different: " + json.dumps(left.metadata) + " != " + json.dumps(
|
||||
right.metadata)))
|
||||
if len(left.traces) != len(right.traces):
|
||||
mismatches.add((test_name, "trace count are different: " + str(len(left.traces)) +
|
||||
" != " + str(len(right.traces))))
|
||||
else:
|
||||
self.check_traces(test_name, left, right, mismatches)
|
||||
|
||||
for mismatch in mismatches:
|
||||
print(mismatch[0])
|
||||
print(mismatch[1])
|
||||
|
||||
print(len(intersection), "test-cases - ", len(mismatches), " mismatche(s)")
|
||||
|
||||
@classmethod
|
||||
def check_traces(cls, test_name, left, right, mismatches):
|
||||
for trace_id, trace in enumerate(left.traces):
|
||||
left_trace = trace
|
||||
right_trace = right.traces[trace_id]
|
||||
assert left_trace.kind == right_trace.kind
|
||||
if str(left_trace) != str(right_trace):
|
||||
mismatch_info = " " + str(left_trace) + "\n"
|
||||
mismatch_info += " " + str(right_trace) + "\n"
|
||||
mismatch_info += " "
|
||||
for ch in range(0, len(str(left_trace))):
|
||||
if ch < len(str(left_trace)) and ch < len(str(right_trace)):
|
||||
if str(left_trace)[ch] != str(right_trace)[ch]:
|
||||
mismatch_info += "|"
|
||||
else:
|
||||
mismatch_info += " "
|
||||
else:
|
||||
mismatch_info += "|"
|
||||
mismatch_info += "\n"
|
||||
mismatches.add((test_name, mismatch_info))
|
||||
|
||||
|
||||
def main(argv):
|
||||
extracted_tests_trace_file = None
|
||||
end_to_end_trace_file = None
|
||||
try:
|
||||
opts, _args = getopt.getopt(argv, "s:e:")
|
||||
except getopt.GetoptError:
|
||||
print("verify-testcases.py [-s <path to semantic-trace>] [-e <path to endToEndExtraction-trace>]")
|
||||
sys.exit(2)
|
||||
|
||||
for opt, arg in opts:
|
||||
if opt in '-s':
|
||||
extracted_tests_trace_file = arg
|
||||
elif opt in '-e':
|
||||
end_to_end_trace_file = arg
|
||||
|
||||
base_path = os.path.dirname(__file__)
|
||||
if not extracted_tests_trace_file:
|
||||
extracted_tests_trace_file = base_path + "/extracted-tests.trace"
|
||||
if not end_to_end_trace_file:
|
||||
end_to_end_trace_file = base_path + "/endToEndExtraction-tests.trace"
|
||||
|
||||
for f in [extracted_tests_trace_file, end_to_end_trace_file]:
|
||||
if not os.path.isfile(f):
|
||||
print("trace file '" + f + "' not found. aborting.")
|
||||
sys.exit(1)
|
||||
|
||||
if not os.path.isfile(extracted_tests_trace_file):
|
||||
print("semantic trace file '" + extracted_tests_trace_file + "' not found. aborting.")
|
||||
sys.exit(1)
|
||||
|
||||
semantic_trace = TraceAnalyser(extracted_tests_trace_file)
|
||||
end_to_end_trace = TraceAnalyser(end_to_end_trace_file)
|
||||
|
||||
semantic_trace.diff(end_to_end_trace)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1:])
|
||||
Loading…
Reference in New Issue
Block a user