mirror of
https://github.com/ethereum/solidity
synced 2023-10-03 13:03:40 +00:00
Removed folder scripts/endToEndExtraction/
This commit is contained in:
Zachinquarantine
Matheus Aguiar
parent
973eda17b4
commit
45a3d8228d
@ -1,25 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
BASE_PATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 && pwd -P )"
|
|
||||||
|
|
||||||
mkdir -p build
|
|
||||||
cd build
|
|
||||||
cmake ../../../
|
|
||||||
make soltest
|
|
||||||
cd test/
|
|
||||||
echo "running soltest on 'semanticTests/extracted'..."
|
|
||||||
./soltest --color_output=false --log_level=test_suite -t semanticTests/extracted/ -- --testpath "${BASE_PATH}/../../test" --no-smt --evmonepath /Users/alex/evmone/lib/libevmone.dylib --show-messages --show-metadata > "${BASE_PATH}/extracted-tests.trace"
|
|
||||||
echo "running soltest on 'semanticTests/extracted'... done"
|
|
||||||
|
|
||||||
cd "$BASE_PATH"
|
|
||||||
git clone git@github.com:ethereum/solidity.git solidity-develop
|
|
||||||
cd solidity-develop
|
|
||||||
mkdir -p build
|
|
||||||
cd build
|
|
||||||
cmake ..
|
|
||||||
make soltest
|
|
||||||
cd test/
|
|
||||||
echo "running soltest on 'SolidityEndToEndTest'..."
|
|
||||||
./soltest --color_output=false --log_level=test_suite -t SolidityEndToEndTest/ -- --testpath "${BASE_PATH}/solidity-develop/test" --no-smt --evmonepath /Users/alex/evmone/lib/libevmone.dylib --show-messages --show-metadata > "${BASE_PATH}/endToEndExtraction-tests.trace"
|
|
||||||
echo "running soltest on 'SolidityEndToEndTest'... done"
|
|
||||||
@ -1,183 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
# pylint: disable=consider-using-enumerate, import-error
|
|
||||||
|
|
||||||
import re
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import getopt
|
|
||||||
import tempfile
|
|
||||||
from getkey import getkey
|
|
||||||
|
|
||||||
|
|
||||||
def parse_call(call):
|
|
||||||
function = ''
|
|
||||||
arguments = ""
|
|
||||||
results = ""
|
|
||||||
search = re.search(r'// (.*):(.*)\s->\s(.*)', call, re.MULTILINE | re.DOTALL)
|
|
||||||
if search:
|
|
||||||
function = search.group(1)
|
|
||||||
arguments = search.group(2)
|
|
||||||
results = search.group(3)
|
|
||||||
if results.find("#") != -1:
|
|
||||||
results = results[:results.find("#")]
|
|
||||||
else:
|
|
||||||
search = re.search(r'// (.*)(.*)\s->\s(.*)', call, re.MULTILINE | re.DOTALL)
|
|
||||||
if search:
|
|
||||||
function = search.group(1)
|
|
||||||
arguments = search.group(2)
|
|
||||||
results = search.group(3)
|
|
||||||
if results.find("#") != -1:
|
|
||||||
results = results[:results.find("#")]
|
|
||||||
if function.find("wei") >= 0:
|
|
||||||
function = function[:function.find(",")]
|
|
||||||
return function.strip(), arguments.strip(), results.strip()
|
|
||||||
|
|
||||||
|
|
||||||
def colorize(left, right, index):
|
|
||||||
red = "\x1b[31m"
|
|
||||||
yellow = "\x1b[33m"
|
|
||||||
reset = "\x1b[0m"
|
|
||||||
colors = [red, yellow]
|
|
||||||
color = colors[index % len(colors)]
|
|
||||||
function, _arguments, _results = parse_call(right)
|
|
||||||
left = left.replace("compileAndRun", color + "compileAndRun" + reset)
|
|
||||||
right = right.replace("constructor", color + "constructor" + reset)
|
|
||||||
if function:
|
|
||||||
left = left.replace(function, color + function + reset)
|
|
||||||
right = right.replace(function, color + function + reset)
|
|
||||||
if left.find(function):
|
|
||||||
bottom = " " * (left.find(function) - 4) + right
|
|
||||||
else:
|
|
||||||
bottom = " " + right
|
|
||||||
return " " + left + "\n" + bottom # " {:<90} {:<90}\n{}".format(left, right, bottom)
|
|
||||||
|
|
||||||
|
|
||||||
def get_checks(content, sol_file_path):
|
|
||||||
constructors = []
|
|
||||||
checks = []
|
|
||||||
for line in content.split("\n"):
|
|
||||||
line = line.strip()
|
|
||||||
if line.startswith("compileAndRun"):
|
|
||||||
constructors.append(line)
|
|
||||||
if line.startswith("ABI_CHECK") or line.startswith("BOOST_REQUIRE"):
|
|
||||||
checks.append(line)
|
|
||||||
with open(sol_file_path, "r", encoding='utf8') as sol_file:
|
|
||||||
sol_constructors = []
|
|
||||||
sol_checks = []
|
|
||||||
inside_expectations = False
|
|
||||||
for line in sol_file.readlines():
|
|
||||||
if line.startswith("// constructor()"):
|
|
||||||
sol_constructors.append(line)
|
|
||||||
elif inside_expectations and line.startswith("// "):
|
|
||||||
sol_checks.append(line)
|
|
||||||
if line.startswith("// ----"):
|
|
||||||
inside_expectations = True
|
|
||||||
sol_file.close()
|
|
||||||
if len(constructors) == len(sol_constructors) == 1:
|
|
||||||
checks.insert(0, constructors[0])
|
|
||||||
sol_checks.insert(0, sol_constructors[0])
|
|
||||||
return checks, sol_checks
|
|
||||||
|
|
||||||
|
|
||||||
def show_test(name, content, sol_file_path, current_test, test_count):
|
|
||||||
with tempfile.NamedTemporaryFile(delete=False) as cpp_file:
|
|
||||||
cpp_file.write(content.encode())
|
|
||||||
cpp_file.close()
|
|
||||||
|
|
||||||
os.system("clear")
|
|
||||||
print(str(current_test) + " / " + str(test_count) + " - " + name + "\n")
|
|
||||||
diff_env = os.getenv('DIFF', "/usr/local/bin/colordiff -a -d -w -y -W 200 ")
|
|
||||||
os.system(diff_env + " " + cpp_file.name + " " + sol_file_path)
|
|
||||||
os.unlink(cpp_file.name)
|
|
||||||
print("\n")
|
|
||||||
|
|
||||||
checks, sol_checks = get_checks(content, sol_file_path)
|
|
||||||
|
|
||||||
if len(checks) == len(sol_checks):
|
|
||||||
for i in range(0, len(checks)):
|
|
||||||
print(colorize(checks[i].strip(), sol_checks[i].strip(), i))
|
|
||||||
else:
|
|
||||||
print("warning: check count not matching. this should not happen!")
|
|
||||||
|
|
||||||
what = ""
|
|
||||||
print("\nContinue? (ENTER) Abort? (ANY OTHER KEY)")
|
|
||||||
while what != '\n':
|
|
||||||
what = getkey()
|
|
||||||
if what != '\n':
|
|
||||||
sys.exit(0)
|
|
||||||
print()
|
|
||||||
|
|
||||||
|
|
||||||
def get_tests(e2e_path):
|
|
||||||
tests = []
|
|
||||||
for f in os.listdir(e2e_path):
|
|
||||||
if f.endswith(".sol"):
|
|
||||||
tests.append(f.replace(".sol", ""))
|
|
||||||
return tests
|
|
||||||
|
|
||||||
|
|
||||||
def process_input_file(e2e_path, input_file, interactive):
|
|
||||||
tests = get_tests(e2e_path)
|
|
||||||
with open(input_file, "r", encoding='utf8') as cpp_file:
|
|
||||||
inside_test = False
|
|
||||||
test_name = ""
|
|
||||||
inside_extracted_test = False
|
|
||||||
new_lines = 0
|
|
||||||
count = 0
|
|
||||||
test_content = ""
|
|
||||||
for line in cpp_file.readlines():
|
|
||||||
test = re.search(r'BOOST_AUTO_TEST_CASE\((.*)\)', line, re.M | re.I)
|
|
||||||
if test:
|
|
||||||
test_name = test.group(1)
|
|
||||||
inside_test = True
|
|
||||||
inside_extracted_test = inside_test & (test_name in tests)
|
|
||||||
if inside_extracted_test:
|
|
||||||
count = count + 1
|
|
||||||
|
|
||||||
if interactive and inside_extracted_test:
|
|
||||||
test_content = test_content + line
|
|
||||||
|
|
||||||
if not inside_extracted_test:
|
|
||||||
if line == "\n":
|
|
||||||
new_lines = new_lines + 1
|
|
||||||
else:
|
|
||||||
new_lines = 0
|
|
||||||
if not interactive and new_lines <= 1:
|
|
||||||
sys.stdout.write(line)
|
|
||||||
|
|
||||||
if line == "}\n":
|
|
||||||
if interactive and inside_extracted_test:
|
|
||||||
show_test(test_name, test_content.strip(), e2e_path + "/" + test_name + ".sol", count, len(tests))
|
|
||||||
test_content = ""
|
|
||||||
inside_test = False
|
|
||||||
cpp_file.close()
|
|
||||||
sys.stdout.flush()
|
|
||||||
|
|
||||||
|
|
||||||
def main(argv):
|
|
||||||
interactive = False
|
|
||||||
input_file = None
|
|
||||||
try:
|
|
||||||
opts, _args = getopt.getopt(argv, "if:")
|
|
||||||
except getopt.GetoptError:
|
|
||||||
print("./remove-testcases.py [-i] [-f <full path to SolidityEndToEndTest.cpp>]")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
for opt, arg in opts:
|
|
||||||
if opt == '-i':
|
|
||||||
interactive = True
|
|
||||||
elif opt in '-f':
|
|
||||||
input_file = arg
|
|
||||||
|
|
||||||
base_path = os.path.dirname(__file__)
|
|
||||||
|
|
||||||
if not input_file:
|
|
||||||
input_file = base_path + "/../../test/libsolidity/SolidityEndToEndTest.cpp"
|
|
||||||
|
|
||||||
e2e_path = base_path + "/../../test/libsolidity/semanticTests/extracted"
|
|
||||||
|
|
||||||
process_input_file(e2e_path, input_file, interactive)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main(sys.argv[1:])
|
|
||||||
@ -1,214 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
#
|
|
||||||
# - SolidityEndToEndTest.trace was created with soltest with the following command on
|
|
||||||
# ./soltest --color_output=false --log_level=test_suite -t SolidityEndToEndTest/ -- --no-smt
|
|
||||||
# --evmonepath /Users/alex/evmone/lib/libevmone.dylib --show-messages > SolidityEndToEndTest.trace
|
|
||||||
# - a trace of the semantic tests can be created by using
|
|
||||||
# ./soltest --color_output=false --log_level=test_suite -t semanticTests/extracted/ -- --no-smt
|
|
||||||
# --evmonepath /Users/alex/evmone/lib/libevmone.dylib --show-messages > semanticTests.trace
|
|
||||||
#
|
|
||||||
# verify-testcases.py will compare both traces. If these traces are identical, the extracted tests were
|
|
||||||
# identical with the tests specified in SolidityEndToEndTest.cpp.
|
|
||||||
|
|
||||||
import re
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import getopt
|
|
||||||
import json
|
|
||||||
|
|
||||||
|
|
||||||
class Trace:
|
|
||||||
def __init__(self, kind, parameter):
|
|
||||||
self.kind = kind
|
|
||||||
self.parameter = parameter
|
|
||||||
self._input = ""
|
|
||||||
self._output = ""
|
|
||||||
self.value = ""
|
|
||||||
self.result = ""
|
|
||||||
self.gas = ""
|
|
||||||
|
|
||||||
def get_input(self):
|
|
||||||
return self._input
|
|
||||||
|
|
||||||
def set_input(self, bytecode):
|
|
||||||
if self.kind == "create":
|
|
||||||
# remove cbor encoded metadata from bytecode
|
|
||||||
length = int(bytecode[-4:], 16) * 2
|
|
||||||
self._input = bytecode[:len(bytecode) - length - 4]
|
|
||||||
|
|
||||||
def get_output(self):
|
|
||||||
return self._output
|
|
||||||
|
|
||||||
def set_output(self, output):
|
|
||||||
if self.kind == "create":
|
|
||||||
# remove cbor encoded metadata from bytecode
|
|
||||||
length = int(output[-4:], 16) * 2
|
|
||||||
self._output = output[:len(output) - length - 4]
|
|
||||||
|
|
||||||
def __str__(self):
|
|
||||||
# we ignore the used gas
|
|
||||||
result = str(
|
|
||||||
"kind='" + self.kind + "' parameter='" + self.parameter + "' input='" + self._input +
|
|
||||||
"' output='" + self._output + "' value='" + self.value + "' result='" + self.result + "'"
|
|
||||||
)
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
class TestCase:
|
|
||||||
def __init__(self, name):
|
|
||||||
self.name = name
|
|
||||||
self.metadata = None
|
|
||||||
self.traces = []
|
|
||||||
|
|
||||||
def add_trace(self, kind, parameter):
|
|
||||||
trace = Trace(kind, parameter)
|
|
||||||
self.traces.append(trace)
|
|
||||||
return trace
|
|
||||||
|
|
||||||
|
|
||||||
class TraceAnalyser:
|
|
||||||
def __init__(self, file):
|
|
||||||
self.file = file
|
|
||||||
self.tests = {}
|
|
||||||
self.ready = False
|
|
||||||
|
|
||||||
def analyse(self):
|
|
||||||
with open(self.file, "r", encoding='utf8') as trace_file:
|
|
||||||
trace = None
|
|
||||||
test_case = None
|
|
||||||
for line in trace_file.readlines():
|
|
||||||
test = re.search(r'Entering test case "(.*)"', line, re.M | re.I)
|
|
||||||
if test:
|
|
||||||
test_name = test.group(1)
|
|
||||||
test_case = TestCase(test_name)
|
|
||||||
self.tests[test_name] = test_case
|
|
||||||
|
|
||||||
metadata = re.search(r'\s*metadata:\s*(.*)$', line, re.M | re.I)
|
|
||||||
if metadata:
|
|
||||||
test_case.metadata = json.loads(metadata.group(1))
|
|
||||||
del test_case.metadata["sources"]
|
|
||||||
del test_case.metadata["compiler"]["version"]
|
|
||||||
|
|
||||||
create = re.search(r'CREATE\s*([a-fA-F0-9]*):', line, re.M | re.I)
|
|
||||||
if create:
|
|
||||||
trace = test_case.add_trace("create", create.group(1))
|
|
||||||
|
|
||||||
call = re.search(r'CALL\s*([a-fA-F0-9]*)\s*->\s*([a-fA-F0-9]*):', line, re.M | re.I)
|
|
||||||
if call:
|
|
||||||
trace = test_case.add_trace("call", call.group(1)) # + "->" + call.group(2))
|
|
||||||
|
|
||||||
if not create and not call:
|
|
||||||
self.parse_parameters(line, trace)
|
|
||||||
|
|
||||||
trace_file.close()
|
|
||||||
|
|
||||||
print(self.file + ":", len(self.tests), "test-cases.")
|
|
||||||
|
|
||||||
self.ready = True
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def parse_parameters(line, trace):
|
|
||||||
input_match = re.search(r'\s*in:\s*([a-fA-F0-9]*)', line, re.M | re.I)
|
|
||||||
if input_match:
|
|
||||||
trace.input = input_match.group(1)
|
|
||||||
output_match = re.search(r'\s*out:\s*([a-fA-F0-9]*)', line, re.M | re.I)
|
|
||||||
if output_match:
|
|
||||||
trace.output = output_match.group(1)
|
|
||||||
result_match = re.search(r'\s*result:\s*([a-fA-F0-9]*)', line, re.M | re.I)
|
|
||||||
if result_match:
|
|
||||||
trace.result = result_match.group(1)
|
|
||||||
gas_used_match = re.search(r'\s*gas\sused:\s*([a-fA-F0-9]*)', line, re.M | re.I)
|
|
||||||
if gas_used_match:
|
|
||||||
trace.gas = gas_used_match.group(1)
|
|
||||||
value_match = re.search(r'\s*value:\s*([a-fA-F0-9]*)', line, re.M | re.I)
|
|
||||||
if value_match:
|
|
||||||
trace.value = value_match.group(1)
|
|
||||||
|
|
||||||
def diff(self, analyser):
|
|
||||||
if not self.ready:
|
|
||||||
self.analyse()
|
|
||||||
if not analyser.ready:
|
|
||||||
analyser.analyse()
|
|
||||||
|
|
||||||
intersection = set(self.tests.keys()) & set(analyser.tests.keys())
|
|
||||||
mismatches = set()
|
|
||||||
|
|
||||||
for test_name in intersection:
|
|
||||||
left = self.tests[test_name]
|
|
||||||
right = analyser.tests[test_name]
|
|
||||||
if json.dumps(left.metadata) != json.dumps(right.metadata):
|
|
||||||
mismatches.add(
|
|
||||||
(test_name, "metadata where different: " + json.dumps(left.metadata) + " != " + json.dumps(
|
|
||||||
right.metadata)))
|
|
||||||
if len(left.traces) != len(right.traces):
|
|
||||||
mismatches.add((test_name, "trace count are different: " + str(len(left.traces)) +
|
|
||||||
" != " + str(len(right.traces))))
|
|
||||||
else:
|
|
||||||
self.check_traces(test_name, left, right, mismatches)
|
|
||||||
|
|
||||||
for mismatch in mismatches:
|
|
||||||
print(mismatch[0])
|
|
||||||
print(mismatch[1])
|
|
||||||
|
|
||||||
print(len(intersection), "test-cases - ", len(mismatches), " mismatche(s)")
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def check_traces(cls, test_name, left, right, mismatches):
|
|
||||||
for trace_id, trace in enumerate(left.traces):
|
|
||||||
left_trace = trace
|
|
||||||
right_trace = right.traces[trace_id]
|
|
||||||
assert left_trace.kind == right_trace.kind
|
|
||||||
if str(left_trace) != str(right_trace):
|
|
||||||
mismatch_info = " " + str(left_trace) + "\n"
|
|
||||||
mismatch_info += " " + str(right_trace) + "\n"
|
|
||||||
mismatch_info += " "
|
|
||||||
for ch in range(0, len(str(left_trace))):
|
|
||||||
if ch < len(str(left_trace)) and ch < len(str(right_trace)):
|
|
||||||
if str(left_trace)[ch] != str(right_trace)[ch]:
|
|
||||||
mismatch_info += "|"
|
|
||||||
else:
|
|
||||||
mismatch_info += " "
|
|
||||||
else:
|
|
||||||
mismatch_info += "|"
|
|
||||||
mismatch_info += "\n"
|
|
||||||
mismatches.add((test_name, mismatch_info))
|
|
||||||
|
|
||||||
|
|
||||||
def main(argv):
|
|
||||||
extracted_tests_trace_file = None
|
|
||||||
end_to_end_trace_file = None
|
|
||||||
try:
|
|
||||||
opts, _args = getopt.getopt(argv, "s:e:")
|
|
||||||
except getopt.GetoptError:
|
|
||||||
print("verify-testcases.py [-s <path to semantic-trace>] [-e <path to endToEndExtraction-trace>]")
|
|
||||||
sys.exit(2)
|
|
||||||
|
|
||||||
for opt, arg in opts:
|
|
||||||
if opt in '-s':
|
|
||||||
extracted_tests_trace_file = arg
|
|
||||||
elif opt in '-e':
|
|
||||||
end_to_end_trace_file = arg
|
|
||||||
|
|
||||||
base_path = os.path.dirname(__file__)
|
|
||||||
if not extracted_tests_trace_file:
|
|
||||||
extracted_tests_trace_file = base_path + "/extracted-tests.trace"
|
|
||||||
if not end_to_end_trace_file:
|
|
||||||
end_to_end_trace_file = base_path + "/endToEndExtraction-tests.trace"
|
|
||||||
|
|
||||||
for f in [extracted_tests_trace_file, end_to_end_trace_file]:
|
|
||||||
if not os.path.isfile(f):
|
|
||||||
print("trace file '" + f + "' not found. aborting.")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
if not os.path.isfile(extracted_tests_trace_file):
|
|
||||||
print("semantic trace file '" + extracted_tests_trace_file + "' not found. aborting.")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
semantic_trace = TraceAnalyser(extracted_tests_trace_file)
|
|
||||||
end_to_end_trace = TraceAnalyser(end_to_end_trace_file)
|
|
||||||
|
|
||||||
semantic_trace.diff(end_to_end_trace)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main(sys.argv[1:])
|
|
||||||
Loading…
Reference in New Issue
Block a user