diff --git a/scripts/bytecodecompare/prepare_report.py b/scripts/bytecodecompare/prepare_report.py index 744c6deb5..178ac5699 100755 --- a/scripts/bytecodecompare/prepare_report.py +++ b/scripts/bytecodecompare/prepare_report.py @@ -42,7 +42,9 @@ class FileReport: def load_source(path: Union[Path, str]) -> str: - with open(path, mode='r', encoding='utf8') as source_file: + # NOTE: newline='' disables newline conversion. + # We want the file exactly as is because changing even a single byte in the source affects metadata. + with open(path, mode='r', encoding='utf8', newline='') as source_file: file_content = source_file.read() return file_content diff --git a/scripts/check_style.sh b/scripts/check_style.sh index 610edc3ff..c4df6eb98 100755 --- a/scripts/check_style.sh +++ b/scripts/check_style.sh @@ -8,6 +8,9 @@ EXCLUDE_FILES=( "test/libsolidity/syntaxTests/license/license_cr_endings.sol" "test/libsolidity/syntaxTests/license/license_crlf_endings.sol" "test/libsolidity/syntaxTests/license/license_whitespace_trailing.sol" + "test/scripts/fixtures/smt_contract_with_crlf_newlines.sol" + "test/scripts/fixtures/smt_contract_with_cr_newlines.sol" + "test/scripts/fixtures/smt_contract_with_mixed_newlines.sol" ) EXCLUDE_FILES_JOINED=$(printf "%s\|" "${EXCLUDE_FILES[@]}") EXCLUDE_FILES_JOINED=${EXCLUDE_FILES_JOINED%??} diff --git a/scripts/isolate_tests.py b/scripts/isolate_tests.py index 33fdc0165..393d6ab8b 100755 --- a/scripts/isolate_tests.py +++ b/scripts/isolate_tests.py @@ -13,7 +13,7 @@ import hashlib from os.path import join, isfile, split def extract_test_cases(path): - lines = open(path, encoding="utf8", errors='ignore', mode='r').read().splitlines() + lines = open(path, encoding="utf8", errors='ignore', mode='r', newline='').read().splitlines() inside = False delimiter = '' @@ -43,7 +43,7 @@ def extract_docs_cases(path): tests = [] # Collect all snippets of indented blocks - for l in open(path, mode='r', errors='ignore', encoding='utf8').read().splitlines(): + for l in open(path, mode='r', errors='ignore', encoding='utf8', newline='').read().splitlines(): if l != '': if not inside and l.startswith(' '): # start new test @@ -72,14 +72,14 @@ def write_cases(f, tests): # so before checking remove 4 spaces from each line. remainder = re.sub(r'^ {4}', '', test, 0, re.MULTILINE) sol_filename = 'test_%s_%s.sol' % (hashlib.sha256(test.encode("utf-8")).hexdigest(), cleaned_filename) - open(sol_filename, mode='w', encoding='utf8').write(remainder) + open(sol_filename, mode='w', encoding='utf8', newline='').write(remainder) def extract_and_write(f, path): if docs: cases = extract_docs_cases(path) else: if f.endswith('.sol'): - cases = [open(path, mode='r', encoding='utf8').read()] + cases = [open(path, mode='r', encoding='utf8', newline='').read()] else: cases = extract_test_cases(path) write_cases(f, cases) diff --git a/scripts/splitSources.py b/scripts/splitSources.py index 54ddb2f38..5b78ccb8f 100755 --- a/scripts/splitSources.py +++ b/scripts/splitSources.py @@ -42,7 +42,7 @@ def writeSourceToFile(lines): # print("filePath is", filePath) if filePath != False: os.system("mkdir -p " + filePath) - f = open(srcName, mode='a+', encoding='utf8') + f = open(srcName, mode='a+', encoding='utf8', newline='') createdSources.append(srcName) i = 0 for idx, line in enumerate(lines[1:]): @@ -63,7 +63,7 @@ if __name__ == '__main__': try: # decide if file has multiple sources - lines = open(filePath, mode='r', encoding='utf8').read().splitlines() + lines = open(filePath, mode='r', encoding='utf8', newline='').read().splitlines() if lines[0][:12] == "==== Source:": hasMultipleSources = True writeSourceToFile(lines) diff --git a/test/scripts/fixtures/smt_contract_with_cr_newlines.sol b/test/scripts/fixtures/smt_contract_with_cr_newlines.sol new file mode 100644 index 000000000..1f9994caa --- /dev/null +++ b/test/scripts/fixtures/smt_contract_with_cr_newlines.sol @@ -0,0 +1 @@ +pragma experimental SMTChecker; contract C { } \ No newline at end of file diff --git a/test/scripts/fixtures/smt_contract_with_crlf_newlines.sol b/test/scripts/fixtures/smt_contract_with_crlf_newlines.sol new file mode 100644 index 000000000..f1ea8f40b --- /dev/null +++ b/test/scripts/fixtures/smt_contract_with_crlf_newlines.sol @@ -0,0 +1,4 @@ +pragma experimental SMTChecker; + +contract C { +} diff --git a/test/scripts/fixtures/smt_contract_with_lf_newlines.sol b/test/scripts/fixtures/smt_contract_with_lf_newlines.sol new file mode 100644 index 000000000..80aa57ff5 --- /dev/null +++ b/test/scripts/fixtures/smt_contract_with_lf_newlines.sol @@ -0,0 +1,4 @@ +pragma experimental SMTChecker; + +contract C { +} diff --git a/test/scripts/fixtures/smt_contract_with_mixed_newlines.sol b/test/scripts/fixtures/smt_contract_with_mixed_newlines.sol new file mode 100644 index 000000000..50bb65928 --- /dev/null +++ b/test/scripts/fixtures/smt_contract_with_mixed_newlines.sol @@ -0,0 +1,3 @@ +pragma experimental SMTChecker; + +contract C { } diff --git a/test/scripts/test_bytecodecompare_prepare_report.py b/test/scripts/test_bytecodecompare_prepare_report.py index 514786272..7bf63f73c 100644 --- a/test/scripts/test_bytecodecompare_prepare_report.py +++ b/test/scripts/test_bytecodecompare_prepare_report.py @@ -4,7 +4,7 @@ import json import unittest from pathlib import Path -from unittest_helpers import LIBSOLIDITY_TEST_DIR, load_fixture, load_libsolidity_test_case +from unittest_helpers import FIXTURE_DIR, LIBSOLIDITY_TEST_DIR, load_fixture, load_libsolidity_test_case # NOTE: This test file file only works with scripts/ added to PYTHONPATH so pylint can't find the imports # pragma pylint: disable=import-error @@ -16,6 +16,11 @@ from bytecodecompare.prepare_report import load_source, parse_standard_json_outp SMT_SMOKE_TEST_SOL_PATH = LIBSOLIDITY_TEST_DIR / 'smtCheckerTests/simple/smoke_test.sol' SMT_SMOKE_TEST_SOL_CODE = load_libsolidity_test_case(SMT_SMOKE_TEST_SOL_PATH) +SMT_CONTRACT_WITH_LF_NEWLINES_SOL_PATH = FIXTURE_DIR / 'smt_contract_with_lf_newlines.sol' +SMT_CONTRACT_WITH_CRLF_NEWLINES_SOL_PATH = FIXTURE_DIR / 'smt_contract_with_crlf_newlines.sol' +SMT_CONTRACT_WITH_CR_NEWLINES_SOL_PATH = FIXTURE_DIR / 'smt_contract_with_cr_newlines.sol' +SMT_CONTRACT_WITH_MIXED_NEWLINES_SOL_PATH = FIXTURE_DIR / 'smt_contract_with_mixed_newlines.sol' + SYNTAX_SMOKE_TEST_SOL_PATH = LIBSOLIDITY_TEST_DIR / 'syntaxTests/smoke_test.sol' SYNTAX_SMOKE_TEST_SOL_CODE = load_libsolidity_test_case(SYNTAX_SMOKE_TEST_SOL_PATH) @@ -81,6 +86,46 @@ class TestPrepareReport(unittest.TestCase): def test_load_source(self): self.assertEqual(load_source(SMT_SMOKE_TEST_SOL_PATH), SMT_SMOKE_TEST_SOL_CODE) + def test_load_source_preserves_lf_newlines(self): + expected_output = ( + "pragma experimental SMTChecker;\n" + "\n" + "contract C {\n" + "}\n" + ) + + self.assertEqual(load_source(SMT_CONTRACT_WITH_LF_NEWLINES_SOL_PATH), expected_output) + + def test_load_source_preserves_crlf_newlines(self): + expected_output = ( + "pragma experimental SMTChecker;\r\n" + "\r\n" + "contract C {\r\n" + "}\r\n" + ) + + self.assertEqual(load_source(SMT_CONTRACT_WITH_CRLF_NEWLINES_SOL_PATH), expected_output) + + def test_load_source_preserves_cr_newlines(self): + expected_output = ( + "pragma experimental SMTChecker;\r" + "\r" + "contract C {\r" + "}\r" + ) + + self.assertEqual(load_source(SMT_CONTRACT_WITH_CR_NEWLINES_SOL_PATH), expected_output) + + def test_load_source_preserves_mixed_newlines(self): + expected_output = ( + "pragma experimental SMTChecker;\n" + "\n" + "contract C {\r" + "}\r\n" + ) + + self.assertEqual(load_source(SMT_CONTRACT_WITH_MIXED_NEWLINES_SOL_PATH), expected_output) + def test_prepare_compiler_input(self): expected_compiler_input = { 'language': 'Solidity', @@ -103,6 +148,34 @@ class TestPrepareReport(unittest.TestCase): self.assertEqual(command_line, ['solc', '--standard-json']) self.assertEqual(json.loads(compiler_input), expected_compiler_input) + def test_prepare_compiler_input_preserves_newlines(self): + expected_compiler_input = { + 'language': 'Solidity', + 'sources': { + str(SMT_CONTRACT_WITH_MIXED_NEWLINES_SOL_PATH): { + 'content': + "pragma experimental SMTChecker;\n" + "\n" + "contract C {\r" + "}\r\n" + }, + }, + 'settings': { + 'optimizer': {'enabled': True}, + 'outputSelection': {'*': {'*': ['evm.bytecode.object', 'metadata']}}, + 'modelChecker': {'engine': 'none'}, + } + } + + (command_line, compiler_input) = prepare_compiler_input( + Path('solc'), + SMT_CONTRACT_WITH_MIXED_NEWLINES_SOL_PATH, + optimize=True, + ) + + self.assertEqual(command_line, ['solc', '--standard-json']) + self.assertEqual(json.loads(compiler_input), expected_compiler_input) + def test_parse_standard_json_output(self): expected_report = FileReport( file_name=Path('syntaxTests/scoping/library_inherited2.sol'), diff --git a/test/scripts/unittest_helpers.py b/test/scripts/unittest_helpers.py index 957e9007a..e666485eb 100644 --- a/test/scripts/unittest_helpers.py +++ b/test/scripts/unittest_helpers.py @@ -5,7 +5,9 @@ LIBSOLIDITY_TEST_DIR = Path(__file__).parent.parent / 'libsolidity' FIXTURE_DIR = Path(__file__).parent / 'fixtures' def load_file(path: Union[Path, str]) -> str: - with open(path, 'r', encoding='utf-8') as f: + # NOTE: newline='' disables newline conversion. + # We want the file exactly as is because changing even a single byte in the source affects metadata. + with open(path, 'r', encoding='utf-8', newline='') as f: return f.read() def load_fixture(relative_path: Union[Path, str]) -> str: