Read/write files in python with newline='' option to preserve newlines as \n on Windows

scripts/bytecodecompare/prepare_report.py

@@ -42,7 +42,9 @@ class FileReport:
 
 
 def load_source(path: Union[Path, str]) -> str:
-    with open(path, mode='r', encoding='utf8') as source_file:
+    # NOTE: newline='' disables newline conversion.
+    # We want the file exactly as is because changing even a single byte in the source affects metadata.
+    with open(path, mode='r', encoding='utf8', newline='') as source_file:
         file_content = source_file.read()
 
     return file_content

scripts/check_style.sh

@@ -8,6 +8,9 @@ EXCLUDE_FILES=(
     "test/libsolidity/syntaxTests/license/license_cr_endings.sol"
     "test/libsolidity/syntaxTests/license/license_crlf_endings.sol"
     "test/libsolidity/syntaxTests/license/license_whitespace_trailing.sol"
+    "test/scripts/fixtures/smt_contract_with_crlf_newlines.sol"
+    "test/scripts/fixtures/smt_contract_with_cr_newlines.sol"
+    "test/scripts/fixtures/smt_contract_with_mixed_newlines.sol"
 )
 EXCLUDE_FILES_JOINED=$(printf "%s\|" "${EXCLUDE_FILES[@]}")
 EXCLUDE_FILES_JOINED=${EXCLUDE_FILES_JOINED%??}

scripts/isolate_tests.py

@@ -13,7 +13,7 @@ import hashlib
 from os.path import join, isfile, split
 
 def extract_test_cases(path):
-    lines = open(path, encoding="utf8", errors='ignore', mode='r').read().splitlines()
+    lines = open(path, encoding="utf8", errors='ignore', mode='r', newline='').read().splitlines()
 
     inside = False
     delimiter = ''
@@ -43,7 +43,7 @@ def extract_docs_cases(path):
     tests = []
 
     # Collect all snippets of indented blocks
-    for l in open(path, mode='r', errors='ignore', encoding='utf8').read().splitlines():
+    for l in open(path, mode='r', errors='ignore', encoding='utf8', newline='').read().splitlines():
         if l != '':
             if not inside and l.startswith(' '):
                 # start new test
@@ -72,14 +72,14 @@ def write_cases(f, tests):
         # so before checking remove 4 spaces from each line.
         remainder = re.sub(r'^ {4}', '', test, 0, re.MULTILINE)
         sol_filename = 'test_%s_%s.sol' % (hashlib.sha256(test.encode("utf-8")).hexdigest(), cleaned_filename)
-        open(sol_filename, mode='w', encoding='utf8').write(remainder)
+        open(sol_filename, mode='w', encoding='utf8', newline='').write(remainder)
 
 def extract_and_write(f, path):
     if docs:
         cases = extract_docs_cases(path)
     else:
         if f.endswith('.sol'):
-            cases = [open(path, mode='r', encoding='utf8').read()]
+            cases = [open(path, mode='r', encoding='utf8', newline='').read()]
         else:
             cases = extract_test_cases(path)
     write_cases(f, cases)

scripts/splitSources.py

@@ -42,7 +42,7 @@ def writeSourceToFile(lines):
     # print("filePath is", filePath)
     if filePath != False:
         os.system("mkdir -p " + filePath)
-    f = open(srcName, mode='a+', encoding='utf8')
+    f = open(srcName, mode='a+', encoding='utf8', newline='')
     createdSources.append(srcName)
     i = 0
     for idx, line in enumerate(lines[1:]):
@@ -63,7 +63,7 @@ if __name__ == '__main__':
 
     try:
         # decide if file has multiple sources
-        lines = open(filePath, mode='r', encoding='utf8').read().splitlines()
+        lines = open(filePath, mode='r', encoding='utf8', newline='').read().splitlines()
         if lines[0][:12] == "==== Source:":
             hasMultipleSources = True
             writeSourceToFile(lines)

test/scripts/fixtures/smt_contract_with_cr_newlines.sol

@@ -0,0 +1 @@
+pragma experimental SMTChecker;

contract C {
}

test/scripts/fixtures/smt_contract_with_crlf_newlines.sol

@@ -0,0 +1,4 @@
+pragma experimental SMTChecker;
+
+contract C {
+}

test/scripts/fixtures/smt_contract_with_lf_newlines.sol

@@ -0,0 +1,4 @@
+pragma experimental SMTChecker;
+
+contract C {
+}

test/scripts/fixtures/smt_contract_with_mixed_newlines.sol

@@ -0,0 +1,3 @@
+pragma experimental SMTChecker;
+
+contract C {
}

test/scripts/test_bytecodecompare_prepare_report.py

@@ -4,7 +4,7 @@ import json
 import unittest
 from pathlib import Path
 
-from unittest_helpers import LIBSOLIDITY_TEST_DIR, load_fixture, load_libsolidity_test_case
+from unittest_helpers import FIXTURE_DIR, LIBSOLIDITY_TEST_DIR, load_fixture, load_libsolidity_test_case
 
 # NOTE: This test file file only works with scripts/ added to PYTHONPATH so pylint can't find the imports
 # pragma pylint: disable=import-error
@@ -16,6 +16,11 @@ from bytecodecompare.prepare_report import load_source, parse_standard_json_outp
 SMT_SMOKE_TEST_SOL_PATH = LIBSOLIDITY_TEST_DIR / 'smtCheckerTests/simple/smoke_test.sol'
 SMT_SMOKE_TEST_SOL_CODE = load_libsolidity_test_case(SMT_SMOKE_TEST_SOL_PATH)
 
+SMT_CONTRACT_WITH_LF_NEWLINES_SOL_PATH = FIXTURE_DIR / 'smt_contract_with_lf_newlines.sol'
+SMT_CONTRACT_WITH_CRLF_NEWLINES_SOL_PATH = FIXTURE_DIR / 'smt_contract_with_crlf_newlines.sol'
+SMT_CONTRACT_WITH_CR_NEWLINES_SOL_PATH = FIXTURE_DIR / 'smt_contract_with_cr_newlines.sol'
+SMT_CONTRACT_WITH_MIXED_NEWLINES_SOL_PATH = FIXTURE_DIR / 'smt_contract_with_mixed_newlines.sol'
+
 SYNTAX_SMOKE_TEST_SOL_PATH = LIBSOLIDITY_TEST_DIR / 'syntaxTests/smoke_test.sol'
 SYNTAX_SMOKE_TEST_SOL_CODE = load_libsolidity_test_case(SYNTAX_SMOKE_TEST_SOL_PATH)
 
@@ -81,6 +86,46 @@ class TestPrepareReport(unittest.TestCase):
     def test_load_source(self):
         self.assertEqual(load_source(SMT_SMOKE_TEST_SOL_PATH), SMT_SMOKE_TEST_SOL_CODE)
 
+    def test_load_source_preserves_lf_newlines(self):
+        expected_output = (
+            "pragma experimental SMTChecker;\n"
+            "\n"
+            "contract C {\n"
+            "}\n"
+        )
+
+        self.assertEqual(load_source(SMT_CONTRACT_WITH_LF_NEWLINES_SOL_PATH), expected_output)
+
+    def test_load_source_preserves_crlf_newlines(self):
+        expected_output = (
+            "pragma experimental SMTChecker;\r\n"
+            "\r\n"
+            "contract C {\r\n"
+            "}\r\n"
+        )
+
+        self.assertEqual(load_source(SMT_CONTRACT_WITH_CRLF_NEWLINES_SOL_PATH), expected_output)
+
+    def test_load_source_preserves_cr_newlines(self):
+        expected_output = (
+            "pragma experimental SMTChecker;\r"
+            "\r"
+            "contract C {\r"
+            "}\r"
+        )
+
+        self.assertEqual(load_source(SMT_CONTRACT_WITH_CR_NEWLINES_SOL_PATH), expected_output)
+
+    def test_load_source_preserves_mixed_newlines(self):
+        expected_output = (
+            "pragma experimental SMTChecker;\n"
+            "\n"
+            "contract C {\r"
+            "}\r\n"
+        )
+
+        self.assertEqual(load_source(SMT_CONTRACT_WITH_MIXED_NEWLINES_SOL_PATH), expected_output)
+
     def test_prepare_compiler_input(self):
         expected_compiler_input = {
             'language': 'Solidity',
@@ -103,6 +148,34 @@ class TestPrepareReport(unittest.TestCase):
         self.assertEqual(command_line, ['solc', '--standard-json'])
         self.assertEqual(json.loads(compiler_input), expected_compiler_input)
 
+    def test_prepare_compiler_input_preserves_newlines(self):
+        expected_compiler_input = {
+            'language': 'Solidity',
+            'sources': {
+                str(SMT_CONTRACT_WITH_MIXED_NEWLINES_SOL_PATH): {
+                    'content':
+                        "pragma experimental SMTChecker;\n"
+                        "\n"
+                        "contract C {\r"
+                        "}\r\n"
+                },
+            },
+            'settings': {
+                'optimizer': {'enabled': True},
+                'outputSelection': {'*': {'*': ['evm.bytecode.object', 'metadata']}},
+                'modelChecker': {'engine': 'none'},
+            }
+        }
+
+        (command_line, compiler_input) = prepare_compiler_input(
+            Path('solc'),
+            SMT_CONTRACT_WITH_MIXED_NEWLINES_SOL_PATH,
+            optimize=True,
+        )
+
+        self.assertEqual(command_line, ['solc', '--standard-json'])
+        self.assertEqual(json.loads(compiler_input), expected_compiler_input)
+
     def test_parse_standard_json_output(self):
         expected_report = FileReport(
             file_name=Path('syntaxTests/scoping/library_inherited2.sol'),

test/scripts/unittest_helpers.py

@@ -5,7 +5,9 @@ LIBSOLIDITY_TEST_DIR = Path(__file__).parent.parent / 'libsolidity'
 FIXTURE_DIR = Path(__file__).parent / 'fixtures'
 
 def load_file(path: Union[Path, str]) -> str:
-    with open(path, 'r', encoding='utf-8') as f:
+    # NOTE: newline='' disables newline conversion.
+    # We want the file exactly as is because changing even a single byte in the source affects metadata.
+    with open(path, 'r', encoding='utf-8', newline='') as f:
         return f.read()
 
 def load_fixture(relative_path: Union[Path, str]) -> str: