diff --git a/docs/internals/optimizer.rst b/docs/internals/optimizer.rst index c1107d1b4..142a89602 100644 --- a/docs/internals/optimizer.rst +++ b/docs/internals/optimizer.rst @@ -519,7 +519,7 @@ compact again at the end. ExpressionSplitter ^^^^^^^^^^^^^^^^^^ -The expression splitter turns expressions like ``add(mload(x), mul(mload(y), 0x20))`` +The expression splitter turns expressions like ``add(mload(0x123), mul(mload(0x456), 0x20))`` into a sequence of declarations of unique variables that are assigned sub-expressions of that expression so that each function call has only variables or literals as arguments. @@ -529,9 +529,9 @@ The above would be transformed into .. code-block:: yul { - let _1 := mload(y) + let _1 := mload(0x123) let _2 := mul(_1, 0x20) - let _3 := mload(x) + let _3 := mload(0x456) let z := add(_3, _2) } @@ -633,7 +633,7 @@ The SSA transform converts this snippet to the following: { let a_1 := 1 - a := a_1 + let a := a_1 let a_2 := mload(a_1) a := a_2 let a_3 := sload(a_2) @@ -1186,16 +1186,18 @@ The SSA transform rewrites .. code-block:: yul - a := E + let a := calldataload(0) mstore(a, 1) to .. code-block:: yul - let a_1 := E - a := a_1 + let a_1 := calldataload(0) + let a := a_1 mstore(a_1, 1) + let a_2 := calldataload(0x20) + a := a_2 The problem is that instead of ``a``, the variable ``a_1`` is used whenever ``a`` was referenced. The SSA transform changes statements @@ -1204,9 +1206,11 @@ snippet is turned into .. code-block:: yul - a := E + let a := calldataload(0) let a_1 := a mstore(a_1, 1) + a := calldataload(0x20) + let a_2 := a This is a very simple equivalence transform, but when we now run the Common Subexpression Eliminator, it will replace all occurrences of ``a_1`` diff --git a/docs/yul.rst b/docs/yul.rst index 55e8b824f..093bf6597 100644 --- a/docs/yul.rst +++ b/docs/yul.rst @@ -198,7 +198,8 @@ has to be specified after a colon: .. code-block:: yul - let x := and("abc":uint32, add(3:uint256, 2:uint256)) + // This will not compile (u32 and u256 type not implemented yet) + let x := and("abc":u32, add(3:u256, 2:u256)) Function Calls @@ -212,10 +213,9 @@ they have to be assigned to local variables. .. code-block:: yul + function f(x, y) -> a, b { /* ... */ } mstore(0x80, add(mload(0x80), 3)) - // Here, the user-defined function `f` returns - // two values. The definition of the function - // is missing from the example. + // Here, the user-defined function `f` returns two values. let x, y := f(1, mload(0)) For built-in functions of the EVM, functional expressions @@ -271,9 +271,10 @@ that returns multiple values. .. code-block:: yul + // This will not compile (u32 and u256 type not implemented yet) { - let zero:uint32 := 0:uint32 - let v:uint256, t:uint32 := f() + let zero:u32 := 0:u32 + let v:u256, t:u32 := f() let x, y := g() } @@ -314,7 +315,7 @@ you need multiple alternatives. .. code-block:: yul - if eq(value, 0) { revert(0, 0) } + if lt(calldatasize(), 4) { revert(0, 0) } The curly braces for the body are required. diff --git a/scripts/common_cmdline.sh b/scripts/common_cmdline.sh index 2637fdedb..88dab9cd0 100644 --- a/scripts/common_cmdline.sh +++ b/scripts/common_cmdline.sh @@ -19,6 +19,7 @@ # (c) 2016-2019 solidity contributors. # ------------------------------------------------------------------------------ +YULARGS=(--strict-assembly) FULLARGS=(--optimize --ignore-missing --combined-json "abi,asm,ast,bin,bin-runtime,compact-format,devdoc,hashes,interface,metadata,opcodes,srcmap,srcmap-runtime,userdoc") OLDARGS=(--optimize --combined-json "abi,asm,ast,bin,bin-runtime,devdoc,interface,metadata,opcodes,srcmap,srcmap-runtime,userdoc") function compileFull() @@ -53,10 +54,18 @@ function compileFull() local stderr_path; stderr_path=$(mktemp) + if [ "${files: -4}" == ".yul" ] + then + args=("${YULARGS[@]}") + fi + set +e "$SOLC" "${args[@]}" "${files[@]}" >/dev/null 2>"$stderr_path" local exit_code=$? - local errors; errors=$(grep -v -E 'Warning: This is a pre-release compiler version|Warning: Experimental features are turned on|pragma experimental ABIEncoderV2|^ +--> |^ +\||^[0-9]+ +\|' < "$stderr_path") + local errors; errors=$(grep -v -E \ + -e 'Warning: This is a pre-release compiler version|Warning: Experimental features are turned on|pragma experimental ABIEncoderV2|^ +--> |^ +\||^[0-9]+ +\| ' \ + -e 'Warning: Yul is still experimental. Please use the output with care.' < "$stderr_path") + set -e rm "$stderr_path" diff --git a/scripts/isolate_tests.py b/scripts/isolate_tests.py index fdb398c4b..f3a121f3d 100755 --- a/scripts/isolate_tests.py +++ b/scripts/isolate_tests.py @@ -10,8 +10,9 @@ import sys import re import os import hashlib -from os.path import join, isfile, split, basename +from os.path import join, isfile, basename from argparse import ArgumentParser +from textwrap import indent, dedent def extract_test_cases(path): with open(path, encoding="utf8", errors='ignore', mode='r', newline='') as file: @@ -36,11 +37,42 @@ def extract_test_cases(path): return tests -# Extract code examples based on a start marker +def extract_solidity_docs_cases(path): + tests = extract_docs_cases(path, [".. code-block:: solidity", '::']) + + codeStart = "(// SPDX-License-Identifier:|pragma solidity|contract.*{|library.*{|interface.*{)" + + # Filter out tests that are not supposed to be compilable. + return [ + test.lstrip("\n") + for test in tests + if re.search(r'^\s{4}' + codeStart, test, re.MULTILINE) is not None + ] + +def extract_yul_docs_cases(path): + tests = extract_docs_cases(path, [".. code-block:: yul"]) + + def wrap_in_object(code): + for line in code.splitlines(): + line = line.lstrip() + if line.startswith("//"): + continue + if not line.startswith("object") and not line.startswith("{"): + return indent("{{\n{}\n}}\n\n".format(code.rstrip()), " ") + break + + return code + + return [ + wrap_in_object(test) + for test in tests + if test.strip() != "" + ] + +# Extract code examples based on the 'beginMarker' parameter # up until we reach EOF or a line that is not empty and doesn't start with 4 # spaces. -def extract_docs_cases(path): - beginMarkers = ['.. code-block:: solidity', '::'] +def extract_docs_cases(path, beginMarkers): immediatelyAfterMarker = False insideBlock = False tests = [] @@ -59,48 +91,45 @@ def extract_docs_cases(path): if line == '' or line.startswith(" "): tests[-1] += line + "\n" immediatelyAfterMarker = False - else: - insideBlock = False - elif any(map(line.lower().startswith, beginMarkers)): + continue + + insideBlock = False + if any(map(line.lower().startswith, beginMarkers)): insideBlock = True immediatelyAfterMarker = True tests += [''] - codeStart = "(// SPDX-License-Identifier:|pragma solidity|contract.*{|library.*{|interface.*{)" + return tests - for test in tests: - if re.search(r'^\s{0,3}' + codeStart, test, re.MULTILINE): - print("Indentation error in " + path + ":") - print(test) - exit(1) - - # Filter out tests that are not supposed to be compilable. - return [ - test.lstrip("\n") - for test in tests - if re.search(r'^\s{4}' + codeStart, test, re.MULTILINE) is not None - ] - -def write_cases(f, tests): +def write_cases(f, solidityTests, yulTests): cleaned_filename = f.replace(".","_").replace("-","_").replace(" ","_").lower() - for test in tests: + for language, test in [("sol", t) for t in solidityTests] + [("yul", t) for t in yulTests]: # When code examples are extracted they are indented by 8 spaces, which violates the style guide, # so before checking remove 4 spaces from each line. - remainder = re.sub(r'^ {4}', '', test, 0, re.MULTILINE) - sol_filename = 'test_%s_%s.sol' % (hashlib.sha256(test.encode("utf-8")).hexdigest(), cleaned_filename) + remainder = dedent(test) + sol_filename = 'test_%s_%s.%s' % (hashlib.sha256(test.encode("utf-8")).hexdigest(), cleaned_filename, language) with open(sol_filename, mode='w', encoding='utf8', newline='') as fi: fi.write(remainder) -def extract_and_write(path): +def extract_and_write(path, language): + assert language in ["solidity", "yul", ""] + yulCases = [] + cases = [] + if path.lower().endswith('.rst'): - cases = extract_docs_cases(path) + if language in ("solidity", ""): + cases = extract_solidity_docs_cases(path) + + if language in ("yul", ""): + yulCases = extract_yul_docs_cases(path) elif path.endswith('.sol'): - with open(path, mode='r', encoding='utf8', newline='') as f: - cases = [f.read()] + if language in ("solidity", ""): + with open(path, mode='r', encoding='utf8', newline='') as f: + cases = [f.read()] else: cases = extract_test_cases(path) - write_cases(basename(path), cases) + write_cases(basename(path), cases, yulCases) if __name__ == '__main__': script_description = ( @@ -110,11 +139,19 @@ if __name__ == '__main__': parser = ArgumentParser(description=script_description) parser.add_argument(dest='path', help='Path to file or directory to look for code in.') + parser.add_argument( + '-l', '--language', + dest='language', + choices=["yul", "solidity"], + default="", + action='store', + help="Extract only code blocks in the given language" + ) options = parser.parse_args() path = options.path if isfile(path): - extract_and_write(path) + extract_and_write(path, options.language) else: for root, subdirs, files in os.walk(path): if '_build' in subdirs: @@ -125,4 +162,4 @@ if __name__ == '__main__': if basename(f) == "invalid_utf8_sequence.sol": continue # ignore the test with broken utf-8 encoding path = join(root, f) - extract_and_write(path) + extract_and_write(path, options.language) diff --git a/test/cmdlineTests.sh b/test/cmdlineTests.sh index 7a14fe6c8..a864ce17a 100755 --- a/test/cmdlineTests.sh +++ b/test/cmdlineTests.sh @@ -364,7 +364,7 @@ SOLTMPDIR=$(mktemp -d) "$REPO_ROOT"/scripts/isolate_tests.py "$REPO_ROOT"/docs/ developmentVersion=$("$REPO_ROOT/scripts/get_version.sh") - for f in *.sol + for f in *.yul *.sol do # The contributors guide uses syntax tests, but we cannot # really handle them here. diff --git a/test/scripts/fixtures/code_block.rst b/test/scripts/fixtures/code_block.rst index f97967b93..a0bb7a91e 100644 --- a/test/scripts/fixtures/code_block.rst +++ b/test/scripts/fixtures/code_block.rst @@ -20,3 +20,25 @@ Some text contract C {} More text. + +.. code-block:: yul + + let x := add(1, 5) + +.. code-block:: yul + + // Yul code wrapped in object + { + { + let y := mul(3, 5) + } + } + +.. code-block:: yul + // Yul code wrapped in named object + object "Test" { + { + let y := mul(6, 9) + } + } + diff --git a/test/scripts/fixtures/code_block_with_directives.rst b/test/scripts/fixtures/code_block_with_directives.rst index ff52d5b0e..fba8ed7dd 100644 --- a/test/scripts/fixtures/code_block_with_directives.rst +++ b/test/scripts/fixtures/code_block_with_directives.rst @@ -46,3 +46,26 @@ Sphinx does not complain about these. contract E {} More text. + +.. code-block:: yul + + :force: + let x := add(1, 5) + +.. code-block:: yul + + :linenos: + :language: Yul + // Yul code wrapped in object + { + let y := mul(3, 5) + } + +.. code-block:: yul + + // Yul code wrapped in named object + object "Test" { + let y := mul(3, 5) + :linenos: + } + diff --git a/test/scripts/test_isolate_tests.py b/test/scripts/test_isolate_tests.py index a62a528a2..16874b747 100644 --- a/test/scripts/test_isolate_tests.py +++ b/test/scripts/test_isolate_tests.py @@ -8,7 +8,7 @@ from unittest_helpers import FIXTURE_DIR, load_fixture # NOTE: This test file file only works with scripts/ added to PYTHONPATH so pylint can't find the imports # pragma pylint: disable=import-error -from isolate_tests import extract_docs_cases +from isolate_tests import extract_solidity_docs_cases, extract_yul_docs_cases # pragma pylint: enable=import-error CODE_BLOCK_RST_PATH = FIXTURE_DIR / 'code_block.rst' @@ -41,7 +41,7 @@ class TestExtractDocsCases(unittest.TestCase): """, ]] - self.assertEqual(extract_docs_cases(CODE_BLOCK_RST_PATH), expected_cases) + self.assertEqual(extract_solidity_docs_cases(CODE_BLOCK_RST_PATH), expected_cases) def test_solidity_block_with_directives(self): expected_cases = [formatCase(case) for case in [ @@ -66,4 +66,55 @@ class TestExtractDocsCases(unittest.TestCase): """, ]] - self.assertEqual(extract_docs_cases(CODE_BLOCK_WITH_DIRECTIVES_RST_PATH), expected_cases) + self.assertEqual(extract_solidity_docs_cases(CODE_BLOCK_WITH_DIRECTIVES_RST_PATH), expected_cases) + + def test_yul_block(self): + expected_cases = [formatCase(case) for case in [ + """ + { + let x := add(1, 5) + } + """, + """ + // Yul code wrapped in object + { + { + let y := mul(3, 5) + } + } + """, + """ + // Yul code wrapped in named object + object "Test" { + { + let y := mul(6, 9) + } + } + """, + ]] + + self.assertEqual(extract_yul_docs_cases(CODE_BLOCK_RST_PATH), expected_cases) + + def test_yul_block_with_directives(self): + expected_cases = [formatCase(case) for case in [ + """ + { + let x := add(1, 5) + } + """, + """ + // Yul code wrapped in object + { + let y := mul(3, 5) + } + """, + """ + // Yul code wrapped in named object + object "Test" { + let y := mul(3, 5) + :linenos: + } + """, + ]] + + self.assertEqual(extract_yul_docs_cases(CODE_BLOCK_WITH_DIRECTIVES_RST_PATH), expected_cases)