Merge pull request #11616 from ethereum/issue-8191-yul

Test yul code blocks in documentation.

docs/internals/optimizer.rst

@@ -519,7 +519,7 @@ compact again at the end.
 ExpressionSplitter
 ^^^^^^^^^^^^^^^^^^
 
-The expression splitter turns expressions like ``add(mload(x), mul(mload(y), 0x20))``
+The expression splitter turns expressions like ``add(mload(0x123), mul(mload(0x456), 0x20))``
 into a sequence of declarations of unique variables that are assigned sub-expressions
 of that expression so that each function call has only variables or literals
 as arguments.
@@ -529,9 +529,9 @@ The above would be transformed into
 .. code-block:: yul
 
     {
-        let _1 := mload(y)
+        let _1 := mload(0x123)
         let _2 := mul(_1, 0x20)
-        let _3 := mload(x)
+        let _3 := mload(0x456)
         let z := add(_3, _2)
     }
 
@@ -633,7 +633,7 @@ The SSA transform converts this snippet to the following:
 
     {
         let a_1 := 1
-        a := a_1
+        let a := a_1
         let a_2 := mload(a_1)
         a := a_2
         let a_3 := sload(a_2)
@@ -1186,16 +1186,18 @@ The SSA transform rewrites
 
 .. code-block:: yul
 
-    a := E
+    let a := calldataload(0)
     mstore(a, 1)
 
 to
 
 .. code-block:: yul
 
-    let a_1 := E
+    let a_1 := calldataload(0)
-    a := a_1
+    let a := a_1
     mstore(a_1, 1)
+    let a_2 := calldataload(0x20)
+    a := a_2
 
 The problem is that instead of ``a``, the variable ``a_1`` is used
 whenever ``a`` was referenced. The SSA transform changes statements
@@ -1204,9 +1206,11 @@ snippet is turned into
 
 .. code-block:: yul
 
-    a := E
+    let a := calldataload(0)
     let a_1 := a
     mstore(a_1, 1)
+    a := calldataload(0x20)
+    let a_2 := a
 
 This is a very simple equivalence transform, but when we now run the
 Common Subexpression Eliminator, it will replace all occurrences of ``a_1``

docs/yul.rst

@@ -198,7 +198,8 @@ has to be specified after a colon:
 
 .. code-block:: yul
 
-    let x := and("abc":uint32, add(3:uint256, 2:uint256))
+    // This will not compile (u32 and u256 type not implemented yet)
+    let x := and("abc":u32, add(3:u256, 2:u256))
 
 
 Function Calls
@@ -212,10 +213,9 @@ they have to be assigned to local variables.
 
 .. code-block:: yul
 
+    function f(x, y) -> a, b { /* ... */ }
     mstore(0x80, add(mload(0x80), 3))
-    // Here, the user-defined function `f` returns
+    // Here, the user-defined function `f` returns two values.
-    // two values. The definition of the function
-    // is missing from the example.
     let x, y := f(1, mload(0))
 
 For built-in functions of the EVM, functional expressions
@@ -271,9 +271,10 @@ that returns multiple values.
 
 .. code-block:: yul
 
+    // This will not compile (u32 and u256 type not implemented yet)
     {
-        let zero:uint32 := 0:uint32
+        let zero:u32 := 0:u32
-        let v:uint256, t:uint32 := f()
+        let v:u256, t:u32 := f()
         let x, y := g()
     }
 
@@ -314,7 +315,7 @@ you need multiple alternatives.
 
 .. code-block:: yul
 
-    if eq(value, 0) { revert(0, 0) }
+    if lt(calldatasize(), 4) { revert(0, 0) }
 
 The curly braces for the body are required.
 

scripts/common_cmdline.sh

@@ -19,6 +19,7 @@
 # (c) 2016-2019 solidity contributors.
 # ------------------------------------------------------------------------------
 
+YULARGS=(--strict-assembly)
 FULLARGS=(--optimize --ignore-missing --combined-json "abi,asm,ast,bin,bin-runtime,compact-format,devdoc,hashes,interface,metadata,opcodes,srcmap,srcmap-runtime,userdoc")
 OLDARGS=(--optimize --combined-json "abi,asm,ast,bin,bin-runtime,devdoc,interface,metadata,opcodes,srcmap,srcmap-runtime,userdoc")
 function compileFull()
@@ -53,10 +54,18 @@ function compileFull()
 
     local stderr_path; stderr_path=$(mktemp)
 
+    if [ "${files: -4}" == ".yul" ]
+    then
+        args=("${YULARGS[@]}")
+    fi
+
     set +e
     "$SOLC" "${args[@]}" "${files[@]}" >/dev/null 2>"$stderr_path"
     local exit_code=$?
-    local errors; errors=$(grep -v -E 'Warning: This is a pre-release compiler version|Warning: Experimental features are turned on|pragma experimental ABIEncoderV2|^ +--> |^ +\||^[0-9]+ +\|' < "$stderr_path")
+    local errors; errors=$(grep -v -E \
+        -e 'Warning: This is a pre-release compiler version|Warning: Experimental features are turned on|pragma experimental ABIEncoderV2|^ +--> |^ +\||^[0-9]+ +\| ' \
+        -e 'Warning: Yul is still experimental. Please use the output with care.' < "$stderr_path")
+
     set -e
     rm "$stderr_path"
 

scripts/isolate_tests.py

@@ -10,8 +10,9 @@ import sys
 import re
 import os
 import hashlib
-from os.path import join, isfile, split, basename
+from os.path import join, isfile, basename
 from argparse import ArgumentParser
+from textwrap import indent, dedent
 
 def extract_test_cases(path):
     with open(path, encoding="utf8", errors='ignore', mode='r', newline='') as file:
@@ -36,11 +37,42 @@ def extract_test_cases(path):
 
     return tests
 
-# Extract code examples based on a start marker
+def extract_solidity_docs_cases(path):
+    tests = extract_docs_cases(path, [".. code-block:: solidity", '::'])
+
+    codeStart = "(// SPDX-License-Identifier:|pragma solidity|contract.*{|library.*{|interface.*{)"
+
+    # Filter out tests that are not supposed to be compilable.
+    return [
+        test.lstrip("\n")
+        for test in tests
+        if re.search(r'^\s{4}' + codeStart, test, re.MULTILINE) is not None
+    ]
+
+def extract_yul_docs_cases(path):
+    tests = extract_docs_cases(path, [".. code-block:: yul"])
+
+    def wrap_in_object(code):
+        for line in code.splitlines():
+            line = line.lstrip()
+            if line.startswith("//"):
+                continue
+            if not line.startswith("object") and not line.startswith("{"):
+                return indent("{{\n{}\n}}\n\n".format(code.rstrip()), "    ")
+            break
+
+        return code
+
+    return [
+        wrap_in_object(test)
+        for test in tests
+        if test.strip() != ""
+    ]
+
+# Extract code examples based on the 'beginMarker' parameter
 # up until we reach EOF or a line that is not empty and doesn't start with 4
 # spaces.
-def extract_docs_cases(path):
+def extract_docs_cases(path, beginMarkers):
-    beginMarkers = ['.. code-block:: solidity', '::']
     immediatelyAfterMarker = False
     insideBlock = False
     tests = []
@@ -59,48 +91,45 @@ def extract_docs_cases(path):
             if line == '' or line.startswith(" "):
                 tests[-1] += line + "\n"
                 immediatelyAfterMarker = False
-            else:
+                continue
+
             insideBlock = False
-        elif any(map(line.lower().startswith, beginMarkers)):
+        if any(map(line.lower().startswith, beginMarkers)):
             insideBlock = True
             immediatelyAfterMarker = True
             tests += ['']
 
-    codeStart = "(// SPDX-License-Identifier:|pragma solidity|contract.*{|library.*{|interface.*{)"
+    return tests
 
-    for test in tests:
+def write_cases(f, solidityTests, yulTests):
-        if re.search(r'^\s{0,3}' + codeStart, test, re.MULTILINE):
-            print("Indentation error in " + path + ":")
-            print(test)
-            exit(1)
-
-    # Filter out tests that are not supposed to be compilable.
-    return [
-        test.lstrip("\n")
-        for test in tests
-        if re.search(r'^\s{4}' + codeStart, test, re.MULTILINE) is not None
-    ]
-
-def write_cases(f, tests):
     cleaned_filename = f.replace(".","_").replace("-","_").replace(" ","_").lower()
-    for test in tests:
+    for language, test in [("sol", t) for t in solidityTests] + [("yul", t) for t in yulTests]:
         # When code examples are extracted they are indented by 8 spaces, which violates the style guide,
         # so before checking remove 4 spaces from each line.
-        remainder = re.sub(r'^ {4}', '', test, 0, re.MULTILINE)
+        remainder = dedent(test)
-        sol_filename = 'test_%s_%s.sol' % (hashlib.sha256(test.encode("utf-8")).hexdigest(), cleaned_filename)
+        sol_filename = 'test_%s_%s.%s' % (hashlib.sha256(test.encode("utf-8")).hexdigest(), cleaned_filename, language)
         with open(sol_filename, mode='w', encoding='utf8', newline='') as fi:
             fi.write(remainder)
 
-def extract_and_write(path):
+def extract_and_write(path, language):
+    assert language in ["solidity", "yul", ""]
+    yulCases = []
+    cases = []
+
     if path.lower().endswith('.rst'):
-        cases = extract_docs_cases(path)
+        if language in ("solidity", ""):
+            cases = extract_solidity_docs_cases(path)
+
+        if language in ("yul", ""):
+            yulCases  = extract_yul_docs_cases(path)
     elif path.endswith('.sol'):
+        if language in ("solidity", ""):
             with open(path, mode='r', encoding='utf8', newline='') as f:
                 cases = [f.read()]
     else:
         cases = extract_test_cases(path)
 
-    write_cases(basename(path), cases)
+    write_cases(basename(path), cases, yulCases)
 
 if __name__ == '__main__':
     script_description = (
@@ -110,11 +139,19 @@ if __name__ == '__main__':
 
     parser = ArgumentParser(description=script_description)
     parser.add_argument(dest='path', help='Path to file or directory to look for code in.')
+    parser.add_argument(
+        '-l', '--language',
+        dest='language',
+        choices=["yul", "solidity"],
+        default="",
+        action='store',
+        help="Extract only code blocks in the given language"
+    )
     options = parser.parse_args()
     path = options.path
 
     if isfile(path):
-        extract_and_write(path)
+        extract_and_write(path, options.language)
     else:
         for root, subdirs, files in os.walk(path):
             if '_build' in subdirs:
@@ -125,4 +162,4 @@ if __name__ == '__main__':
                 if basename(f) == "invalid_utf8_sequence.sol":
                     continue  # ignore the test with broken utf-8 encoding
                 path = join(root, f)
-                extract_and_write(path)
+                extract_and_write(path, options.language)

test/cmdlineTests.sh

@@ -364,7 +364,7 @@ SOLTMPDIR=$(mktemp -d)
     "$REPO_ROOT"/scripts/isolate_tests.py "$REPO_ROOT"/docs/
     developmentVersion=$("$REPO_ROOT/scripts/get_version.sh")
 
-    for f in *.sol
+    for f in *.yul *.sol
     do
         # The contributors guide uses syntax tests, but we cannot
         # really handle them here.

test/scripts/fixtures/code_block.rst

@@ -20,3 +20,25 @@ Some text
     contract C {}
 
 More text.
+
+.. code-block:: yul
+
+    let x := add(1, 5)
+
+.. code-block:: yul
+
+    // Yul code wrapped in object
+    {
+        {
+            let y := mul(3, 5)
+        }
+    }
+
+.. code-block:: yul
+    // Yul code wrapped in named object
+    object "Test" {
+        {
+            let y := mul(6, 9)
+        }
+    }
+

test/scripts/fixtures/code_block_with_directives.rst

@@ -46,3 +46,26 @@ Sphinx does not complain about these.
     contract E {}
 
 More text.
+
+.. code-block:: yul
+
+    :force:
+    let x := add(1, 5)
+
+.. code-block:: yul
+
+    :linenos:
+    :language: Yul
+    // Yul code wrapped in object
+    {
+        let y := mul(3, 5)
+    }
+
+.. code-block:: yul
+
+    // Yul code wrapped in named object
+    object "Test" {
+        let y := mul(3, 5)
+    :linenos:
+    }
+

test/scripts/test_isolate_tests.py

@@ -2,61 +2,119 @@
 
 import unittest
 
+from textwrap import dedent, indent
+
 from unittest_helpers import FIXTURE_DIR, load_fixture
 
 # NOTE: This test file file only works with scripts/ added to PYTHONPATH so pylint can't find the imports
 # pragma pylint: disable=import-error
-from isolate_tests import extract_docs_cases
+from isolate_tests import extract_solidity_docs_cases, extract_yul_docs_cases
 # pragma pylint: enable=import-error
 
-
 CODE_BLOCK_RST_PATH = FIXTURE_DIR / 'code_block.rst'
 CODE_BLOCK_RST_CONTENT = load_fixture(CODE_BLOCK_RST_PATH)
 CODE_BLOCK_WITH_DIRECTIVES_RST_PATH = FIXTURE_DIR / 'code_block_with_directives.rst'
 CODE_BLOCK_WITH_DIRECTIVES_RST_CONTENT = load_fixture(CODE_BLOCK_WITH_DIRECTIVES_RST_PATH)
 
+def formatCase(text):
+    """Formats code to contain only one indentation and terminate with a \n"""
+    return indent(dedent(text.lstrip("\n")), "    ") + "\n"
 
 class TestExtractDocsCases(unittest.TestCase):
     def setUp(self):
         self.maxDiff = 10000
 
+
     def test_solidity_block(self):
-        expected_cases = [
+        expected_cases = [formatCase(case) for case in [
-            "    // SPDX-License-Identifier: GPL-3.0\n"
+            """
-            "    pragma solidity >=0.7.0 <0.9.0;\n"
+                // SPDX-License-Identifier: GPL-3.0
-            "\n"
+                pragma solidity >=0.7.0 <0.9.0;
-            "    contract C {\n"
-            "        function foo() public view {}\n"
-            "    }\n"
-            "\n"
-            "\n",
 
-            "    contract C {}\n"
+                contract C {
-            "\n",
+                    function foo() public view {}
-        ]
+                }
 
-        self.assertEqual(extract_docs_cases(CODE_BLOCK_RST_PATH), expected_cases)
+            """,
+            """
+                contract C {}
+            """,
+        ]]
+
+        self.assertEqual(extract_solidity_docs_cases(CODE_BLOCK_RST_PATH), expected_cases)
 
     def test_solidity_block_with_directives(self):
-        expected_cases = [
+        expected_cases = [formatCase(case) for case in [
-            "    // SPDX-License-Identifier: GPL-3.0\n"
+            """
-            "    pragma solidity >=0.7.0 <0.9.0;\n"
+                // SPDX-License-Identifier: GPL-3.0
-            "\n"
+                pragma solidity >=0.7.0 <0.9.0;
-            "    contract C {\n"
-            "        function foo() public view {}\n"
-            "    }\n"
-            "\n"
-            "\n",
 
-            "    contract C {}\n"
+                contract C {
-            "\n",
+                    function foo() public view {}
+                }
 
-            "    contract D {}\n"
+            """,
-            "    :linenos:\n"
+            """
-            "\n",
+                contract C {}
+            """,
+            """
+                contract D {}
+                :linenos:
+            """,
+            """
+                contract E {}
+            """,
+        ]]
 
-            "    contract E {}\n"
+        self.assertEqual(extract_solidity_docs_cases(CODE_BLOCK_WITH_DIRECTIVES_RST_PATH), expected_cases)
-            "\n",
-        ]
 
-        self.assertEqual(extract_docs_cases(CODE_BLOCK_WITH_DIRECTIVES_RST_PATH), expected_cases)
+    def test_yul_block(self):
+        expected_cases = [formatCase(case) for case in [
+            """
+            {
+                let x := add(1, 5)
+            }
+            """,
+            """
+            // Yul code wrapped in object
+            {
+                {
+                    let y := mul(3, 5)
+                }
+            }
+            """,
+            """
+            // Yul code wrapped in named object
+            object "Test" {
+                {
+                    let y := mul(6, 9)
+                }
+            }
+            """,
+        ]]
+
+        self.assertEqual(extract_yul_docs_cases(CODE_BLOCK_RST_PATH), expected_cases)
+
+    def test_yul_block_with_directives(self):
+        expected_cases = [formatCase(case) for case in [
+            """
+            {
+                let x := add(1, 5)
+            }
+            """,
+            """
+            // Yul code wrapped in object
+            {
+                let y := mul(3, 5)
+            }
+            """,
+            """
+            // Yul code wrapped in named object
+            object "Test" {
+                let y := mul(3, 5)
+            :linenos:
+            }
+            """,
+        ]]
+
+        self.assertEqual(extract_yul_docs_cases(CODE_BLOCK_WITH_DIRECTIVES_RST_PATH), expected_cases)