Merge pull request #12604 from ethereum/develop

Merge develop into breaking
2023-10-03 13:03:40 +00:00 · 2022-01-31 17:59:03 +01:00
parent 017519f555 ef8911a629
commit 814e233b67
206 changed files with 3675 additions and 1271 deletions
@@ -34,7 +34,7 @@ else
    BUILD_DIR="$1"
 fi

-# solbuildpackpusher/solidity-buildpack-deps:emscripten-8
+# solbuildpackpusher/solidity-buildpack-deps:emscripten-9
 docker run -v "$(pwd):/root/project" -w /root/project \
-    solbuildpackpusher/solidity-buildpack-deps@sha256:842d6074e0e7e5355c89122c1cafc1fdb59696596750e7d56e5f35c0d883ad59 \
+    solbuildpackpusher/solidity-buildpack-deps@sha256:d51534dfdd05ece86f69ed7beafd68c15b88606da00a4b7fe2873ccfbd0dce24\
    ./scripts/ci/build_emscripten.sh "$BUILD_DIR"
@@ -2,7 +2,7 @@
 const process = require('process')
 const fs = require('fs')

-const compiler = require('./solc-js/wrapper.js')(require('./solc-js/soljson.js'))
+const compiler = require('solc')


 function loadSource(sourceFileName, stripSMTPragmas)
@@ -187,7 +187,7 @@ def parse_cli_output(source_file_name: Path, cli_output: str) -> FileReport:
    return file_report


-def prepare_compiler_input(  # pylint: disable=too-many-arguments
+def prepare_compiler_input(
    compiler_path: Path,
    source_file_name: Path,
    optimize: bool,
@@ -256,7 +256,7 @@ def detect_metadata_cli_option_support(compiler_path: Path):
    return process.returncode == 0


-def run_compiler(  # pylint: disable=too-many-arguments
+def run_compiler(
    compiler_path: Path,
    source_file_name: Path,
    optimize: bool,
@@ -320,7 +320,7 @@ def run_compiler(  # pylint: disable=too-many-arguments
        return parse_cli_output(Path(source_file_name), process.stdout)


-def generate_report(  # pylint: disable=too-many-arguments,too-many-locals
+def generate_report(
    source_file_names: List[str],
    compiler_path: Path,
    interface: CompilerInterface,
@@ -42,11 +42,18 @@ TMPDIR=$(mktemp -d)

    if [[ "$SOLC_EMSCRIPTEN" = "On" ]]
    then
+        echo "Installing solc-js..."
        # npm install solc
        git clone --depth 1 https://github.com/ethereum/solc-js.git solc-js
-        ( cd solc-js; npm install )
        cp "$REPO_ROOT/emscripten_build/libsolc/soljson.js" solc-js/
+        pushd solc-js/
+        npm install
+        npm run build
+        popd
+
        cp "$REPO_ROOT/scripts/bytecodecompare/prepare_report.js" .
+        npm install solc-js/
+
        echo "Running the compiler..."
        # shellcheck disable=SC2035
        ./prepare_report.js *.sol > report.txt
@@ -25,9 +25,9 @@ set -ev
 keyid=70D110489D66E2F6
 email=builds@ethereum.org
 packagename=z3-static
-version=4.8.13
+version=4.8.14

-DISTRIBUTIONS="focal groovy hirsute"
+DISTRIBUTIONS="focal hirsute impish jammy"

 for distribution in $DISTRIBUTIONS
 do
@@ -33,12 +33,12 @@
 # Using $(em-config CACHE)/sysroot/usr seems to work, though, and still has cmake find the
 # dependencies automatically.
 FROM emscripten/emsdk:2.0.33 AS base
-LABEL version="8"
+LABEL version="9"

 ADD emscripten.jam /usr/src
 RUN set -ex; \
 	cd /usr/src; \
-	git clone https://github.com/Z3Prover/z3.git -b z3-4.8.13 --depth 1 ; \
+	git clone https://github.com/Z3Prover/z3.git -b z3-4.8.14 --depth 1 ; \
 	cd z3; \
 	mkdir build; \
 	cd build; \
@@ -22,7 +22,7 @@
 # (c) 2016-2021 solidity contributors.
 #------------------------------------------------------------------------------
 FROM gcr.io/oss-fuzz-base/base-clang:latest as base
-LABEL version="14"
+LABEL version="15"

 ARG DEBIAN_FRONTEND=noninteractive

@@ -61,7 +61,7 @@ RUN set -ex; \

 # Z3
 RUN set -ex; \
-    git clone --depth 1 -b z3-4.8.13 https://github.com/Z3Prover/z3.git \
+    git clone --depth 1 -b z3-4.8.14 https://github.com/Z3Prover/z3.git \
    /usr/src/z3; \
    cd /usr/src/z3; \
    mkdir build; \
@@ -22,7 +22,7 @@
 # (c) 2016-2019 solidity contributors.
 #------------------------------------------------------------------------------
 FROM buildpack-deps:focal AS base
-LABEL version="9"
+LABEL version="10"

 ARG DEBIAN_FRONTEND=noninteractive

@@ -22,7 +22,7 @@
 # (c) 2016-2019 solidity contributors.
 #------------------------------------------------------------------------------
 FROM buildpack-deps:focal AS base
-LABEL version="9"
+LABEL version="10"

 ARG DEBIAN_FRONTEND=noninteractive

@@ -33,12 +33,12 @@ def parse_call(call):
    return function.strip(), arguments.strip(), results.strip()


-def colorize(left, right, id):
+def colorize(left, right, index):
    red = "\x1b[31m"
    yellow = "\x1b[33m"
    reset = "\x1b[0m"
    colors = [red, yellow]
-    color = colors[id % len(colors)]
+    color = colors[index % len(colors)]
    function, _arguments, _results = parse_call(right)
    left = left.replace("compileAndRun", color + "compileAndRun" + reset)
    right = right.replace("constructor", color + "constructor" + reset)
@@ -9,8 +9,6 @@
 #
 # verify-testcases.py will compare both traces. If these traces are identical, the extracted tests were
 # identical with the tests specified in SolidityEndToEndTest.cpp.
-#
-# pylint: disable=too-many-instance-attributes

 import re
 import os
@@ -32,11 +30,11 @@ class Trace:
    def get_input(self):
        return self._input

-    def set_input(self, input):
+    def set_input(self, bytecode):
        if self.kind == "create":
            # remove cbor encoded metadata from bytecode
-            length = int(input[-4:], 16) * 2
-            self._input = input[:len(input) - length - 4]
+            length = int(bytecode[-4:], 16) * 2
+            self._input = bytecode[:len(bytecode) - length - 4]

    def get_output(self):
        return self._output
@@ -110,21 +108,21 @@ class TraceAnalyser:

    @staticmethod
    def parse_parameters(line, trace):
-        input = re.search(r'\s*in:\s*([a-fA-F0-9]*)', line, re.M | re.I)
-        if input:
-            trace.input = input.group(1)
-        output = re.search(r'\s*out:\s*([a-fA-F0-9]*)', line, re.M | re.I)
-        if output:
-            trace.output = output.group(1)
-        result = re.search(r'\s*result:\s*([a-fA-F0-9]*)', line, re.M | re.I)
-        if result:
-            trace.result = result.group(1)
-        gas_used = re.search(r'\s*gas\sused:\s*([a-fA-F0-9]*)', line, re.M | re.I)
-        if gas_used:
-            trace.gas = gas_used.group(1)
-        value = re.search(r'\s*value:\s*([a-fA-F0-9]*)', line, re.M | re.I)
-        if value:
-            trace.value = value.group(1)
+        input_match = re.search(r'\s*in:\s*([a-fA-F0-9]*)', line, re.M | re.I)
+        if input_match:
+            trace.input = input_match.group(1)
+        output_match = re.search(r'\s*out:\s*([a-fA-F0-9]*)', line, re.M | re.I)
+        if output_match:
+            trace.output = output_match.group(1)
+        result_match = re.search(r'\s*result:\s*([a-fA-F0-9]*)', line, re.M | re.I)
+        if result_match:
+            trace.result = result_match.group(1)
+        gas_used_match = re.search(r'\s*gas\sused:\s*([a-fA-F0-9]*)', line, re.M | re.I)
+        if gas_used_match:
+            trace.gas = gas_used_match.group(1)
+        value_match = re.search(r'\s*value:\s*([a-fA-F0-9]*)', line, re.M | re.I)
+        if value_match:
+            trace.value = value_match.group(1)

    def diff(self, analyser):
        if not self.ready:
@@ -154,7 +152,8 @@ class TraceAnalyser:

        print(len(intersection), "test-cases - ", len(mismatches), " mismatche(s)")

-    def check_traces(self, test_name, left, right, mismatches):
+    @classmethod
+    def check_traces(cls, test_name, left, right, mismatches):
        for trace_id, trace in enumerate(left.traces):
            left_trace = trace
            right_trace = right.traces[trace_id]
@@ -18,7 +18,7 @@ def read_file(file_name):
        with open(file_name, "r", encoding="latin-1" if is_latin else ENCODING) as f:
            content = f.read()
    finally:
-        if content == None:
+        if content is None:
            print(f"Error reading: {file_name}")
    return content

@@ -44,11 +44,11 @@ def find_ids_in_source_file(file_name, id_to_file_names):
        if in_comment(source, m.start()):
            continue
        underscore_pos = m.group(0).index("_")
-        id = m.group(0)[0:underscore_pos]
-        if id in id_to_file_names:
-            id_to_file_names[id].append(file_name)
+        error_id = m.group(0)[0:underscore_pos]
+        if error_id in id_to_file_names:
+            id_to_file_names[error_id].append(file_name)
        else:
-            id_to_file_names[id] = [file_name]
+            id_to_file_names[error_id] = [file_name]


 def find_ids_in_source_files(file_names):
@@ -76,16 +76,16 @@ def fix_ids_in_source_file(file_name, id_to_count, available_ids):
        destination.extend(source[k:m.start()])

        underscore_pos = m.group(0).index("_")
-        id = m.group(0)[0:underscore_pos]
+        error_id = m.group(0)[0:underscore_pos]

        # incorrect id or id has a duplicate somewhere
-        if not in_comment(source, m.start()) and (len(id) != 4 or id[0] == "0" or id_to_count[id] > 1):
-            assert id in id_to_count
+        if not in_comment(source, m.start()) and (len(error_id) != 4 or error_id[0] == "0" or id_to_count[error_id] > 1):
+            assert error_id in id_to_count
            new_id = get_next_id(available_ids)
            assert new_id not in id_to_count
-            id_to_count[id] -= 1
+            id_to_count[error_id] -= 1
        else:
-            new_id = id
+            new_id = error_id

        destination.extend(new_id + "_error")
        k = m.end()
@@ -104,7 +104,7 @@ def fix_ids_in_source_files(file_names, id_to_count):
    id_to_count contains number of appearances of every id in sources
    """

-    available_ids = {str(id) for id in range(1000, 10000)} - id_to_count.keys()
+    available_ids = {str(error_id) for error_id in range(1000, 10000)} - id_to_count.keys()
    for file_name in file_names:
        fix_ids_in_source_file(file_name, id_to_count, available_ids)

@@ -113,8 +113,8 @@ def find_files(top_dir, sub_dirs, extensions):
    """Builds a list of files with given extensions in specified subdirectories"""

    source_file_names = []
-    for dir in sub_dirs:
-        for root, _, file_names in os.walk(os.path.join(top_dir, dir), onerror=lambda e: exit(f"Walk error: {e}")):
+    for directory in sub_dirs:
+        for root, _, file_names in os.walk(os.path.join(top_dir, directory), onerror=lambda e: sys.exit(f"Walk error: {e}")):
            for file_name in file_names:
                _, ext = path.splitext(file_name)
                if ext in extensions:
@@ -145,27 +145,27 @@ def find_ids_in_cmdline_test_err(file_name):


 def print_ids(ids):
-    for k, id in enumerate(sorted(ids)):
+    for k, error_id in enumerate(sorted(ids)):
        if k % 10 > 0:
            print(" ", end="")
        elif k > 0:
            print()
-        print(id, end="")
+        print(error_id, end="")


 def print_ids_per_file(ids, id_to_file_names, top_dir):
    file_name_to_ids = {}
-    for id in ids:
-        for file_name in id_to_file_names[id]:
+    for error_id in ids:
+        for file_name in id_to_file_names[error_id]:
            relpath = path.relpath(file_name, top_dir)
            if relpath not in file_name_to_ids:
                file_name_to_ids[relpath] = []
-            file_name_to_ids[relpath].append(id)
+            file_name_to_ids[relpath].append(error_id)

    for file_name in sorted(file_name_to_ids):
        print(file_name)
-        for id in sorted(file_name_to_ids[file_name]):
-            print(f" {id}", end="")
+        for error_id in sorted(file_name_to_ids[file_name]):
+            print(f" {error_id}", end="")
        print()


@@ -254,8 +254,6 @@ def examine_id_coverage(top_dir, source_id_to_file_names, new_ids_only=False):


 def main(argv):
-    # pylint: disable=too-many-branches, too-many-locals, too-many-statements
-
    check = False
    fix = False
    no_confirm = False
@@ -277,7 +275,7 @@ def main(argv):

    if [check, fix, examine_coverage, next_id].count(True) != 1:
        print("usage: python error_codes.py --check | --fix [--no-confirm] | --examine-coverage | --next")
-        exit(1)
+        sys.exit(1)

    cwd = os.getcwd()

@@ -289,23 +287,23 @@ def main(argv):
    source_id_to_file_names = find_ids_in_source_files(source_file_names)

    ok = True
-    for id in sorted(source_id_to_file_names):
-        if len(id) != 4:
-            print(f"ID {id} length != 4")
+    for error_id in sorted(source_id_to_file_names):
+        if len(error_id) != 4:
+            print(f"ID {error_id} length != 4")
            ok = False
-        if id[0] == "0":
-            print(f"ID {id} starts with zero")
+        if error_id[0] == "0":
+            print(f"ID {error_id} starts with zero")
            ok = False
-        if len(source_id_to_file_names[id]) > 1:
-            print(f"ID {id} appears {len(source_id_to_file_names[id])} times")
+        if len(source_id_to_file_names[error_id]) > 1:
+            print(f"ID {error_id} appears {len(source_id_to_file_names[error_id])} times")
            ok = False

    if examine_coverage:
        if not ok:
            print("Incorrect IDs have to be fixed before applying --examine-coverage")
-            exit(1)
+            sys.exit(1)
        res = 0 if examine_id_coverage(cwd, source_id_to_file_names) else 1
-        exit(res)
+        sys.exit(res)

    ok &= examine_id_coverage(cwd, source_id_to_file_names, new_ids_only=True)

@@ -314,18 +312,18 @@ def main(argv):
    if next_id:
        if not ok:
            print("Incorrect IDs have to be fixed before applying --next")
-            exit(1)
-        available_ids = {str(id) for id in range(1000, 10000)} - source_id_to_file_names.keys()
+            sys.exit(1)
+        available_ids = {str(error_id) for error_id in range(1000, 10000)} - source_id_to_file_names.keys()
        next_id = get_next_id(available_ids)
        print(f"Next ID: {next_id}")
-        exit(0)
+        sys.exit(0)

    if ok:
        print("No incorrect IDs found")
-        exit(0)
+        sys.exit(0)

    if check:
-        exit(1)
+        sys.exit(1)

    assert fix, "Unexpected state, should not come here without --fix"

@@ -338,14 +336,14 @@ def main(argv):
        while len(answer) == 0 or answer not in "YNyn":
            answer = input("[Y/N]? ")
        if answer not in "yY":
-            exit(1)
+            sys.exit(1)

    # number of appearances for every id
-    source_id_to_count = { id: len(file_names) for id, file_names in source_id_to_file_names.items() }
+    source_id_to_count = { error_id: len(file_names) for error_id, file_names in source_id_to_file_names.items() }

    fix_ids_in_source_files(source_file_names, source_id_to_count)
    print("Fixing completed")
-    exit(2)
+    sys.exit(2)


 if __name__ == "__main__":
@@ -106,8 +106,8 @@ def write_cases(f, solidityTests, yulTests):
        # When code examples are extracted they are indented by 8 spaces, which violates the style guide,
        # so before checking remove 4 spaces from each line.
        remainder = dedent(test)
-        hash = hashlib.sha256(test.encode("utf-8")).hexdigest()
-        sol_filename = f'test_{hash}_{cleaned_filename}.{language}'
+        source_code_hash = hashlib.sha256(test.encode("utf-8")).hexdigest()
+        sol_filename = f'test_{source_code_hash}_{cleaned_filename}.{language}'
        with open(sol_filename, mode='w', encoding='utf8', newline='') as fi:
            fi.write(remainder)

@@ -6,9 +6,9 @@ Runs pylint on all Python files in project directories known to contain Python s

 from argparse import ArgumentParser
 from os import path, walk
-from sys import exit
 from textwrap import dedent
 import subprocess
+import sys

 PROJECT_ROOT = path.dirname(path.dirname(path.realpath(__file__)))
 PYLINT_RCFILE = f"{PROJECT_ROOT}/scripts/pylintrc"
@@ -89,7 +89,7 @@ def main():
    success = pylint_all_filenames(options.dev_mode, rootdirs)

    if not success:
-        exit(1)
+        sys.exit(1)
    else:
        print("No problems found.")

@@ -98,4 +98,4 @@ if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
-        exit("Interrupted by user. Exiting.")
+        sys.exit("Interrupted by user. Exiting.")
@@ -14,24 +14,23 @@
 # ATTENTION: This list should be extended with care, consider using NOLINT comments inside your
 # python files instead, as the goal is actually to reduce the list of globally disabled checks.
 #
-# TODO: What could be eliminated in future PRs: bad-continuation, invalid-name, redefined-builtin,
-# undefined-variable, unused-*, useless-object-inheritance.
+# TODO: What could be eliminated in future PRs: invalid-name, pointless-string-statement, redefined-outer-name.
 disable=
-    bad-continuation,
    bad-indentation,
    bad-whitespace,
-    consider-using-sys-exit,
    duplicate-code,
    invalid-name,
    missing-docstring,
    no-else-return,
-    no-self-use,
    pointless-string-statement,
-    redefined-builtin,
    redefined-outer-name,
-    singleton-comparison,
    too-few-public-methods,
+    too-many-arguments,
+    too-many-branches,
+    too-many-instance-attributes,
+    too-many-locals,
    too-many-public-methods,
+    too-many-statements,
    ungrouped-imports

 [BASIC]
@@ -41,7 +41,8 @@ class regressor:
                                         "build/test/tools/ossfuzz")
        self._logpath = os.path.join(self._repo_root, "test_results")

-    def parseCmdLine(self, description, args):
+    @classmethod
+    def parseCmdLine(cls, description, args):
        argParser = ArgumentParser(description)
        argParser.add_argument('-o', '--out-dir', required=True, type=str,
                               help="""Directory where test results will be written""")
@@ -57,7 +57,7 @@ packagename=solc

 static_build_distribution=hirsute

-DISTRIBUTIONS="focal hirsute impish"
+DISTRIBUTIONS="focal hirsute impish jammy"

 if is_release
 then
@@ -109,6 +109,7 @@ cd "$tmp_dir"
 git clone https://github.com/ethereum/solc-js.git "$solcjs_dir"
 cd "$solcjs_dir"
 npm install
+npm run build

 cd "${solc_bin_dir}/${platform}/"
 echo "Commit range: ${base_ref}..${top_ref}"
@@ -147,11 +148,12 @@ for binary_name in $platform_binaries; do

        if [[ $platform == emscripten-wasm32 ]] || [[ $platform == emscripten-asmjs ]]; then
            ln -sf "${solc_bin_dir}/${platform}/${binary_name}" "${solcjs_dir}/soljson.js"
+            ln -sf "${solc_bin_dir}/${platform}/${binary_name}" "${solcjs_dir}/dist/soljson.js"
            ln -s "${solcjs_dir}" solc-js
            cp "${script_dir}/bytecodecompare/prepare_report.js" prepare_report.js

            validate_reported_version \
-                "$(solc-js/solcjs --version)" \
+                "$(solc-js/dist/solc.js --version)" \
                "$solidity_version_and_commit"

            # shellcheck disable=SC2035
@@ -40,7 +40,7 @@ def writeSourceToFile(lines):
    filePath, srcName = extractSourceName(lines[0])
    # print("sourceName is ", srcName)
    # print("filePath is", filePath)
-    if filePath != False:
+    if filePath:
        os.system("mkdir -p " + filePath)
    with open(srcName, mode='a+', encoding='utf8', newline='') as f:
        createdSources.append(srcName)
@@ -47,8 +47,8 @@ def write_cases(f, tests):
    cleaned_filename = f.replace(".","_").replace("-","_").replace(" ","_").lower()
    for test in tests:
        remainder = re.sub(r'^ {4}', '', test, 0, re.MULTILINE)
-        hash = hashlib.sha256(test).hexdigest()
-        with open(f'test_{hash}_{cleaned_filename}.sol', 'w', encoding='utf8') as _f:
+        source_code_hash = hashlib.sha256(test).hexdigest()
+        with open(f'test_{source_code_hash}_{cleaned_filename}.sol', 'w', encoding='utf8') as _f:
            _f.write(remainder)


@@ -228,6 +228,7 @@ mkdir -p "${OUTPUTDIR}"/bin
 echo "Prepare solc-js."
 cd /root/solc-js
 npm install >/dev/null 2>&1
+npm run build >/dev/null 2>&1

 echo "Install semver helper."
 npm install -g semver >/dev/null 2>&1