Merge pull request #12380 from ethereum/native-solc-in-ext-tests

External tests with native compiler binary
2023-10-03 13:03:40 +00:00 · 2021-12-13 11:45:15 +01:00 · 2021-12-13 11:45:15 +01:00 · 6b771735a0
commit 6b771735a0
parent a8f82b4716 3649103e42
9 changed files with 174 additions and 125 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -396,6 +396,11 @@ defaults:
      requires:
        - b_ubu_release

+  - workflow_ubuntu2004_static: &workflow_ubuntu2004_static
+      <<: *workflow_trigger_on_tags
+      requires:
+        - b_ubu_static
+
  - workflow_archlinux: &workflow_archlinux
      <<: *workflow_trigger_on_tags
      requires:
@ -666,10 +671,11 @@ jobs:
      MAKEFLAGS: -j 10

  b_ubu_static:
-    # Runs 2x faster on large and 3x on xlarge but no other jobs depend on it we can live with medium.
-    <<: *base_ubuntu2004
+    # On large runs 2x faster than on medium. 3x on xlarge.
+    <<: *base_ubuntu2004_xlarge
    environment:
-      MAKEFLAGS: -j 5
+      TERM: xterm
+      MAKEFLAGS: -j 10
      CMAKE_OPTIONS: -DCMAKE_BUILD_TYPE=Release -DUSE_Z3_DLOPEN=ON -DUSE_CVC4=OFF -DSOLC_STATIC_STDLIBS=ON
    steps:
      - checkout
@ -678,6 +684,7 @@ jobs:
          name: strip binary
          command: strip build/solc/solc
      - store_artifacts: *artifacts_solc
+      - persist_to_workspace: *artifacts_executables
      - gitter_notify_failure_unless_pr

  b_ubu_codecov:
@ -1015,6 +1022,11 @@ jobs:
    parameters:
      project:
        type: string
+      binary_type:
+        type: enum
+        enum:
+          - solcjs
+          - native
      compile_only:
        type: integer
        default: 0
@ -1037,14 +1049,26 @@ jobs:
      - attach_workspace:
          at: /tmp/workspace
      - run:
-          name: Install dependencies
+          name: Install lsof
          command: |
            # lsof is used by Colony in its stop-blockchain-client.sh script
-            sudo apt-get -qy install lsof
-      - run:
-          name: External <<parameters.project>> tests
-          command: |
-            test/externalTests/<<parameters.project>>.sh /tmp/workspace/soljson.js
+            sudo apt-get --quiet --assume-yes --no-install-recommends install lsof
+      - when:
+          condition:
+            equal: [<< parameters.binary_type >>, "solcjs"]
+          steps:
+            - run:
+                name: External <<parameters.project>> tests (solcjs)
+                command: |
+                  test/externalTests/<<parameters.project>>.sh solcjs /tmp/workspace/soljson.js
+      - when:
+          condition:
+            equal: [<< parameters.binary_type >>, "native"]
+          steps:
+            - run:
+                name: External <<parameters.project>> tests (native)
+                command: |
+                  test/externalTests/<<parameters.project>>.sh native /tmp/workspace/solc/solc
      - gitter_notify_failure_unless_pr

  b_win: &b_win
@ -1282,12 +1306,14 @@ workflows:
          <<: *workflow_emscripten
          name: t_ems_compile_ext_colony
          project: colony
+          binary_type: solcjs
          compile_only: 1
          nodejs_version: '14'
      - t_ems_ext:
-          <<: *workflow_emscripten
-          name: t_ems_compile_ext_gnosis
+          <<: *workflow_ubuntu2004_static
+          name: t_native_compile_ext_gnosis
          project: gnosis
+          binary_type: native
          compile_only: 1
          nodejs_version: '14'

@ -1295,27 +1321,31 @@ workflows:
      # and there are also other less frequent problems. See https://github.com/gnosis/safe-contracts/issues/216.
      #- t_ems_ext:
      #    <<: *workflow_emscripten
-      #    name: t_ems_test_ext_gnosis
+      #    name: t_native_test_ext_gnosis
      #    project: gnosis
+      #    binary_type: native
      #    # NOTE: Tests do not start on node.js 14 ("ganache-cli exited early with code 1").
      #    nodejs_version: '12'
      - t_ems_ext:
-          <<: *workflow_emscripten
-          name: t_ems_test_ext_gnosis_v2
+          <<: *workflow_ubuntu2004_static
+          name: t_native_test_ext_gnosis_v2
          project: gnosis-v2
+          binary_type: native
          # NOTE: Tests do not start on node.js 14 ("ganache-cli exited early with code 1").
          nodejs_version: '12'
      - t_ems_ext:
-          <<: *workflow_emscripten
-          name: t_ems_test_ext_zeppelin
+          <<: *workflow_ubuntu2004_static
+          name: t_native_test_ext_zeppelin
          project: zeppelin
+          binary_type: native
          # NOTE: Tests crash on nodejs 17: "Error: error:0308010C:digital envelope routines::unsupported"
          nodejs_version: '16'
          resource_class: large
      - t_ems_ext:
-          <<: *workflow_emscripten
-          name: t_ems_test_ext_ens
+          <<: *workflow_ubuntu2004_static
+          name: t_native_test_ext_ens
          project: ens
+          binary_type: native
          # NOTE: One of the dependencies (fsevents) fails to build its native extension on node.js 12+.
          nodejs_version: '10'

@ -1382,4 +1412,5 @@ workflows:
          <<: *workflow_emscripten
          name: t_ems_test_ext_colony
          project: colony
+          binary_type: solcjs
          resource_class: medium
--- a/test/externalTests.sh
+++ b/test/externalTests.sh
@ -28,25 +28,17 @@

 set -e

-if [ ! -f "$1" ]
-then
-  echo "Usage: $0 <path to soljson.js>"
-  exit 1
-fi
-
-SOLJSON="$1"
 REPO_ROOT="$(dirname "$0")"

 source scripts/common.sh
 source test/externalTests/common.sh

+verify_input "$@"
+
 printTask "Running external tests..."

-"$REPO_ROOT/externalTests/zeppelin.sh" "$SOLJSON"
-"$REPO_ROOT/externalTests/gnosis.sh" "$SOLJSON"
-"$REPO_ROOT/externalTests/gnosis-v2.sh" "$SOLJSON"
-"$REPO_ROOT/externalTests/colony.sh" "$SOLJSON"
-"$REPO_ROOT/externalTests/ens.sh" "$SOLJSON"
-
-# Disabled temporarily as it needs to be updated to latest Truffle first.
-#test_truffle Gnosis https://github.com/axic/pm-contracts.git solidity-050
+"$REPO_ROOT/externalTests/zeppelin.sh" "$@"
+"$REPO_ROOT/externalTests/gnosis.sh" "$@"
+"$REPO_ROOT/externalTests/gnosis-v2.sh" "$@"
+"$REPO_ROOT/externalTests/colony.sh" "$@"
+"$REPO_ROOT/externalTests/ens.sh" "$@"
--- a/test/externalTests/colony.sh
+++ b/test/externalTests/colony.sh
@ -24,8 +24,9 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh

-verify_input "$1"
-SOLJSON="$1"
+verify_input "$@"
+BINARY_TYPE="$1"
+BINARY_PATH="$2"

 function compile_fn { yarn run provision:token:contracts; }
 function test_fn { yarn run test:contracts; }
@ -42,11 +43,12 @@ function colony_test
    selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")")
    print_optimizer_levels_or_exit "$selected_optimizer_levels"

-    setup_solcjs "$DIR" "$SOLJSON"
+    setup_solc "$DIR" "$BINARY_TYPE" "$BINARY_PATH"
    download_project "$repo" "$branch" "$DIR"
+    [[ $BINARY_TYPE == native ]] && replace_global_solc "$BINARY_PATH"

    neutralize_package_json_hooks
-    force_truffle_compiler_settings "$config_file" "${DIR}/solc" "$min_optimizer_level"
+    force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$min_optimizer_level"
    yarn
    git submodule update --init

@ -56,10 +58,10 @@ function colony_test
    cd ..

    replace_version_pragmas
-    force_solc_modules "${DIR}/solc"
+    [[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc"

    for level in $selected_optimizer_levels; do
-        truffle_run_test "$config_file" "${DIR}/solc" "$level" compile_fn test_fn
+        truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$level" compile_fn test_fn
    done
 }

--- a/test/externalTests/common.sh
+++ b/test/externalTests/common.sh
@ -35,58 +35,53 @@ function print_optimizer_levels_or_exit

 function verify_input
 {
-    if [ ! -f "$1" ]; then
-        printError "Usage: $0 <path to soljson.js>"
-        exit 1
+    local binary_type="$1"
+    local binary_path="$2"
+
+    (( $# == 2 )) || fail "Usage: $0 native|solcjs <path to solc or soljson.js>"
+    [[ $binary_type == native || $binary_type == solcjs ]] || fail "Invalid binary type: '${binary_type}'. Must be either 'native' or 'solcjs'."
+    [[ -f "$binary_path" ]] || fail "The compiler binary does not exist at '${binary_path}'"
+}
+
+function setup_solc
+{
+    local test_dir="$1"
+    local binary_type="$2"
+    local binary_path="$3"
+    local solcjs_branch="${4:-master}"
+    local install_dir="${5:-solc/}"
+
+    [[ $binary_type == native || $binary_type == solcjs ]] || assertFail
+
+    cd "$test_dir"
+
+    if [[ $binary_type == solcjs ]]
+    then
+        printLog "Setting up solc-js..."
+        git clone --depth 1 -b "$solcjs_branch" https://github.com/ethereum/solc-js.git "$install_dir"
+
+        pushd "$install_dir"
+        npm install
+        cp "$binary_path" soljson.js
+        SOLCVERSION=$(./solcjs --version)
+        popd
+    else
+        printLog "Setting up solc..."
+        SOLCVERSION=$("$binary_path" --version | tail -n 1 | sed -n -E 's/^Version: (.*)$/\1/p')
    fi
-}

-function verify_version_input
-{
-    if [ -z "$1" ] || [ ! -f "$1" ] || [ -z "$2" ]; then
-        printError "Usage: $0 <path to soljson.js> <version>"
-        exit 1
-    fi
-}
-
-function setup
-{
-    local soljson="$1"
-    local branch="$2"
-
-    setup_solcjs "$DIR" "$soljson" "$branch" "solc"
-    cd solc
-}
-
-function setup_solcjs
-{
-    local dir="$1"
-    local soljson="$2"
-    local branch="${3:-master}"
-    local path="${4:-solc/}"
-
-    cd "$dir"
-    printLog "Setting up solc-js..."
-    git clone --depth 1 -b "$branch" https://github.com/ethereum/solc-js.git "$path"
-
-    cd "$path"
-
-    npm install
-    cp "$soljson" soljson.js
-    SOLCVERSION=$(./solcjs --version)
    SOLCVERSION_SHORT=$(echo "$SOLCVERSION" | sed -En 's/^([0-9.]+).*\+commit\.[0-9a-f]+.*$/\1/p')
-    printLog "Using solcjs version $SOLCVERSION"
-    cd ..
+    printLog "Using compiler version $SOLCVERSION"
 }

 function download_project
 {
    local repo="$1"
-    local branch="$2"
-    local dir="$3"
+    local solcjs_branch="$2"
+    local test_dir="$3"

-    printLog "Cloning $branch of $repo..."
-    git clone --depth 1 "$repo" -b "$branch" "$dir/ext"
+    printLog "Cloning $solcjs_branch of $repo..."
+    git clone --depth 1 "$repo" -b "$solcjs_branch" "$test_dir/ext"
    cd ext
    echo "Current commit hash: $(git rev-parse HEAD)"
 }
@ -144,13 +139,19 @@ function force_solc_modules
 function force_truffle_compiler_settings
 {
    local config_file="$1"
-    local solc_path="$2"
-    local level="$3"
-    local evm_version="${4:-"$CURRENT_EVM_VERSION"}"
+    local binary_type="$2"
+    local solc_path="$3"
+    local level="$4"
+    local evm_version="${5:-"$CURRENT_EVM_VERSION"}"
+
+    [[ $binary_type == native || $binary_type == solcjs ]] || assertFail
+
+    [[ $binary_type == native ]] && local solc_path="native"

    printLog "Forcing Truffle compiler settings..."
    echo "-------------------------------------"
    echo "Config file: $config_file"
+    echo "Binary type: $binary_type"
    echo "Compiler path: $solc_path"
    echo "Optimization level: $level"
    echo "Optimizer settings: $(optimizer_settings_for_level "$level")"
@ -165,13 +166,15 @@ function force_truffle_compiler_settings
 function force_hardhat_compiler_binary
 {
    local config_file="$1"
-    local solc_path="$2"
+    local binary_type="$2"
+    local solc_path="$3"

    printLog "Configuring Hardhat..."
    echo "-------------------------------------"
    echo "Config file: ${config_file}"
+    echo "Binary type: ${binary_type}"
    echo "Compiler path: ${solc_path}"
-    hardhat_solc_build_subtask "$SOLCVERSION_SHORT" "$SOLCVERSION" "$solc_path" >> "$config_file"
+    hardhat_solc_build_subtask "$SOLCVERSION_SHORT" "$SOLCVERSION" "$binary_type" "$solc_path" >> "$config_file"
 }

 function force_hardhat_compiler_settings
@ -248,12 +251,21 @@ function optimizer_settings_for_level
        2) echo "{enabled: true}" ;;
        3) echo "{enabled: true, details: {yul: true}}" ;;
        *)
-            printError "Optimizer level not found. Please define OPTIMIZER_LEVEL=[1, 2, 3]"
-            exit 1
+            fail "Optimizer level not found. Please define OPTIMIZER_LEVEL=[1, 2, 3]"
            ;;
    esac
 }

+function replace_global_solc
+{
+    local solc_path="$1"
+
+    [[ ! -e solc ]] || fail "A file named 'solc' already exists in '${PWD}'."
+
+    ln -s "$solc_path" solc
+    export PATH="$PWD:$PATH"
+}
+
 function truffle_compiler_settings
 {
    local solc_path="$1"
@ -274,7 +286,13 @@ function truffle_compiler_settings
 function hardhat_solc_build_subtask {
    local solc_version="$1"
    local full_solc_version="$2"
-    local solc_path="$3"
+    local binary_type="$3"
+    local solc_path="$4"
+
+    [[ $binary_type == native || $binary_type == solcjs ]] || assertFail
+
+    [[ $binary_type == native ]] && local is_solcjs=false
+    [[ $binary_type == solcjs ]] && local is_solcjs=true

    echo "const {TASK_COMPILE_SOLIDITY_GET_SOLC_BUILD} = require('hardhat/builtin-tasks/task-names');"
    echo "const assert = require('assert');"
@ -283,7 +301,7 @@ function hardhat_solc_build_subtask {
    echo "    assert(args.solcVersion == '${solc_version}', 'Unexpected solc version: ' + args.solcVersion)"
    echo "    return {"
    echo "        compilerPath: '$(realpath "$solc_path")',"
-    echo "        isSolcJs: true,"
+    echo "        isSolcJs: ${is_solcjs},"
    echo "        version: args.solcVersion,"
    echo "        longVersion: '${full_solc_version}'"
    echo "    }"
@ -325,13 +343,14 @@ function compile_and_run_test
 function truffle_run_test
 {
    local config_file="$1"
-    local solc_path="$2"
-    local optimizer_level="$3"
-    local compile_fn="$4"
-    local test_fn="$5"
+    local binary_type="$2"
+    local solc_path="$3"
+    local optimizer_level="$4"
+    local compile_fn="$5"
+    local test_fn="$6"

    truffle_clean
-    force_truffle_compiler_settings "$config_file" "$solc_path" "$optimizer_level"
+    force_truffle_compiler_settings "$config_file" "$binary_type" "$solc_path" "$optimizer_level"
    compile_and_run_test compile_fn test_fn truffle_verify_compiler_version
 }

@ -356,10 +375,7 @@ function external_test
    echo "==========================="
    DIR=$(mktemp -d -t "ext-test-${name}-XXXXXX")
    (
-        if [ -z "$main_fn" ]; then
-            printError "Test main function not defined."
-            exit 1
-        fi
+        [[ "$main_fn" != "" ]] || fail "Test main function not defined."
        $main_fn
    )
    rm -rf "$DIR"
--- a/test/externalTests/ens.sh
+++ b/test/externalTests/ens.sh
@ -24,8 +24,9 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh

-verify_input "$1"
-export SOLJSON="$1"
+verify_input "$@"
+BINARY_TYPE="$1"
+BINARY_PATH="$2"

 function compile_fn { npx truffle compile; }
 function test_fn { npm run test; }
@ -42,22 +43,23 @@ function ens_test
    selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")")
    print_optimizer_levels_or_exit "$selected_optimizer_levels"

-    setup_solcjs "$DIR" "$SOLJSON"
+    setup_solc "$DIR" "$BINARY_TYPE" "$BINARY_PATH"
    download_project "$repo" "$branch" "$DIR"
+    [[ $BINARY_TYPE == native ]] && replace_global_solc "$BINARY_PATH"

    # Use latest Truffle. Older versions crash on the output from 0.8.0.
    force_truffle_version ^5.1.55

    neutralize_package_lock
    neutralize_package_json_hooks
-    force_truffle_compiler_settings "$config_file" "${DIR}/solc" "$min_optimizer_level"
+    force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$min_optimizer_level"
    npm install

    replace_version_pragmas
-    force_solc_modules "${DIR}/solc"
+    [[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc"

    for level in $selected_optimizer_levels; do
-        truffle_run_test "$config_file" "${DIR}/solc" "$level" compile_fn test_fn
+        truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$level" compile_fn test_fn
    done
 }

--- a/test/externalTests/gnosis-v2.sh
+++ b/test/externalTests/gnosis-v2.sh
@ -24,10 +24,10 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh

-verify_input "$1"
-SOLJSON="$1"
+verify_input "$@"
+BINARY_TYPE="$1"
+BINARY_PATH="$2"

-function install_fn { npm install --package-lock; }
 function compile_fn { npx truffle compile; }
 function test_fn { npm test; }

@ -43,22 +43,23 @@ function gnosis_safe_test
    selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")")
    print_optimizer_levels_or_exit "$selected_optimizer_levels"

-    setup_solcjs "$DIR" "$SOLJSON"
+    setup_solc "$DIR" "$BINARY_TYPE" "$BINARY_PATH"
    download_project "$repo" "$branch" "$DIR"
+    [[ $BINARY_TYPE == native ]] && replace_global_solc "$BINARY_PATH"

    sed -i 's|github:gnosis/mock-contract#sol_0_5_0|github:solidity-external-tests/mock-contract#master_080|g' package.json
    sed -i -E 's|"@gnosis.pm/util-contracts": "[^"]+"|"@gnosis.pm/util-contracts": "github:solidity-external-tests/util-contracts#solc-7_080"|g' package.json

    neutralize_package_lock
    neutralize_package_json_hooks
-    force_truffle_compiler_settings "$config_file" "${DIR}/solc" "$min_optimizer_level"
+    force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$min_optimizer_level"
    npm install --package-lock

    replace_version_pragmas
-    force_solc_modules "${DIR}/solc"
+    [[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc"

    for level in $selected_optimizer_levels; do
-        truffle_run_test "$config_file" "${DIR}/solc" "$level" compile_fn test_fn
+        truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$level" compile_fn test_fn
    done
 }

--- a/test/externalTests/gnosis.sh
+++ b/test/externalTests/gnosis.sh
@ -24,8 +24,9 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh

-verify_input "$1"
-SOLJSON="$1"
+verify_input "$@"
+BINARY_TYPE="$1"
+BINARY_PATH="$2"

 function compile_fn { npx truffle compile; }
 function test_fn { npm test; }
@ -42,21 +43,22 @@ function gnosis_safe_test
    selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")")
    print_optimizer_levels_or_exit "$selected_optimizer_levels"

-    setup_solcjs "$DIR" "$SOLJSON"
+    setup_solc "$DIR" "$BINARY_TYPE" "$BINARY_PATH"
    download_project "$repo" "$branch" "$DIR"
+    [[ $BINARY_TYPE == native ]] && replace_global_solc "$BINARY_PATH"

    sed -i 's|github:gnosis/mock-contract#sol_0_5_0|github:solidity-external-tests/mock-contract#master_080|g' package.json

    neutralize_package_lock
    neutralize_package_json_hooks
-    force_truffle_compiler_settings "$config_file" "${DIR}/solc" "$min_optimizer_level"
+    force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$min_optimizer_level"
    npm install --package-lock

    replace_version_pragmas
-    force_solc_modules "${DIR}/solc"
+    [[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc"

    for level in $selected_optimizer_levels; do
-        truffle_run_test "$config_file" "${DIR}/solc" "$level" compile_fn test_fn
+        truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$level" compile_fn test_fn
    done
 }

--- a/test/externalTests/solc-js/solc-js.sh
+++ b/test/externalTests/solc-js/solc-js.sh
@ -24,10 +24,11 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh

-verify_version_input "$1" "$2"
 SOLJSON="$1"
 VERSION="$2"

+[[ $SOLJSON != "" && -f "$SOLJSON" && $VERSION != "" ]] || fail "Usage: $0 <path to soljson.js> <version>"
+
 function compile_fn { echo "Nothing to compile."; }
 function test_fn { npm test; }

@ -37,7 +38,8 @@ function solcjs_test
    SOLCJS_INPUT_DIR="$TEST_DIR"/test/externalTests/solc-js

    # set up solc-js on the branch specified
-    setup "$SOLJSON" master
+    setup_solc "$DIR" solcjs "$SOLJSON" master solc/
+    cd solc/

    printLog "Updating index.js file..."
    echo "require('./determinism.js');" >> test/index.js
--- a/test/externalTests/zeppelin.sh
+++ b/test/externalTests/zeppelin.sh
@ -24,8 +24,9 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh

-verify_input "$1"
-SOLJSON="$1"
+verify_input "$@"
+BINARY_TYPE="$1"
+BINARY_PATH="$2"

 function compile_fn { npm run compile; }
 function test_fn { npm test; }
@ -42,11 +43,11 @@ function zeppelin_test
    selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")")
    print_optimizer_levels_or_exit "$selected_optimizer_levels"

-    setup_solcjs "$DIR" "$SOLJSON"
+    setup_solc "$DIR" "$BINARY_TYPE" "$BINARY_PATH"
    download_project "$repo" "$branch" "$DIR"

    neutralize_package_json_hooks
-    force_hardhat_compiler_binary "$config_file" "$SOLJSON"
+    force_hardhat_compiler_binary "$config_file" "$BINARY_TYPE" "$BINARY_PATH"
    force_hardhat_compiler_settings "$config_file" "$min_optimizer_level"
    npm install