Merge pull request #12380 from ethereum/native-solc-in-ext-tests

External tests with native compiler binary

.circleci/config.yml

@@ -396,6 +396,11 @@ defaults:
       requires:
         - b_ubu_release
 
+  - workflow_ubuntu2004_static: &workflow_ubuntu2004_static
+      <<: *workflow_trigger_on_tags
+      requires:
+        - b_ubu_static
+
   - workflow_archlinux: &workflow_archlinux
       <<: *workflow_trigger_on_tags
       requires:
@@ -666,10 +671,11 @@ jobs:
       MAKEFLAGS: -j 10
 
   b_ubu_static:
-    # Runs 2x faster on large and 3x on xlarge but no other jobs depend on it we can live with medium.
+    # On large runs 2x faster than on medium. 3x on xlarge.
-    <<: *base_ubuntu2004
+    <<: *base_ubuntu2004_xlarge
     environment:
-      MAKEFLAGS: -j 5
+      TERM: xterm
+      MAKEFLAGS: -j 10
       CMAKE_OPTIONS: -DCMAKE_BUILD_TYPE=Release -DUSE_Z3_DLOPEN=ON -DUSE_CVC4=OFF -DSOLC_STATIC_STDLIBS=ON
     steps:
       - checkout
@@ -678,6 +684,7 @@ jobs:
           name: strip binary
           command: strip build/solc/solc
       - store_artifacts: *artifacts_solc
+      - persist_to_workspace: *artifacts_executables
       - gitter_notify_failure_unless_pr
 
   b_ubu_codecov:
@@ -1015,6 +1022,11 @@ jobs:
     parameters:
       project:
         type: string
+      binary_type:
+        type: enum
+        enum:
+          - solcjs
+          - native
       compile_only:
         type: integer
         default: 0
@@ -1037,14 +1049,26 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - run:
-          name: Install dependencies
+          name: Install lsof
           command: |
             # lsof is used by Colony in its stop-blockchain-client.sh script
-            sudo apt-get -qy install lsof
+            sudo apt-get --quiet --assume-yes --no-install-recommends install lsof
-      - run:
+      - when:
-          name: External <<parameters.project>> tests
+          condition:
-          command: |
+            equal: [<< parameters.binary_type >>, "solcjs"]
-            test/externalTests/<<parameters.project>>.sh /tmp/workspace/soljson.js
+          steps:
+            - run:
+                name: External <<parameters.project>> tests (solcjs)
+                command: |
+                  test/externalTests/<<parameters.project>>.sh solcjs /tmp/workspace/soljson.js
+      - when:
+          condition:
+            equal: [<< parameters.binary_type >>, "native"]
+          steps:
+            - run:
+                name: External <<parameters.project>> tests (native)
+                command: |
+                  test/externalTests/<<parameters.project>>.sh native /tmp/workspace/solc/solc
       - gitter_notify_failure_unless_pr
 
   b_win: &b_win
@@ -1282,12 +1306,14 @@ workflows:
           <<: *workflow_emscripten
           name: t_ems_compile_ext_colony
           project: colony
+          binary_type: solcjs
           compile_only: 1
           nodejs_version: '14'
       - t_ems_ext:
-          <<: *workflow_emscripten
+          <<: *workflow_ubuntu2004_static
-          name: t_ems_compile_ext_gnosis
+          name: t_native_compile_ext_gnosis
           project: gnosis
+          binary_type: native
           compile_only: 1
           nodejs_version: '14'
 
@@ -1295,27 +1321,31 @@ workflows:
       # and there are also other less frequent problems. See https://github.com/gnosis/safe-contracts/issues/216.
       #- t_ems_ext:
       #    <<: *workflow_emscripten
-      #    name: t_ems_test_ext_gnosis
+      #    name: t_native_test_ext_gnosis
       #    project: gnosis
+      #    binary_type: native
       #    # NOTE: Tests do not start on node.js 14 ("ganache-cli exited early with code 1").
       #    nodejs_version: '12'
       - t_ems_ext:
-          <<: *workflow_emscripten
+          <<: *workflow_ubuntu2004_static
-          name: t_ems_test_ext_gnosis_v2
+          name: t_native_test_ext_gnosis_v2
           project: gnosis-v2
+          binary_type: native
           # NOTE: Tests do not start on node.js 14 ("ganache-cli exited early with code 1").
           nodejs_version: '12'
       - t_ems_ext:
-          <<: *workflow_emscripten
+          <<: *workflow_ubuntu2004_static
-          name: t_ems_test_ext_zeppelin
+          name: t_native_test_ext_zeppelin
           project: zeppelin
+          binary_type: native
           # NOTE: Tests crash on nodejs 17: "Error: error:0308010C:digital envelope routines::unsupported"
           nodejs_version: '16'
           resource_class: large
       - t_ems_ext:
-          <<: *workflow_emscripten
+          <<: *workflow_ubuntu2004_static
-          name: t_ems_test_ext_ens
+          name: t_native_test_ext_ens
           project: ens
+          binary_type: native
           # NOTE: One of the dependencies (fsevents) fails to build its native extension on node.js 12+.
           nodejs_version: '10'
 
@@ -1382,4 +1412,5 @@ workflows:
           <<: *workflow_emscripten
           name: t_ems_test_ext_colony
           project: colony
+          binary_type: solcjs
           resource_class: medium

test/externalTests.sh

@@ -28,25 +28,17 @@
 
 set -e
 
-if [ ! -f "$1" ]
-then
-  echo "Usage: $0 <path to soljson.js>"
-  exit 1
-fi
-
-SOLJSON="$1"
 REPO_ROOT="$(dirname "$0")"
 
 source scripts/common.sh
 source test/externalTests/common.sh
 
+verify_input "$@"
+
 printTask "Running external tests..."
 
-"$REPO_ROOT/externalTests/zeppelin.sh" "$SOLJSON"
+"$REPO_ROOT/externalTests/zeppelin.sh" "$@"
-"$REPO_ROOT/externalTests/gnosis.sh" "$SOLJSON"
+"$REPO_ROOT/externalTests/gnosis.sh" "$@"
-"$REPO_ROOT/externalTests/gnosis-v2.sh" "$SOLJSON"
+"$REPO_ROOT/externalTests/gnosis-v2.sh" "$@"
-"$REPO_ROOT/externalTests/colony.sh" "$SOLJSON"
+"$REPO_ROOT/externalTests/colony.sh" "$@"
-"$REPO_ROOT/externalTests/ens.sh" "$SOLJSON"
+"$REPO_ROOT/externalTests/ens.sh" "$@"
-
-# Disabled temporarily as it needs to be updated to latest Truffle first.
-#test_truffle Gnosis https://github.com/axic/pm-contracts.git solidity-050

test/externalTests/colony.sh

@@ -24,8 +24,9 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh
 
-verify_input "$1"
+verify_input "$@"
-SOLJSON="$1"
+BINARY_TYPE="$1"
+BINARY_PATH="$2"
 
 function compile_fn { yarn run provision:token:contracts; }
 function test_fn { yarn run test:contracts; }
@@ -42,11 +43,12 @@ function colony_test
     selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")")
     print_optimizer_levels_or_exit "$selected_optimizer_levels"
 
-    setup_solcjs "$DIR" "$SOLJSON"
+    setup_solc "$DIR" "$BINARY_TYPE" "$BINARY_PATH"
     download_project "$repo" "$branch" "$DIR"
+    [[ $BINARY_TYPE == native ]] && replace_global_solc "$BINARY_PATH"
 
     neutralize_package_json_hooks
-    force_truffle_compiler_settings "$config_file" "${DIR}/solc" "$min_optimizer_level"
+    force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$min_optimizer_level"
     yarn
     git submodule update --init
 
@@ -56,10 +58,10 @@ function colony_test
     cd ..
 
     replace_version_pragmas
-    force_solc_modules "${DIR}/solc"
+    [[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc"
 
     for level in $selected_optimizer_levels; do
-        truffle_run_test "$config_file" "${DIR}/solc" "$level" compile_fn test_fn
+        truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$level" compile_fn test_fn
     done
 }
 

test/externalTests/common.sh

@@ -35,58 +35,53 @@ function print_optimizer_levels_or_exit
 
 function verify_input
 {
-    if [ ! -f "$1" ]; then
+    local binary_type="$1"
-        printError "Usage: $0 <path to soljson.js>"
+    local binary_path="$2"
-        exit 1
+
+    (( $# == 2 )) || fail "Usage: $0 native|solcjs <path to solc or soljson.js>"
+    [[ $binary_type == native || $binary_type == solcjs ]] || fail "Invalid binary type: '${binary_type}'. Must be either 'native' or 'solcjs'."
+    [[ -f "$binary_path" ]] || fail "The compiler binary does not exist at '${binary_path}'"
+}
+
+function setup_solc
+{
+    local test_dir="$1"
+    local binary_type="$2"
+    local binary_path="$3"
+    local solcjs_branch="${4:-master}"
+    local install_dir="${5:-solc/}"
+
+    [[ $binary_type == native || $binary_type == solcjs ]] || assertFail
+
+    cd "$test_dir"
+
+    if [[ $binary_type == solcjs ]]
+    then
+        printLog "Setting up solc-js..."
+        git clone --depth 1 -b "$solcjs_branch" https://github.com/ethereum/solc-js.git "$install_dir"
+
+        pushd "$install_dir"
+        npm install
+        cp "$binary_path" soljson.js
+        SOLCVERSION=$(./solcjs --version)
+        popd
+    else
+        printLog "Setting up solc..."
+        SOLCVERSION=$("$binary_path" --version | tail -n 1 | sed -n -E 's/^Version: (.*)$/\1/p')
     fi
-}
 
-function verify_version_input
-{
-    if [ -z "$1" ] || [ ! -f "$1" ] || [ -z "$2" ]; then
-        printError "Usage: $0 <path to soljson.js> <version>"
-        exit 1
-    fi
-}
-
-function setup
-{
-    local soljson="$1"
-    local branch="$2"
-
-    setup_solcjs "$DIR" "$soljson" "$branch" "solc"
-    cd solc
-}
-
-function setup_solcjs
-{
-    local dir="$1"
-    local soljson="$2"
-    local branch="${3:-master}"
-    local path="${4:-solc/}"
-
-    cd "$dir"
-    printLog "Setting up solc-js..."
-    git clone --depth 1 -b "$branch" https://github.com/ethereum/solc-js.git "$path"
-
-    cd "$path"
-
-    npm install
-    cp "$soljson" soljson.js
-    SOLCVERSION=$(./solcjs --version)
     SOLCVERSION_SHORT=$(echo "$SOLCVERSION" | sed -En 's/^([0-9.]+).*\+commit\.[0-9a-f]+.*$/\1/p')
-    printLog "Using solcjs version $SOLCVERSION"
+    printLog "Using compiler version $SOLCVERSION"
-    cd ..
 }
 
 function download_project
 {
     local repo="$1"
-    local branch="$2"
+    local solcjs_branch="$2"
-    local dir="$3"
+    local test_dir="$3"
 
-    printLog "Cloning $branch of $repo..."
+    printLog "Cloning $solcjs_branch of $repo..."
-    git clone --depth 1 "$repo" -b "$branch" "$dir/ext"
+    git clone --depth 1 "$repo" -b "$solcjs_branch" "$test_dir/ext"
     cd ext
     echo "Current commit hash: $(git rev-parse HEAD)"
 }
@@ -144,13 +139,19 @@ function force_solc_modules
 function force_truffle_compiler_settings
 {
     local config_file="$1"
-    local solc_path="$2"
+    local binary_type="$2"
-    local level="$3"
+    local solc_path="$3"
-    local evm_version="${4:-"$CURRENT_EVM_VERSION"}"
+    local level="$4"
+    local evm_version="${5:-"$CURRENT_EVM_VERSION"}"
+
+    [[ $binary_type == native || $binary_type == solcjs ]] || assertFail
+
+    [[ $binary_type == native ]] && local solc_path="native"
 
     printLog "Forcing Truffle compiler settings..."
     echo "-------------------------------------"
     echo "Config file: $config_file"
+    echo "Binary type: $binary_type"
     echo "Compiler path: $solc_path"
     echo "Optimization level: $level"
     echo "Optimizer settings: $(optimizer_settings_for_level "$level")"
@@ -165,13 +166,15 @@ function force_truffle_compiler_settings
 function force_hardhat_compiler_binary
 {
     local config_file="$1"
-    local solc_path="$2"
+    local binary_type="$2"
+    local solc_path="$3"
 
     printLog "Configuring Hardhat..."
     echo "-------------------------------------"
     echo "Config file: ${config_file}"
+    echo "Binary type: ${binary_type}"
     echo "Compiler path: ${solc_path}"
-    hardhat_solc_build_subtask "$SOLCVERSION_SHORT" "$SOLCVERSION" "$solc_path" >> "$config_file"
+    hardhat_solc_build_subtask "$SOLCVERSION_SHORT" "$SOLCVERSION" "$binary_type" "$solc_path" >> "$config_file"
 }
 
 function force_hardhat_compiler_settings
@@ -248,12 +251,21 @@ function optimizer_settings_for_level
         2) echo "{enabled: true}" ;;
         3) echo "{enabled: true, details: {yul: true}}" ;;
         *)
-            printError "Optimizer level not found. Please define OPTIMIZER_LEVEL=[1, 2, 3]"
+            fail "Optimizer level not found. Please define OPTIMIZER_LEVEL=[1, 2, 3]"
-            exit 1
             ;;
     esac
 }
 
+function replace_global_solc
+{
+    local solc_path="$1"
+
+    [[ ! -e solc ]] || fail "A file named 'solc' already exists in '${PWD}'."
+
+    ln -s "$solc_path" solc
+    export PATH="$PWD:$PATH"
+}
+
 function truffle_compiler_settings
 {
     local solc_path="$1"
@@ -274,7 +286,13 @@ function truffle_compiler_settings
 function hardhat_solc_build_subtask {
     local solc_version="$1"
     local full_solc_version="$2"
-    local solc_path="$3"
+    local binary_type="$3"
+    local solc_path="$4"
+
+    [[ $binary_type == native || $binary_type == solcjs ]] || assertFail
+
+    [[ $binary_type == native ]] && local is_solcjs=false
+    [[ $binary_type == solcjs ]] && local is_solcjs=true
 
     echo "const {TASK_COMPILE_SOLIDITY_GET_SOLC_BUILD} = require('hardhat/builtin-tasks/task-names');"
     echo "const assert = require('assert');"
@@ -283,7 +301,7 @@ function hardhat_solc_build_subtask {
     echo "    assert(args.solcVersion == '${solc_version}', 'Unexpected solc version: ' + args.solcVersion)"
     echo "    return {"
     echo "        compilerPath: '$(realpath "$solc_path")',"
-    echo "        isSolcJs: true,"
+    echo "        isSolcJs: ${is_solcjs},"
     echo "        version: args.solcVersion,"
     echo "        longVersion: '${full_solc_version}'"
     echo "    }"
@@ -325,13 +343,14 @@ function compile_and_run_test
 function truffle_run_test
 {
     local config_file="$1"
-    local solc_path="$2"
+    local binary_type="$2"
-    local optimizer_level="$3"
+    local solc_path="$3"
-    local compile_fn="$4"
+    local optimizer_level="$4"
-    local test_fn="$5"
+    local compile_fn="$5"
+    local test_fn="$6"
 
     truffle_clean
-    force_truffle_compiler_settings "$config_file" "$solc_path" "$optimizer_level"
+    force_truffle_compiler_settings "$config_file" "$binary_type" "$solc_path" "$optimizer_level"
     compile_and_run_test compile_fn test_fn truffle_verify_compiler_version
 }
 
@@ -356,10 +375,7 @@ function external_test
     echo "==========================="
     DIR=$(mktemp -d -t "ext-test-${name}-XXXXXX")
     (
-        if [ -z "$main_fn" ]; then
+        [[ "$main_fn" != "" ]] || fail "Test main function not defined."
-            printError "Test main function not defined."
-            exit 1
-        fi
         $main_fn
     )
     rm -rf "$DIR"

test/externalTests/ens.sh

@@ -24,8 +24,9 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh
 
-verify_input "$1"
+verify_input "$@"
-export SOLJSON="$1"
+BINARY_TYPE="$1"
+BINARY_PATH="$2"
 
 function compile_fn { npx truffle compile; }
 function test_fn { npm run test; }
@@ -42,22 +43,23 @@ function ens_test
     selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")")
     print_optimizer_levels_or_exit "$selected_optimizer_levels"
 
-    setup_solcjs "$DIR" "$SOLJSON"
+    setup_solc "$DIR" "$BINARY_TYPE" "$BINARY_PATH"
     download_project "$repo" "$branch" "$DIR"
+    [[ $BINARY_TYPE == native ]] && replace_global_solc "$BINARY_PATH"
 
     # Use latest Truffle. Older versions crash on the output from 0.8.0.
     force_truffle_version ^5.1.55
 
     neutralize_package_lock
     neutralize_package_json_hooks
-    force_truffle_compiler_settings "$config_file" "${DIR}/solc" "$min_optimizer_level"
+    force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$min_optimizer_level"
     npm install
 
     replace_version_pragmas
-    force_solc_modules "${DIR}/solc"
+    [[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc"
 
     for level in $selected_optimizer_levels; do
-        truffle_run_test "$config_file" "${DIR}/solc" "$level" compile_fn test_fn
+        truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$level" compile_fn test_fn
     done
 }
 

test/externalTests/gnosis-v2.sh

@@ -24,10 +24,10 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh
 
-verify_input "$1"
+verify_input "$@"
-SOLJSON="$1"
+BINARY_TYPE="$1"
+BINARY_PATH="$2"
 
-function install_fn { npm install --package-lock; }
 function compile_fn { npx truffle compile; }
 function test_fn { npm test; }
 
@@ -43,22 +43,23 @@ function gnosis_safe_test
     selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")")
     print_optimizer_levels_or_exit "$selected_optimizer_levels"
 
-    setup_solcjs "$DIR" "$SOLJSON"
+    setup_solc "$DIR" "$BINARY_TYPE" "$BINARY_PATH"
     download_project "$repo" "$branch" "$DIR"
+    [[ $BINARY_TYPE == native ]] && replace_global_solc "$BINARY_PATH"
 
     sed -i 's|github:gnosis/mock-contract#sol_0_5_0|github:solidity-external-tests/mock-contract#master_080|g' package.json
     sed -i -E 's|"@gnosis.pm/util-contracts": "[^"]+"|"@gnosis.pm/util-contracts": "github:solidity-external-tests/util-contracts#solc-7_080"|g' package.json
 
     neutralize_package_lock
     neutralize_package_json_hooks
-    force_truffle_compiler_settings "$config_file" "${DIR}/solc" "$min_optimizer_level"
+    force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$min_optimizer_level"
     npm install --package-lock
 
     replace_version_pragmas
-    force_solc_modules "${DIR}/solc"
+    [[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc"
 
     for level in $selected_optimizer_levels; do
-        truffle_run_test "$config_file" "${DIR}/solc" "$level" compile_fn test_fn
+        truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$level" compile_fn test_fn
     done
 }
 

test/externalTests/gnosis.sh

@@ -24,8 +24,9 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh
 
-verify_input "$1"
+verify_input "$@"
-SOLJSON="$1"
+BINARY_TYPE="$1"
+BINARY_PATH="$2"
 
 function compile_fn { npx truffle compile; }
 function test_fn { npm test; }
@@ -42,21 +43,22 @@ function gnosis_safe_test
     selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")")
     print_optimizer_levels_or_exit "$selected_optimizer_levels"
 
-    setup_solcjs "$DIR" "$SOLJSON"
+    setup_solc "$DIR" "$BINARY_TYPE" "$BINARY_PATH"
     download_project "$repo" "$branch" "$DIR"
+    [[ $BINARY_TYPE == native ]] && replace_global_solc "$BINARY_PATH"
 
     sed -i 's|github:gnosis/mock-contract#sol_0_5_0|github:solidity-external-tests/mock-contract#master_080|g' package.json
 
     neutralize_package_lock
     neutralize_package_json_hooks
-    force_truffle_compiler_settings "$config_file" "${DIR}/solc" "$min_optimizer_level"
+    force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$min_optimizer_level"
     npm install --package-lock
 
     replace_version_pragmas
-    force_solc_modules "${DIR}/solc"
+    [[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc"
 
     for level in $selected_optimizer_levels; do
-        truffle_run_test "$config_file" "${DIR}/solc" "$level" compile_fn test_fn
+        truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$level" compile_fn test_fn
     done
 }
 

test/externalTests/solc-js/solc-js.sh

@@ -24,10 +24,11 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh
 
-verify_version_input "$1" "$2"
 SOLJSON="$1"
 VERSION="$2"
 
+[[ $SOLJSON != "" && -f "$SOLJSON" && $VERSION != "" ]] || fail "Usage: $0 <path to soljson.js> <version>"
+
 function compile_fn { echo "Nothing to compile."; }
 function test_fn { npm test; }
 
@@ -37,7 +38,8 @@ function solcjs_test
     SOLCJS_INPUT_DIR="$TEST_DIR"/test/externalTests/solc-js
 
     # set up solc-js on the branch specified
-    setup "$SOLJSON" master
+    setup_solc "$DIR" solcjs "$SOLJSON" master solc/
+    cd solc/
 
     printLog "Updating index.js file..."
     echo "require('./determinism.js');" >> test/index.js

test/externalTests/zeppelin.sh

@@ -24,8 +24,9 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh
 
-verify_input "$1"
+verify_input "$@"
-SOLJSON="$1"
+BINARY_TYPE="$1"
+BINARY_PATH="$2"
 
 function compile_fn { npm run compile; }
 function test_fn { npm test; }
@@ -42,11 +43,11 @@ function zeppelin_test
     selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")")
     print_optimizer_levels_or_exit "$selected_optimizer_levels"
 
-    setup_solcjs "$DIR" "$SOLJSON"
+    setup_solc "$DIR" "$BINARY_TYPE" "$BINARY_PATH"
     download_project "$repo" "$branch" "$DIR"
 
     neutralize_package_json_hooks
-    force_hardhat_compiler_binary "$config_file" "$SOLJSON"
+    force_hardhat_compiler_binary "$config_file" "$BINARY_TYPE" "$BINARY_PATH"
     force_hardhat_compiler_settings "$config_file" "$min_optimizer_level"
     npm install