Merge pull request #14288 from ethereum/cli-and-standard-json-equivalence-tests

CLI and Standard JSON equivalence tests

scripts/common.sh

@@ -121,7 +121,7 @@ function assertFail
 
 function msg_on_error
 {
-    local error_message
+    local error_message=""
     local no_stdout=false
     local no_stderr=false
 

scripts/common_cmdline.sh

@@ -103,3 +103,69 @@ function compileFull
         false
     fi
 }
+
+function singleContractOutputViaStandardJSON
+{
+    (( $# == 4 )) || assertFail
+    local language="$1"
+    local selected_output="$2"
+    local extra_settings="$3"
+    local input_file="$4"
+    [[ $selected_output != "*" ]] || assertFail
+
+    json_output=$(
+        "$SOLC" --standard-json --allow-paths "$(basename "$input_file")" - <<EOF
+        {
+            "language": "${language}",
+            "sources": {"${input_file}": {"urls": ["${input_file}"]}},
+            "settings": {
+                "outputSelection": {"*": { "*": ["${selected_output}"]}},
+                ${extra_settings}
+            }
+        }
+EOF
+    )
+
+    local error_list output has_contract_level_outputs
+
+    error_list=$(echo "$json_output" | jq '.errors[] | select(.severity=="error")')
+    [[ $error_list == '' ]] || \
+        fail "Failed to compile ${input_file} via Standard JSON. Errors: ${error_list}"
+
+    has_contract_level_outputs=$(echo "$json_output" | jq 'has("contracts")')
+    [[ $has_contract_level_outputs == true ]] || \
+        fail "Standard JSON output ${selected_output} was ignored by the compiler."
+
+    output=$(echo "$json_output" | jq --raw-output ".contracts[\"${input_file}\"][].${selected_output}")
+    [[ $output != null ]] || \
+        fail "Compiler failed to produce the ${selected_output} output when compiling ${input_file} via Standard JSON."
+
+    echo "$output"
+}
+
+function stripCLIDecorations
+{
+    sed -e '/^=======.*=======$/d' \
+        -e '/^Binary:$/d' \
+        -e '/^Binary of the runtime part:$/d' \
+        -e '/^Opcodes:$/d' \
+        -e '/^IR:$/d' \
+        -e '/^Optimized IR:$/d' \
+        -e '/^EVM assembly:$/d' \
+        -e '/^JSON AST (compact format):$/d' \
+        -e '/^Function signatures:$/d' \
+        -e '/^Contract Storage Layout:$/d' \
+        -e '/^Developer Documentation$/d' \
+        -e '/^User Documentation$/d' \
+        -e '/^Contract JSON ABI$/d' \
+        -e '/^Metadata:$/d' \
+        -e '/^EVM$/d' \
+        -e '/^Pretty printed source:$/d' \
+        -e '/^Text representation:$/d' \
+        -e '/^Binary representation:$/d'
+}
+
+function stripEmptyLines
+{
+    sed -e '/^\s*$/d'
+}

test/cmdlineTests/~cli_and_standard_json_equivalence/test.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# shellcheck source=scripts/common.sh
+source "${REPO_ROOT}/scripts/common.sh"
+# shellcheck source=scripts/common_cmdline.sh
+source "${REPO_ROOT}/scripts/common_cmdline.sh"
+
+function test_cli_and_standard_json_equivalence
+{
+    (( $# == 5 )) || assertFail
+    local cli_options="$1"
+    local selected_cli_output="$2"
+    local standard_json_settings="$3"
+    local selected_standard_json_output="$4"
+    local input_file_relative_path="$5"
+
+    # CLI normalizes paths, Standard JSON uses them as is. Using paths that would change under this
+    # normalization will make the comparison fail. To avoid this use already normalized paths.
+    # The sanity check below should reject most of these by disallowing absolute paths, relative
+    # paths with ./ or ../ segments and paths with redundant slashes, but keep in mind it's not foolproof.
+    [[ $input_file_relative_path =~ ^/|^\.$|\./|^\.\.$|\.\./|// ]] && assertfail
+
+    local cli_output standard_json_output
+    cli_output=$(
+        # shellcheck disable=SC2086 # Intentionally unquoted. May contain multiple options.
+        msg_on_error --no-stderr \
+            "$SOLC" $cli_options "$selected_cli_output" "$input_file_relative_path"
+    )
+    standard_json_output=$(
+        singleContractOutputViaStandardJSON \
+            Solidity \
+            "$selected_standard_json_output" \
+            "$standard_json_settings" \
+            "$input_file_relative_path"
+    )
+
+    diff_values \
+        "$(echo "$cli_output" | stripCLIDecorations | stripEmptyLines)" \
+        "$(echo "$standard_json_output" | stripEmptyLines)" \
+        --ignore-space-change \
+        --ignore-blank-lines
+}
+
+cd "$REPO_ROOT"
+
+printTask "    - --optimize vs optimizer.enabled: true (--asm output)"
+test_cli_and_standard_json_equivalence \
+    '--optimize' \
+    '--asm' \
+    '"optimizer": {"enabled": true}' \
+    'evm.assembly' \
+    "test/libsolidity/semanticTests/various/erc20.sol"
+
+printTask "    - --optimize-yul vs optimizer.details.yul: true (--asm output)"
+test_cli_and_standard_json_equivalence \
+    '--optimize-yul' \
+    '--asm' \
+    '"optimizer": {"enabled": false, "details": {"yul": true}}' \
+    'evm.assembly' \
+    "test/libsolidity/semanticTests/various/erc20.sol"

test/cmdlineTests/~via_ir_equivalence/test.sh

@@ -3,6 +3,8 @@ set -eo pipefail
 
 # shellcheck source=scripts/common.sh
 source "${REPO_ROOT}/scripts/common.sh"
+# shellcheck source=scripts/common_cmdline.sh
+source "${REPO_ROOT}/scripts/common_cmdline.sh"
 
 function test_via_ir_equivalence()
 {
@@ -21,24 +23,24 @@ function test_via_ir_equivalence()
     [[ $optimize_flag == "" ]] || optimizer_flags+=("$optimize_flag")
     [[ $optimize_flag == "" ]] || output_file_prefix+="_optimize"
 
-    msg_on_error --no-stderr "$SOLC" --ir-optimized --debug-info location "${optimizer_flags[@]}" "$solidity_file" |
-        sed '/^Optimized IR:$/d' |
-        split_on_empty_lines_into_numbered_files "$output_file_prefix" ".yul"
+    msg_on_error --no-stderr \
+        "$SOLC" --ir-optimized --debug-info location "${optimizer_flags[@]}" "$solidity_file" |
+            stripCLIDecorations |
+            split_on_empty_lines_into_numbered_files "$output_file_prefix" ".yul"
 
     local asm_output_two_stage asm_output_via_ir
 
     for yul_file in $(find . -name "${output_file_prefix}*.yul" | sort -V); do
         asm_output_two_stage+=$(
-            msg_on_error --no-stderr "$SOLC" --strict-assembly --asm "${optimizer_flags[@]}" "$yul_file" |
-                sed '/^Text representation:$/d' |
-                sed '/^=======/d'
+            msg_on_error --no-stderr \
+                "$SOLC" --strict-assembly --asm "${optimizer_flags[@]}" "$yul_file" | stripCLIDecorations
         )
     done
 
     asm_output_via_ir=$(
-        msg_on_error --no-stderr "$SOLC" --via-ir --asm --debug-info location "${optimizer_flags[@]}" "$solidity_file" |
-            sed '/^EVM assembly:$/d' |
-            sed '/^=======/d'
+        msg_on_error --no-stderr \
+            "$SOLC" --via-ir --asm --debug-info location "${optimizer_flags[@]}" "$solidity_file" |
+                stripCLIDecorations
     )
 
     diff_values "$asm_output_two_stage" "$asm_output_via_ir" --ignore-space-change --ignore-blank-lines
@@ -47,16 +49,14 @@ function test_via_ir_equivalence()
 
     for yul_file in $(find . -name "${output_file_prefix}*.yul" | sort -V); do
         bin_output_two_stage+=$(
-            msg_on_error --no-stderr "$SOLC" --strict-assembly --bin "${optimizer_flags[@]}" "$yul_file" |
-                sed '/^Binary representation:$/d' |
-                sed '/^=======/d'
+            msg_on_error --no-stderr \
+                "$SOLC" --strict-assembly --bin "${optimizer_flags[@]}" "$yul_file" | stripCLIDecorations
         )
     done
 
     bin_output_via_ir=$(
-        msg_on_error --no-stderr "$SOLC" --via-ir --bin "${optimizer_flags[@]}" "$solidity_file" |
-            sed '/^Binary:$/d' |
-            sed '/^=======/d'
+        msg_on_error --no-stderr \
+            "$SOLC" --via-ir --bin "${optimizer_flags[@]}" "$solidity_file" | stripCLIDecorations
     )
 
     diff_values "$bin_output_two_stage" "$bin_output_via_ir" --ignore-space-change --ignore-blank-lines