Command-line test for equivalence of optimizer options on the CLI and in Standard JSON

scripts/common_cmdline.sh

@@ -104,6 +104,45 @@ function compileFull
     fi
 }
 
+function singleContractOutputViaStandardJSON
+{
+    (( $# == 4 )) || assertFail
+    local language="$1"
+    local selected_output="$2"
+    local extra_settings="$3"
+    local input_file="$4"
+    [[ $selected_output != "*" ]] || assertFail
+
+    json_output=$(
+        "$SOLC" --standard-json --allow-paths "$(basename "$input_file")" - <<EOF
+        {
+            "language": "${language}",
+            "sources": {"${input_file}": {"urls": ["${input_file}"]}},
+            "settings": {
+                "outputSelection": {"*": { "*": ["${selected_output}"]}},
+                ${extra_settings}
+            }
+        }
+EOF
+    )
+
+    local error_list output has_contract_level_outputs
+
+    error_list=$(echo "$json_output" | jq '.errors[] | select(.severity=="error")')
+    [[ $error_list == '' ]] || \
+        fail "Failed to compile ${input_file} via Standard JSON. Errors: ${error_list}"
+
+    has_contract_level_outputs=$(echo "$json_output" | jq 'has("contracts")')
+    [[ $has_contract_level_outputs == true ]] || \
+        fail "Standard JSON output ${selected_output} was ignored by the compiler."
+
+    output=$(echo "$json_output" | jq --raw-output ".contracts[\"${input_file}\"][].${selected_output}")
+    [[ $output != null ]] || \
+        fail "Compiler failed to produce the ${selected_output} output when compiling ${input_file} via Standard JSON."
+
+    echo "$output"
+}
+
 function stripCLIDecorations
 {
     sed -e '/^=======.*=======$/d' \

test/cmdlineTests/~cli_and_standard_json_equivalence/test.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# shellcheck source=scripts/common.sh
+source "${REPO_ROOT}/scripts/common.sh"
+# shellcheck source=scripts/common_cmdline.sh
+source "${REPO_ROOT}/scripts/common_cmdline.sh"
+
+function test_cli_and_standard_json_equivalence
+{
+    (( $# == 5 )) || assertFail
+    local cli_options="$1"
+    local selected_cli_output="$2"
+    local standard_json_settings="$3"
+    local selected_standard_json_output="$4"
+    local input_file_relative_path="$5"
+
+    # CLI normalizes paths, Standard JSON uses them as is. Using paths that would change under this
+    # normalization will make the comparison fail. To avoid this use already normalized paths.
+    # The sanity check below should reject most of these by disallowing absolute paths, relative
+    # paths with ./ or ../ segments and paths with redundant slashes, but keep in mind it's not foolproof.
+    [[ $input_file_relative_path =~ ^/|^\.$|\./|^\.\.$|\.\./|// ]] && assertfail
+
+    local cli_output standard_json_output
+    cli_output=$(
+        # shellcheck disable=SC2086 # Intentionally unquoted. May contain multiple options.
+        msg_on_error --no-stderr \
+            "$SOLC" $cli_options "$selected_cli_output" "$input_file_relative_path"
+    )
+    standard_json_output=$(
+        singleContractOutputViaStandardJSON \
+            Solidity \
+            "$selected_standard_json_output" \
+            "$standard_json_settings" \
+            "$input_file_relative_path"
+    )
+
+    diff_values \
+        "$(echo "$cli_output" | stripCLIDecorations | stripEmptyLines)" \
+        "$(echo "$standard_json_output" | stripEmptyLines)" \
+        --ignore-space-change \
+        --ignore-blank-lines
+}
+
+cd "$REPO_ROOT"
+
+printTask "    - --optimize vs optimizer.enabled: true (--asm output)"
+test_cli_and_standard_json_equivalence \
+    '--optimize' \
+    '--asm' \
+    '"optimizer": {"enabled": true}' \
+    'evm.assembly' \
+    "test/libsolidity/semanticTests/various/erc20.sol"
+
+printTask "    - --optimize-yul vs optimizer.details.yul: true (--asm output)"
+test_cli_and_standard_json_equivalence \
+    '--optimize-yul' \
+    '--asm' \
+    '"optimizer": {"enabled": false, "details": {"yul": true}}' \
+    'evm.assembly' \
+    "test/libsolidity/semanticTests/various/erc20.sol"