Merge pull request #6488 from sifmelcara/cmdlineTests-auto-update

Auto-correct command line tests

test/cmdlineTests.sh

@@ -93,6 +93,27 @@ function compileFull()
     fi
 }
 
+function ask_expectation_update()
+{
+    local newExpectation="${1}"
+    local expectationFile="${2}"
+    while true;
+    do
+        set +e
+        read -t10 -p "(u)pdate expectation/(q)uit? "
+        if [ $? -gt 128 ];
+        then
+            echo -e "\nUser input timed out."
+            exit 1
+        fi
+        set -e
+        case $REPLY in
+            u* ) echo "$newExpectation" > $expectationFile ; break;;
+            q* ) exit 1;;
+        esac
+    done
+}
+
 # General helper function for testing SOLC behaviour, based on file name, compile opts, exit code, stdout and stderr.
 # An failure is expected.
 function test_solc_behaviour()
@@ -103,17 +124,22 @@ function test_solc_behaviour()
     local stdout_expected="${4}"
     local exit_code_expected="${5}"
     local stderr_expected="${6}"
+    local stdout_expectation_file="${7}" # the file to write to when user chooses to update stdout expectation
+    local stderr_expectation_file="${8}" # the file to write to when user chooses to update stderr expectation
     local stdout_path=`mktemp`
     local stderr_path=`mktemp`
+
+    trap "rm -f $stdout_path $stderr_path" EXIT
+
     if [[ "$exit_code_expected" = "" ]]; then exit_code_expected="0"; fi
 
+    local solc_command="$SOLC ${filename} ${solc_args}"
+    if [[ -n "$solc_stdin"  ]]; then solc_command+=" <$solc_stdin"  ; fi
+    if [[ -n "$stdout_path" ]]; then solc_command+=" 1>$stdout_path"; fi
+    if [[ -n "$stderr_path" ]]; then solc_command+=" 2>$stderr_path"; fi
+
     set +e
-    if [[ "$solc_stdin" = "" ]]
+    eval "$solc_command"
-    then
-        "$SOLC" "${filename}" ${solc_args} 1>$stdout_path 2>$stderr_path
-    else
-        "$SOLC" "${filename}" ${solc_args} <$solc_stdin 1>$stdout_path 2>$stderr_path
-    fi
     exitCode=$?
     set -e
 
@@ -133,37 +159,44 @@ function test_solc_behaviour()
     if [[ $exitCode -ne "$exit_code_expected" ]]
     then
         printError "Incorrect exit code. Expected $exit_code_expected but got $exitCode."
-        rm -f $stdout_path $stderr_path
         exit 1
     fi
 
     if [[ "$(cat $stdout_path)" != "${stdout_expected}" ]]
     then
         printError "Incorrect output on stdout received. Expected:"
-        echo -e "${stdout_expected}"
+        echo "${stdout_expected}"
 
         printError "But got:"
         cat $stdout_path
-        printError "When running $SOLC ${filename} ${solc_args} <$solc_stdin"
 
-        rm -f $stdout_path $stderr_path
+        printError "When running $solc_command"
-        exit 1
+
+        if [ -n "$stdout_expectation_file" ]
+        then
+            ask_expectation_update "$(cat $stdout_path)" "$stdout_expectation_file"
+        else
+            exit 1
+        fi
     fi
 
     if [[ "$(cat $stderr_path)" != "${stderr_expected}" ]]
     then
         printError "Incorrect output on stderr received. Expected:"
-        echo -e "${stderr_expected}"
+        echo "${stderr_expected}"
 
         printError "But got:"
         cat $stderr_path
-        printError "When running $SOLC ${filename} ${solc_args} <$solc_stdin"
 
-        rm -f $stdout_path $stderr_path
+        printError "When running $solc_command"
-        exit 1
+
+        if [ -n "$stderr_expectation_file" ]
+        then
+            ask_expectation_update "$(cat $stderr_path)" "$stderr_expectation_file"
+        else
+            exit 1
+        fi
     fi
-
-    rm -f $stdout_path $stderr_path
 }
 
 
@@ -210,14 +243,14 @@ printTask "Testing unknown options..."
 
 
 printTask "Testing passing files that are not found..."
-test_solc_behaviour "file_not_found.sol" "" "" "" 1 "\"file_not_found.sol\" is not found."
+test_solc_behaviour "file_not_found.sol" "" "" "" 1 "\"file_not_found.sol\" is not found." "" ""
 
 printTask "Testing passing files that are not files..."
-test_solc_behaviour "." "" "" "" 1 "\".\" is not a valid file."
+test_solc_behaviour "." "" "" "" 1 "\".\" is not a valid file." "" ""
 
 printTask "Testing passing empty remappings..."
-test_solc_behaviour "${0}" "=/some/remapping/target" "" "" 1 "Invalid remapping: \"=/some/remapping/target\"."
+test_solc_behaviour "${0}" "=/some/remapping/target" "" "" 1 "Invalid remapping: \"=/some/remapping/target\"." "" ""
-test_solc_behaviour "${0}" "ctx:=/some/remapping/target" "" "" 1 "Invalid remapping: \"ctx:=/some/remapping/target\"."
+test_solc_behaviour "${0}" "ctx:=/some/remapping/target" "" "" 1 "Invalid remapping: \"ctx:=/some/remapping/target\"." "" ""
 
 printTask "Running general commandline tests..."
 (
@@ -228,18 +261,28 @@ printTask "Running general commandline tests..."
         then
             inputFile=""
             stdin="${tdir}/input.json"
-            stdout=$(cat ${tdir}/output.json 2>/dev/null || true)
+            stdout="$(cat ${tdir}/output.json 2>/dev/null || true)"
+            stdoutExpectationFile="$(pwd)/${tdir}/output.json"
             args="--standard-json "$(cat ${tdir}/args 2>/dev/null || true)
         else
             inputFile="${tdir}input.sol"
             stdin=""
-            stdout=$(cat ${tdir}/output 2>/dev/null || true)
+            stdout="$(cat ${tdir}/output 2>/dev/null || true)"
+            stdoutExpectationFile="$(pwd)/${tdir}/output"
             args=$(cat ${tdir}/args 2>/dev/null || true)
         fi
         exitCode=$(cat ${tdir}/exit 2>/dev/null || true)
-        err=$(cat ${tdir}/err 2>/dev/null || true)
+        err="$(cat ${tdir}/err 2>/dev/null || true)"
+        stderrExpectationFile="${tdir}/err"
         printTask " - ${tdir}"
-        test_solc_behaviour "$inputFile" "$args" "$stdin" "$stdout" "$exitCode" "$err"
+        test_solc_behaviour "$inputFile" \
+                            "$args" \
+                            "$stdin" \
+                            "$stdout" \
+                            "$exitCode" \
+                            "$err" \
+                            "$stdoutExpectationFile" \
+                            "$stderrExpectationFile"
     done
 )
 

test/cmdlineTests/gas_test_abiv2/err

@@ -1,3 +1,3 @@
 gas_test_abiv2/input.sol:2:1: Warning: Experimental features are turned on. Do not use experimental features on live deployments.
 pragma experimental ABIEncoderV2;
-^-------------------------------^
+^-------------------------------^

test/cmdlineTests/object_compiler/output

@@ -65,4 +65,3 @@ sub_0: assembly {
         /* "object_compiler/input.sol":407:422   */
       return
 }
-

test/cmdlineTests/standard_missing_key_useLiteralContent/output.json

@@ -1 +1 @@
-{"sources":{"A":{"id":0}}}
+{"sources":{"A":{"id":0}}}

test/cmdlineTests/standard_optimizer_invalid_details/output.json

@@ -1 +1 @@
-{"errors":[{"component":"general","formattedMessage":"Unknown key \"notThere\"","message":"Unknown key \"notThere\"","severity":"error","type":"JSONError"}]}
+{"errors":[{"component":"general","formattedMessage":"Unknown key \"notThere\"","message":"Unknown key \"notThere\"","severity":"error","type":"JSONError"}]}

test/cmdlineTests/standard_wrong_type_auxiliary_input/output.json

@@ -1,2 +1 @@
 {"errors":[{"component":"general","formattedMessage":"\"auxiliaryInput\" must be an object","message":"\"auxiliaryInput\" must be an object","severity":"error","type":"JSONError"}]}
-

test/cmdlineTests/standard_yul_multiple_files/output.json

@@ -1 +1 @@
-{"errors":[{"component":"general","formattedMessage":"Yul mode only supports exactly one input file.","message":"Yul mode only supports exactly one input file.","severity":"error","type":"JSONError"}]}
+{"errors":[{"component":"general","formattedMessage":"Yul mode only supports exactly one input file.","message":"Yul mode only supports exactly one input file.","severity":"error","type":"JSONError"}]}

test/cmdlineTests/standard_yul_multiple_files_selected/output.json

@@ -1 +1 @@
-{"errors":[{"component":"general","formattedMessage":"Yul mode only supports exactly one input file.","message":"Yul mode only supports exactly one input file.","severity":"error","type":"JSONError"}]}
+{"errors":[{"component":"general","formattedMessage":"Yul mode only supports exactly one input file.","message":"Yul mode only supports exactly one input file.","severity":"error","type":"JSONError"}]}

test/cmdlineTests/too_long_line_multiline/err

@@ -1,6 +1,6 @@
-too_long_line_multiline.sol:2:5: Error: No visibility specified. Did you intend to add "public"?
+too_long_line_multiline/input.sol:2:5: Error: No visibility specified. Did you intend to add "public"?
     function f() returns (byte _b, byte ... _b7, bytes22 _b22, bytes32 _b32)  {
     ^ (Relevant source part starts here and spans across multiple lines).
-too_long_line_multiline.sol:1:1: Warning: Source file does not specify required compiler version!
+too_long_line_multiline/input.sol:1:1: Warning: Source file does not specify required compiler version!
 contract C {
 ^ (Relevant source part starts here and spans across multiple lines).

test/cmdlineTests/yul_stack_opt/output

@@ -201,4 +201,3 @@ tag_2:
   swap16
   jump
 tag_4:
-

test/cmdlineTests/yul_stack_opt_disabled/err

@@ -3,4 +3,3 @@ Exception while assembling:
 Dynamic exception type:
 std::exception::what: Variable a1 is 17 slot(s) too deep inside the stack.
 [dev::tag_comment*] = Variable a1 is 17 slot(s) too deep inside the stack.
-

test/cmdlineTests/yul_stack_opt_disabled/output

@@ -28,4 +28,3 @@ object "object" {
         sstore(a1, a2)
     }
 }
-