OSX: Fix readlink issues and that ASTImportTest.sh silently ignores errors.

scripts/ASTImportTest.sh

@@ -1,10 +1,15 @@
 #!/usr/bin/env bash
 
+set -e
+
 # Bash script to test the ast-import option of the compiler by
 # first exporting a .sol file to JSON, then loading it into the compiler
 # and exporting it again. The second JSON should be identical to the first
-
-REPO_ROOT=$(readlink -f "$(dirname "$0")"/..)
+READLINK=readlink
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    READLINK=greadlink
+fi
+REPO_ROOT=$(${READLINK} -f "$(dirname "$0")"/..)
 SOLIDITY_BUILD_DIR=${SOLIDITY_BUILD_DIR:-${REPO_ROOT}/build}
 SOLC=${SOLIDITY_BUILD_DIR}/solc/solc
 SPLITSOURCES=${REPO_ROOT}/scripts/splitSources.py
@@ -83,8 +88,11 @@ do
     FILETMP=$(mktemp -d)
     cd $FILETMP
 
+    set +e
     OUTPUT=$($SPLITSOURCES $solfile)
-    if [ $? != 1 ]
+    SPLITSOURCES_RC=$?
+    set -e
+    if [ ${SPLITSOURCES_RC} == 0 ]
     then
         # echo $OUTPUT
         NSOURCES=$((NSOURCES - 1))
@@ -93,9 +101,26 @@ do
             testImportExportEquivalence $i $OUTPUT
             NSOURCES=$((NSOURCES + 1))
         done
-
-    else
+    elif [ ${SPLITSOURCES_RC} == 1 ]
+    then
         testImportExportEquivalence $solfile
+    elif [ ${SPLITSOURCES_RC} == 2 ]
+    then
+        # The script will exit with return code 2, if an UnicodeDecodeError occurred.
+        # This is the case if e.g. some tests are using invalid utf-8 sequences. We will ignore
+        # these errors, but print the actual output of the script.
+        echo -e "\n${OUTPUT}\n"
+        testImportExportEquivalence $solfile
+    else
+        # All other return codes will be treated as critical errors. The script will exit.
+        echo -e "\nGot unexpected return code ${SPLITSOURCES_RC} from ${SPLITSOURCES}. Aborting."
+        echo -e "\n${OUTPUT}\n"
+
+        cd $WORKINGDIR
+        # Delete temporary files
+        rm -rf $FILETMP
+
+        exit 1
     fi
 
     cd $WORKINGDIR

scripts/splitSources.py

@@ -11,10 +11,20 @@
 
 import sys
 import os
+import traceback
 
 hasMultipleSources = False
 createdSources = []
 
+
+def uncaught_exception_hook(exc_type, exc_value, exc_traceback):
+    # The script `scripts/ASTImportTest.sh` will interpret return code 3
+    # as a critical error (because of the uncaught exception) and will
+    # terminate further execution.
+    print("Unhandled exception: %s", "".join(traceback.format_exception(exc_type, exc_value, exc_traceback)))
+    sys.exit(3)
+
+
 def extractSourceName(line):
     if line.find("/") > -1:
         filePath = line[13: line.rindex("/")]
@@ -23,6 +33,7 @@ def extractSourceName(line):
         return filePath, srcName
     return False, line[line.find(":")+2 : line.find(" ====")]
 
+
 # expects the first line of lines to be "==== Source: sourceName ===="
 # writes the following source into a file named sourceName
 def writeSourceToFile(lines):
@@ -45,18 +56,29 @@ def writeSourceToFile(lines):
             writeSourceToFile(lines[1+idx:])
             break
 
+
 if __name__ == '__main__':
     filePath = sys.argv[1]
-    # decide if file has multiple sources
-    lines = open(filePath, mode='r', encoding='utf8').read().splitlines()
-    if lines[0][:12] == "==== Source:":
-        hasMultipleSources = True
-        writeSourceToFile(lines)
+    sys.excepthook = uncaught_exception_hook
 
-    if hasMultipleSources:
-        srcString = ""
-        for src in createdSources:
-            srcString += src + ' '
-        print(srcString)
-    else:
-        sys.exit(1)
+    try:
+        # decide if file has multiple sources
+        lines = open(filePath, mode='r', encoding='utf8').read().splitlines()
+        if lines[0][:12] == "==== Source:":
+            hasMultipleSources = True
+            writeSourceToFile(lines)
+
+        if hasMultipleSources:
+            srcString = ""
+            for src in createdSources:
+                srcString += src + ' '
+            print(srcString)
+            sys.exit(0)
+        else:
+            sys.exit(1)
+
+    except UnicodeDecodeError as ude:
+        print("UnicodeDecodeError in '" + filePath + "': " + str(ude))
+        print("This is expected for some tests containing invalid utf8 sequences. "
+              "Exception will be ignored.")
+        sys.exit(2)

scripts/test_antlr_grammar.sh

@@ -2,7 +2,11 @@
 
 set -e
 
-ROOT_DIR=$(readlink -f "$(dirname "$0")"/..)
+READLINK=readlink
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    READLINK=greadlink
+fi
+ROOT_DIR=$(${READLINK} -f "$(dirname "$0")"/..)
 WORKDIR="${ROOT_DIR}/build/antlr"
 ANTLR_JAR="${ROOT_DIR}/build/deps/antlr4.jar"
 ANTLR_JAR_URI="https://www.antlr.org/download/antlr-4.8-complete.jar"
@@ -54,7 +58,7 @@ failed_count=0
 test_file()
 {
   local SOL_FILE
-  SOL_FILE="$(readlink -m "${1}")"
+  SOL_FILE="$(${READLINK}  -m "${1}")"
   local cur=${2}
   local max=${3}
   local solOrYul=${4}