External tests: Add nightly job to run external tests

.circleci/config.yml

@@ -830,7 +830,7 @@ jobs:
             export COMPILE_ONLY=1
             test/externalTests/gnosis.sh /tmp/workspace/soljson.js || test/externalTests/gnosis.sh /tmp/workspace/soljson.js
 
-  t_ems_test_ext_gnosis:
+  t_ems_test_external:
     docker:
       - image: circleci/node:10
     environment:
@@ -840,9 +840,11 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - run:
-          name: External GnosisSafe tests
+          name: External tests
           command: |
-            test/externalTests/gnosis.sh /tmp/workspace/soljson.js || test/externalTests/gnosis.sh /tmp/workspace/soljson.js
+            test/externalTests.sh /tmp/workspace/soljson.js
+      - store_artifacts:
+          path: test/externalTests/externalTests.log
       - run: *gitter_notify_failure
       - run: *gitter_notify_success
 
@@ -861,7 +863,7 @@ jobs:
             export COMPILE_ONLY=1
             test/externalTests/zeppelin.sh /tmp/workspace/soljson.js || test/externalTests/zeppelin.sh /tmp/workspace/soljson.js
 
-  t_ems_test_ext_zeppelin:
+  t_ems_compile_ext_zeppelin:
     docker:
       - image: circleci/node:10
     environment:
@@ -871,11 +873,10 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - run:
-          name: External Zeppelin tests
+          name: External Zeppelin compilation
           command: |
+            export COMPILE_ONLY=1
             test/externalTests/zeppelin.sh /tmp/workspace/soljson.js || test/externalTests/zeppelin.sh /tmp/workspace/soljson.js
-      - run: *gitter_notify_failure
-      - run: *gitter_notify_success
 
   t_ems_compile_ext_colony:
     docker:
@@ -896,26 +897,6 @@ jobs:
             export COMPILE_ONLY=1
             test/externalTests/colony.sh /tmp/workspace/soljson.js || test/externalTests/colony.sh /tmp/workspace/soljson.js
 
-  t_ems_test_ext_colony:
-    docker:
-      - image: circleci/node:10
-    environment:
-      TERM: xterm
-    steps:
-      - checkout
-      - attach_workspace:
-          at: /tmp/workspace
-      - run:
-          name: Install test dependencies
-          command: |
-            sudo apt-get -qy install lsof
-      - run:
-          name: External ColonyNetworks tests
-          command: |
-            test/externalTests/colony.sh /tmp/workspace/soljson.js || test/externalTests/colony.sh /tmp/workspace/soljson.js
-      - run: *gitter_notify_failure
-      - run: *gitter_notify_success
-
   t_ems_compile_ext_ens:
     docker:
       - image: circleci/node:10
@@ -935,26 +916,6 @@ jobs:
             export COMPILE_ONLY=1
             test/externalTests/ens.sh /tmp/workspace/soljson.js || test/externalTests/ens.sh /tmp/workspace/soljson.js
 
-  t_ems_test_ext_ens:
-    docker:
-      - image: circleci/node:10
-    environment:
-      TERM: xterm
-    steps:
-      - checkout
-      - attach_workspace:
-          at: /tmp/workspace
-      - run:
-          name: Install test dependencies
-          command: |
-            sudo apt-get -qy install lsof
-      - run:
-          name: External Ens tests
-          command: |
-            test/externalTests/ens.sh /tmp/workspace/soljson.js || test/externalTests/ens.sh /tmp/workspace/soljson.js
-      - run: *gitter_notify_failure
-      - run: *gitter_notify_success
-
   b_win: &b_win
     executor:
       name: win/default
@@ -1176,7 +1137,7 @@ workflows:
             branches:
               only:
                 - develop
-                - develop_060
+                - breaking
 
     jobs:
       # OSSFUZZ builds and (regression) tests
@@ -1193,3 +1154,6 @@ workflows:
       - t_ubu_asan_constantinople: *workflow_ubuntu2004_asan
       - t_ubu_asan_constantinople_clang: *workflow_ubuntu2004_asan_clang
       - t_ubu_asan_cli: *workflow_ubuntu2004_asan
+
+      # External tests e.g., zeppelin, colony etc.
+      - t_ems_test_external: *workflow_emscripten

test/externalTests.sh

@@ -36,16 +36,29 @@ fi
 
 SOLJSON="$1"
 REPO_ROOT="$(dirname "$0")"
+TEST_DIR=""$REPO_ROOT"/test/externalTests"
+TMP_LOG_DIR=$(mktemp -d)
 
 source scripts/common.sh
 source test/externalTests/common.sh
 
 printTask "Running external tests..."
+# Unset so that all tests are forced to complete
+unset -e
+for test in zeppelin gnosis colony ens;
+do
+  $TEST_DIR/$test.sh "$SOLJSON" &> $TMP_LOG_DIR/$test.log &
+done
 
-$REPO_ROOT/externalTests/zeppelin.sh "$SOLJSON"
-$REPO_ROOT/externalTests/gnosis.sh "$SOLJSON"
-$REPO_ROOT/externalTests/colony.sh "$SOLJSON"
-$REPO_ROOT/externalTests/ens.sh "$SOLJSON"
+printTask "Waiting for external tests to complete..."
+wait
+
+set -e
+printTask "Logging test results..."
+rm -f $TEST_DIR/externalTests.log
+cat $TMP_LOG_DIR/*.log >> $TEST_DIR/externalTests.log
+# Remove temp log dir
+rm -rf $TMP_LOG_DIR
 
 # Disabled temporarily as it needs to be updated to latest Truffle first.
 #test_truffle Gnosis https://github.com/axic/pm-contracts.git solidity-050