Test both EVM versions.

circle.yml

@@ -5,10 +5,6 @@ jobs:
       - image: trzeci/emscripten:sdk-tag-1.37.21-64bit
     steps:
       - checkout
-      - run:
-          name: Init submodules
-          command: |
-            git submodule update --init
       - restore_cache:
           name: Restore Boost build
           key: &boost-cache-key emscripten-boost-{{ checksum "scripts/travis-emscripten/install_deps.sh" }}{{ checksum "scripts/travis-emscripten/build_emscripten.sh" }}
@@ -94,10 +90,6 @@ jobs:
           command: |
             apt-get -qq update
             apt-get -qy install ccache cmake libboost-all-dev libz3-dev
-      - run:
-          name: Init submodules
-          command: |
-            git submodule update --init
       - run:
           name: Store commit hash and prerelease
           command: |

scripts/tests.sh

@@ -37,11 +37,9 @@ then
         echo "Usage: $0 [--junit_report <report_directory>]"
         exit 1
     fi
-    testargs_no_opt="--logger=JUNIT,test_suite,$2/no_opt.xml"
-    testargs_opt="--logger=JUNIT,test_suite,$2/opt.xml"
+    log_directory="$2"
 else
-    testargs_no_opt=''
-    testargs_opt=''
+    log_directory=""
 fi
 
 echo "Running commandline tests..."
@@ -98,10 +96,31 @@ then
     progress=""
 fi
 
-echo "--> Running tests without optimizer..."
-"$REPO_ROOT"/build/test/soltest $testargs_no_opt $progress -- --ipcpath /tmp/test/geth.ipc
-echo "--> Running tests WITH optimizer..."
-"$REPO_ROOT"/build/test/soltest $testargs_opt $progress -- --optimize --ipcpath /tmp/test/geth.ipc
+ERROR_CODE=0
+# And then run the Solidity unit-tests in the matrix combination of optimizer / no optimizer
+# and homestead / byzantium VM, # pointing to that IPC endpoint.
+for optimize in "" "--optimize"
+do
+  for vm in homestead byzantium
+  do
+    echo "--> Running tests using "$optimize" --evm-version "$vm"..."
+    log=""
+    if [ -n "$log_directory" ]
+    then
+      if [ -n "$optimize" ]
+      then
+        log=--logger=JUNIT,test_suite,$log_directory/opt_$vm.xml $testargs
+      else
+        log=--logger=JUNIT,test_suite,$log_directory/noopt_$vm.xml $testargs_no_opt
+      fi
+    fi
+    set +e
+    "$REPO_ROOT"/build/test/soltest $progress $log -- "$optimize" --evm-version "$vm" --ipcpath /tmp/test/geth.ipc
+    THIS_ERR=$?
+    set -e
+    if [ $? -ne 0 ]; then ERROR_CODE=$?; fi
+  done
+done
 
 wait $CMDLINE_PID
 

test/ExecutionFramework.cpp

@@ -20,13 +20,15 @@
  * Framework for executing contracts and testing them using RPC.
  */
 
-#include <cstdlib>
-#include <boost/test/framework.hpp>
-#include <libdevcore/CommonIO.h>
 #include <test/ExecutionFramework.h>
 
+#include <libdevcore/CommonIO.h>
+
+#include <boost/test/framework.hpp>
 #include <boost/algorithm/string/replace.hpp>
 
+#include <cstdlib>
+
 using namespace std;
 using namespace dev;
 using namespace dev::test;
@@ -53,6 +55,13 @@ ExecutionFramework::ExecutionFramework() :
 	m_showMessages(dev::test::Options::get().showMessages),
 	m_sender(m_rpc.account(0))
 {
+	if (!dev::test::Options::get().evmVersion.empty())
+	{
+		auto version = solidity::EVMVersion::fromString(dev::test::Options::get().evmVersion);
+		BOOST_REQUIRE_MESSAGE(version, "Invalid EVM version: " + dev::test::Options::get().evmVersion);
+		m_evmVersion = *version;
+	}
+
 	m_rpc.test_rewindToBlock(0);
 }
 

test/ExecutionFramework.h

@@ -25,6 +25,8 @@
 #include <test/TestHelper.h>
 #include <test/RPCSession.h>
 
+#include <libsolidity/interface/EVMVersion.h>
+
 #include <libdevcore/FixedHash.h>
 #include <libdevcore/SHA3.h>
 
@@ -227,6 +229,7 @@ protected:
 		bytes data;
 	};
 
+	solidity::EVMVersion m_evmVersion;
 	unsigned m_optimizeRuns = 200;
 	bool m_optimize = false;
 	bool m_showMessages = false;

test/TestHelper.cpp

@@ -19,8 +19,12 @@
 * @date 2014
 */
 
+#include <test/TestHelper.h>
+
+#include <libsolidity/interface/EVMVersion.h>
+
 #include <boost/test/framework.hpp>
-#include "TestHelper.h"
+
 using namespace std;
 using namespace dev::test;
 
@@ -41,6 +45,11 @@ Options::Options()
 		}
 		else if (string(suite.argv[i]) == "--optimize")
 			optimize = true;
+		else if (string(suite.argv[i]) == "--evm-version")
+		{
+			evmVersion = i + 1 < suite.argc ? suite.argv[i + 1] : "INVALID";
+			++i;
+		}
 		else if (string(suite.argv[i]) == "--show-messages")
 			showMessages = true;
 		else if (string(suite.argv[i]) == "--no-ipc")

test/TestHelper.h

@@ -19,11 +19,14 @@
 
 #pragma once
 
-#include <functional>
+#include <libsolidity/interface/EVMVersion.h>
+
 #include <boost/test/unit_test.hpp>
 #include <boost/filesystem.hpp>
 #include <boost/version.hpp>
 
+#include <functional>
+
 namespace dev
 {
 namespace test
@@ -33,6 +36,7 @@ struct Options: boost::noncopyable
 {
 	std::string ipcPath;
 	bool showMessages = false;
+	std::string evmVersion;
 	bool optimize = false;
 	bool disableIPC = false;
 	bool disableSMT = false;

test/libsolidity/SolidityExecutionFramework.h

@@ -68,6 +68,7 @@ public:
 		m_compiler.reset(false);
 		m_compiler.addSource("", sourceCode);
 		m_compiler.setLibraries(_libraryAddresses);
+		m_compiler.setEVMVersion(m_evmVersion);
 		m_compiler.setOptimiserSettings(m_optimize, m_optimizeRuns);
 		if (!m_compiler.compile())
 		{