Merge pull request #1295 from CJentzsch/randomStateTests

Random state tests

CMakeLists.txt

@@ -1,8 +1,10 @@
 cmake_policy(SET CMP0015 NEW)
 
 aux_source_directory(. SRC_LIST)
-list(REMOVE_ITEM SRC_LIST "./createRandomTest.cpp")
+list(REMOVE_ITEM SRC_LIST "./createRandomVMTest.cpp")
-list(REMOVE_ITEM SRC_LIST "./checkRandomTest.cpp")
+list(REMOVE_ITEM SRC_LIST "./createRandomStateTest.cpp")
+list(REMOVE_ITEM SRC_LIST "./checkRandomVMTest.cpp")
+list(REMOVE_ITEM SRC_LIST "./checkRandomStateTest.cpp")
 
 if (NOT JSONRPC)
 	list(REMOVE_ITEM SRC_LIST "./AccountHolder.cpp")
@@ -17,8 +19,10 @@ include_directories(${JSON_RPC_CPP_INCLUDE_DIRS})
 
 file(GLOB HEADERS "*.h")
 add_executable(testeth ${SRC_LIST} ${HEADERS})
-add_executable(createRandomTest createRandomTest.cpp vm.cpp TestHelper.cpp)
+add_executable(createRandomVMTest createRandomVMTest.cpp vm.cpp TestHelper.cpp)
-add_executable(checkRandomTest checkRandomTest.cpp vm.cpp TestHelper.cpp)
+add_executable(createRandomStateTest createRandomStateTest.cpp TestHelper.cpp)
+add_executable(checkRandomVMTest checkRandomVMTest.cpp vm.cpp TestHelper.cpp)
+add_executable(checkRandomStateTest checkRandomStateTest.cpp TestHelper.cpp)
 
 target_link_libraries(testeth ${Boost_UNIT_TEST_FRAMEWORK_LIBRARIES})
 target_link_libraries(testeth ${CURL_LIBRARIES})
@@ -35,10 +39,16 @@ if (JSONRPC)
 	target_link_libraries(testeth ${JSON_RPC_CPP_CLIENT_LIBRARIES})
 endif()
 
-target_link_libraries(createRandomTest ${Boost_UNIT_TEST_FRAMEWORK_LIBRARIES})
+target_link_libraries(createRandomVMTest ${Boost_UNIT_TEST_FRAMEWORK_LIBRARIES})
-target_link_libraries(createRandomTest ethereum)
+target_link_libraries(createRandomVMTest ethereum)
-target_link_libraries(createRandomTest ethcore)
+target_link_libraries(createRandomVMTest ethcore)
-target_link_libraries(checkRandomTest ${Boost_UNIT_TEST_FRAMEWORK_LIBRARIES})
+target_link_libraries(createRandomStateTest ${Boost_UNIT_TEST_FRAMEWORK_LIBRARIES})
-target_link_libraries(checkRandomTest ethereum)
+target_link_libraries(createRandomStateTest ethereum)
-target_link_libraries(checkRandomTest ethcore)
+target_link_libraries(createRandomStateTest ethcore)
+target_link_libraries(checkRandomVMTest ${Boost_UNIT_TEST_FRAMEWORK_LIBRARIES})
+target_link_libraries(checkRandomVMTest ethereum)
+target_link_libraries(checkRandomVMTest ethcore)
+target_link_libraries(checkRandomStateTest ${Boost_UNIT_TEST_FRAMEWORK_LIBRARIES})
+target_link_libraries(checkRandomStateTest ethereum)
+target_link_libraries(checkRandomStateTest ethcore)
 

TestHelper.cpp

@@ -84,12 +84,12 @@ ImportTest::ImportTest(json_spirit::mObject& _o, bool isFiller) : m_statePre(Add
 
 void ImportTest::importEnv(json_spirit::mObject& _o)
 {
-	BOOST_REQUIRE(_o.count("previousHash") > 0);
+	assert(_o.count("previousHash") > 0);
-	BOOST_REQUIRE(_o.count("currentGasLimit") > 0);
+	assert(_o.count("currentGasLimit") > 0);
-	BOOST_REQUIRE(_o.count("currentDifficulty") > 0);
+	assert(_o.count("currentDifficulty") > 0);
-	BOOST_REQUIRE(_o.count("currentTimestamp") > 0);
+	assert(_o.count("currentTimestamp") > 0);
-	BOOST_REQUIRE(_o.count("currentCoinbase") > 0);
+	assert(_o.count("currentCoinbase") > 0);
-	BOOST_REQUIRE(_o.count("currentNumber") > 0);
+	assert(_o.count("currentNumber") > 0);
 
 	m_environment.previousBlock.hash = h256(_o["previousHash"].get_str());
 	m_environment.currentBlock.number = toInt(_o["currentNumber"]);
@@ -108,10 +108,10 @@ void ImportTest::importState(json_spirit::mObject& _o, State& _state)
 	{
 		json_spirit::mObject o = i.second.get_obj();
 
-		BOOST_REQUIRE(o.count("balance") > 0);
+		assert(o.count("balance") > 0);
-		BOOST_REQUIRE(o.count("nonce") > 0);
+		assert(o.count("nonce") > 0);
-		BOOST_REQUIRE(o.count("storage") > 0);
+		assert(o.count("storage") > 0);
-		BOOST_REQUIRE(o.count("code") > 0);
+		assert(o.count("code") > 0);
 
 		if (bigint(o["balance"].get_str()) >= c_max256plus1)
 			BOOST_THROW_EXCEPTION(ValueTooLarge() << errinfo_comment("State 'balance' is equal or greater than 2**256") );
@@ -144,12 +144,12 @@ void ImportTest::importTransaction(json_spirit::mObject& _o)
 {	
 	if (_o.count("secretKey") > 0)
 	{
-		BOOST_REQUIRE(_o.count("nonce") > 0);
+		assert(_o.count("nonce") > 0);
-		BOOST_REQUIRE(_o.count("gasPrice") > 0);
+		assert(_o.count("gasPrice") > 0);
-		BOOST_REQUIRE(_o.count("gasLimit") > 0);
+		assert(_o.count("gasLimit") > 0);
-		BOOST_REQUIRE(_o.count("to") > 0);
+		assert(_o.count("to") > 0);
-		BOOST_REQUIRE(_o.count("value") > 0);
+		assert(_o.count("value") > 0);
-		BOOST_REQUIRE(_o.count("data") > 0);
+		assert(_o.count("data") > 0);
 
 		if (bigint(_o["nonce"].get_str()) >= c_max256plus1)
 			BOOST_THROW_EXCEPTION(ValueTooLarge() << errinfo_comment("Transaction 'nonce' is equal or greater than 2**256") );

checkRandomStateTest.cpp

@@ -0,0 +1,228 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/** @file checkRandomStateTest.cpp
+ * @author Christoph Jentzsch <jentzsch.simulationsoftware@gmail.com>
+ * @date 2015
+ * Check a random test and return 0/1 for success or failure. To be used for efficiency in the random test simulation.
+ */
+
+#include <libdevcore/Common.h>
+#include <libdevcore/Exceptions.h>
+#include <libdevcore/Log.h>
+#include <libevm/VMFactory.h>
+#include "TestHelper.h"
+#include "vm.h"
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+
+using namespace std;
+using namespace json_spirit;
+using namespace dev::test;
+using namespace dev;
+
+bool doStateTest(mValue& _v);
+
+int main(int argc, char *argv[])
+{
+	g_logVerbosity = 0;
+	bool ret = false;
+
+	try
+	{
+		mValue v;
+		string s;
+		for (int i = 1; i < argc; ++i)
+			s += argv[i];
+		if (asserts(s.length() > 0))
+		{
+			cout << "Content of argument is empty\n";
+			return 1;
+		}
+		read_string(s, v);
+		ret = doStateTest(v);
+	}
+	catch (Exception const& _e)
+	{
+		cout << "Failed test with Exception: " << diagnostic_information(_e) << endl;
+		ret = false;
+	}
+	catch (std::exception const& _e)
+	{
+		cout << "Failed test with Exception: " << _e.what() << endl;
+		ret = false;
+	}
+	return ret;
+}
+
+bool doStateTest(mValue& _v)
+{
+	eth::VMFactory::setKind(eth::VMKind::JIT);
+
+	for (auto& i: _v.get_obj())
+	{
+		mObject& o = i.second.get_obj();
+
+		assert(o.count("env") > 0);
+		assert(o.count("pre") > 0);
+		assert(o.count("transaction") > 0);
+
+		ImportTest importer(o, false);
+
+		eth::State theState = importer.m_statePre;
+		bytes tx = importer.m_transaction.rlp();
+		bytes output;
+
+		try
+		{
+			theState.execute(lastHashes(importer.m_environment.currentBlock.number), tx, &output);
+		}
+		catch (Exception const& _e)
+		{
+			cnote << "state execution did throw an exception: " << diagnostic_information(_e);
+			theState.commit();
+		}
+		catch (std::exception const& _e)
+		{
+			cnote << "state execution did throw an exception: " << _e.what();
+		}
+
+		assert(o.count("post") > 0);
+		assert(o.count("out") > 0);
+
+		//checkOutput(output, o);
+		int j = 0;
+		if (o["out"].type() == array_type)
+			for (auto const& d: o["out"].get_array())
+			{
+				if (asserts(output[j] == toInt(d)))
+				{
+					cout << "Output byte [" << j << "] different!";
+					return 1;
+				}
+				++j;
+			}
+		else if (o["out"].get_str().find("0x") == 0)
+		{
+			if (asserts(output == fromHex(o["out"].get_str().substr(2))))
+				return 1;
+		}
+		else
+		{
+			if (asserts(output == fromHex(o["out"].get_str())))
+				return 1;
+		}
+
+		//checkLog(theState.pending().size() ? theState.log(0) : LogEntries(), importer.m_environment.sub.logs);
+		eth::LogEntries logs = theState.pending().size() ? theState.log(0) : eth::LogEntries();
+
+		if (assertsEqual(logs.size(), importer.m_environment.sub.logs.size()))
+			return 1;
+
+		for (size_t i = 0; i < logs.size(); ++i)
+		{
+			if (assertsEqual(logs[i].address, importer.m_environment.sub.logs[i].address))
+				return 1;
+			if (assertsEqual(logs[i].topics, importer.m_environment.sub.logs[i].topics))
+				return 1;
+			if (asserts(logs[i].data == importer.m_environment.sub.logs[i].data))
+				return 1;
+		}
+
+		// check addresses
+#if ETH_FATDB
+		auto expectedAddrs = importer.m_statePost.addresses();
+		auto resultAddrs = theState.addresses();
+		for (auto& expectedPair : expectedAddrs)
+		{
+			auto& expectedAddr = expectedPair.first;
+			auto resultAddrIt = resultAddrs.find(expectedAddr);
+			if (resultAddrIt == resultAddrs.end())
+			{
+				cout << "Missing expected address " << expectedAddr;
+				return 1;
+			}
+			else
+			{
+				if (importer.m_statePost.balance(expectedAddr) !=  theState.balance(expectedAddr))
+				{
+					cout << expectedAddr << ": incorrect balance " << theState.balance(expectedAddr) << ", expected " << importer.m_statePost.balance(expectedAddr);
+					return 1;
+				}
+				if (importer.m_statePost.transactionsFrom(expectedAddr) !=  theState.transactionsFrom(expectedAddr))
+				{
+					cout << expectedAddr << ": incorrect txCount " << theState.transactionsFrom(expectedAddr) << ", expected " << importer.m_statePost.transactionsFrom(expectedAddr);
+					return 1;
+				}
+				if (importer.m_statePost.code(expectedAddr) != theState.code(expectedAddr))
+				{
+					cout << expectedAddr << ": incorrect code";
+					return 1;
+				}
+
+				//checkStorage(importer.m_statePost.storage(expectedAddr), theState.storage(expectedAddr), expectedAddr);
+				map<u256, u256> _resultStore = theState.storage(expectedAddr);
+
+				for (auto&& expectedStorePair : importer.m_statePost.storage(expectedAddr))
+				{
+					auto& expectedStoreKey = expectedStorePair.first;
+					auto resultStoreIt = _resultStore.find(expectedStoreKey);
+					if (resultStoreIt == _resultStore.end())
+					{
+						cout << expectedAddr << ": missing store key " << expectedStoreKey << endl;
+						return 1;
+					}
+					else
+					{
+						auto& expectedStoreValue = expectedStorePair.second;
+						auto& resultStoreValue = resultStoreIt->second;
+						if (asserts(expectedStoreValue == resultStoreValue))
+						{
+							cout << expectedAddr << ": store[" << expectedStoreKey << "] = " << resultStoreValue << ", expected " << expectedStoreValue << endl;
+							return 1;
+						}
+					}
+				}
+				if (assertsEqual(_resultStore.size(), importer.m_statePost.storage(expectedAddr).size()))
+					return 1;
+				for (auto&& resultStorePair: _resultStore)
+				{
+					if (!importer.m_statePost.storage(expectedAddr).count(resultStorePair.first))
+					{
+						cout << expectedAddr << ": unexpected store key " << resultStorePair.first << endl;
+						return 1;
+					}
+				}
+			}
+		}
+		//checkAddresses<map<Address, u256> >(expectedAddrs, resultAddrs);
+		for (auto& resultPair : resultAddrs)
+		{
+			auto& resultAddr = resultPair.first;
+			auto expectedAddrIt = expectedAddrs.find(resultAddr);
+			if (expectedAddrIt == expectedAddrs.end())
+				return 1;
+		}
+		if (expectedAddrs != resultAddrs)
+			return 1;
+#endif
+		if (theState.rootHash() != h256(o["postStateRoot"].get_str()))
+		{
+			cout << "wrong post state root" << endl;
+			return 1;
+		}
+	}
+	return 0;
+}

checkRandomVMTest.cpp

@@ -32,7 +32,7 @@ using namespace json_spirit;
 using namespace dev::test;
 using namespace dev;
 
-bool doVMTest(mValue& v);
+bool doVMTest(mValue& _v);
 
 int main(int argc, char *argv[])
 {
@@ -66,11 +66,11 @@ int main(int argc, char *argv[])
 	return ret;
 }
 
-bool doVMTest(mValue& v)
+bool doVMTest(mValue& _v)
 {
 	eth::VMFactory::setKind(eth::VMKind::JIT);
 
-	for (auto& i: v.get_obj())
+	for (auto& i: _v.get_obj())
 	{
 		cnote << i.first;
 		mObject& o = i.second.get_obj();

createRandomStateTest.cpp

@@ -0,0 +1,183 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/** @file createRandomStateTest.cpp
+ * @author Christoph Jentzsch <jentzsch.simulationsoftware@gmail.com>
+ * @date 2015
+ * Creating a random state test.
+ */
+
+#include <string>
+#include <iostream>
+#include <chrono>
+
+#include <boost/random.hpp>
+#include <boost/filesystem/path.hpp>
+
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+#include <json_spirit/json_spirit.h>
+#include <json_spirit/json_spirit_reader_template.h>
+#include <json_spirit/json_spirit_writer_template.h>
+#include <libdevcore/CommonIO.h>
+#include <libdevcore/CommonData.h>
+#include <libevmcore/Instruction.h>
+#include <libevm/VMFactory.h>
+#include "TestHelper.h"
+#include "vm.h"
+
+using namespace std;
+using namespace json_spirit;
+using namespace dev;
+
+void doStateTests(json_spirit::mValue& _v);
+
+int main(int argc, char *argv[])
+{
+	g_logVerbosity = 0;
+
+	// create random code
+
+	boost::random::mt19937 gen;
+
+	auto now = chrono::steady_clock::now().time_since_epoch();
+	auto timeSinceEpoch = chrono::duration_cast<chrono::nanoseconds>(now).count();
+	gen.seed(static_cast<unsigned int>(timeSinceEpoch));
+	boost::random::uniform_int_distribution<> lengthOfCodeDist(2, 16);
+	boost::random::uniform_int_distribution<> opcodeDist(0, 255);
+	boost::random::uniform_int_distribution<> BlockInfoOpcodeDist(0x40, 0x45);
+	boost::random::variate_generator<boost::mt19937&, boost::random::uniform_int_distribution<> > randGen(gen, opcodeDist);
+	boost::random::variate_generator<boost::mt19937&, boost::random::uniform_int_distribution<> > randGenBlockInfoOpcode(gen, BlockInfoOpcodeDist);
+
+	int lengthOfCode  = lengthOfCodeDist(gen);
+	string randomCode;
+
+	for (int i = 0; i < lengthOfCode; ++i)
+	{
+		if (i < 8 && (randGen() < 192))
+		{
+			randomCode += toHex(toCompactBigEndian((uint8_t)randGenBlockInfoOpcode()));
+			continue;
+		}
+
+		uint8_t opcode = randGen();
+		// disregard all invalid commands, except of one (0x0c)
+		if ((dev::eth::isValidInstruction(dev::eth::Instruction(opcode)) || (randGen() > 250)))
+			randomCode += toHex(toCompactBigEndian(opcode));
+		else
+			i--;
+	}
+
+	string const s = R"(
+	{
+		"randomStatetest" : {
+			"env" : {
+				"currentCoinbase" : "945304eb96065b2a98b57a48a06ae28d285a71b5",
+				"currentDifficulty" : "5623894562375",
+				"currentGasLimit" : "115792089237316195423570985008687907853269984665640564039457584007913129639935",
+				"currentNumber" : "0",
+				"currentTimestamp" : "1",
+				"previousHash" : "5e20a0453cecd065ea59c37ac63e079ee08998b6045136a8ce6635c7912ec0b6"
+			},
+			"pre" : {
+				"095e7baea6a6c7c4c2dfeb977efac326af552d87" : {
+					"balance" : "1000000000000000000",
+					"code" : "0x6001600101600055",
+					"nonce" : "0",
+					"storage" : {
+					}
+				},
+				"945304eb96065b2a98b57a48a06ae28d285a71b5" : {
+					"balance" : "46",
+					"code" : "0x6000355415600957005b60203560003555",
+					"nonce" : "0",
+					"storage" : {
+					}
+				},
+				"a94f5374fce5edbc8e2a8697c15331677e6ebf0b" : {
+					"balance" : "1000000000000000000",
+					"code" : "0x",
+					"nonce" : "0",
+					"storage" : {
+					}
+				}
+			},
+			"transaction" : {
+				"data" : "0x42",
+				"gasLimit" : "400000",
+				"gasPrice" : "1",
+				"nonce" : "0",
+				"secretKey" : "45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8",
+				"to" : "095e7baea6a6c7c4c2dfeb977efac326af552d87",
+				"value" : "100000"
+			}
+		}
+	}
+)";
+	mValue v;
+	read_string(s, v);
+
+	// insert new random code
+	v.get_obj().find("randomStatetest")->second.get_obj().find("pre")->second.get_obj().begin()->second.get_obj()["code"] = "0x" + randomCode + (randGen() > 128 ? "55" : "");
+
+	// fill test
+	doStateTests(v);
+
+	// stream to output for further handling by the bash script
+	cout << json_spirit::write_string(v, true);
+
+	return 0;
+}
+
+void doStateTests(json_spirit::mValue& _v)
+{
+	eth::VMFactory::setKind(eth::VMKind::Interpreter);
+
+	for (auto& i: _v.get_obj())
+	{
+		//cerr << i.first << endl;
+		mObject& o = i.second.get_obj();
+
+		assert(o.count("env") > 0);
+		assert(o.count("pre") > 0);
+		assert(o.count("transaction") > 0);
+
+		test::ImportTest importer(o, true);
+
+		eth::State theState = importer.m_statePre;
+		bytes tx = importer.m_transaction.rlp();
+		bytes output;
+
+		try
+		{
+			theState.execute(test::lastHashes(importer.m_environment.currentBlock.number), tx, &output);
+		}
+		catch (Exception const& _e)
+		{
+			cnote << "state execution did throw an exception: " << diagnostic_information(_e);
+			theState.commit();
+		}
+		catch (std::exception const& _e)
+		{
+			cnote << "state execution did throw an exception: " << _e.what();
+		}
+#if ETH_FATDB
+		importer.exportTest(output, theState);
+#else
+		cout << "You can not fill tests when FATDB is switched off";
+#endif
+	}
+}
+

createRandomVMTest.cpp

@@ -41,7 +41,7 @@ using namespace std;
 using namespace json_spirit;
 using namespace dev;
 
-void doMyTests(json_spirit::mValue& v);
+void doMyTests(json_spirit::mValue& _v);
 
 int main(int argc, char *argv[])
 {
@@ -127,11 +127,11 @@ int main(int argc, char *argv[])
 	return 0;
 }
 
-void doMyTests(json_spirit::mValue& v)
+void doMyTests(json_spirit::mValue& _v)
 {
 	eth::VMFactory::setKind(eth::VMKind::Interpreter);
 
-	for (auto& i: v.get_obj())
+	for (auto& i: _v.get_obj())
 	{
 		cnote << i.first;
 		mObject& o = i.second.get_obj();

stSystemOperationsTestFiller.json

@@ -389,7 +389,7 @@
             "095e7baea6a6c7c4c2dfeb977efac326af552d87" : {
                 "balance" : "1000000000000000000",
                 "nonce" : "0",
-                "code" : "{ (MSTORE 0 0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff) (MSTORE 32 0xaaffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffaa ) [[ 0 ]] (CALL 1000 0x945304eb96065b2a98b57a48a06ae28d285a71b5 23 0 64 64 0) }",
+                "code" : "{ (MSTORE 0 0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff) (MSTORE 32 0xaaffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffaa ) [[ 0 ]] (CALL 100000 0x945304eb96065b2a98b57a48a06ae28d285a71b5 23 0 64 64 0) }",
                 "storage": {}
             },
             "945304eb96065b2a98b57a48a06ae28d285a71b5" : {
@@ -410,7 +410,7 @@
         "transaction" : {
             "nonce" : "0",
             "gasPrice" : "1",
-            "gasLimit" : "30000",
+            "gasLimit" : "300000",
             "to" : "095e7baea6a6c7c4c2dfeb977efac326af552d87",
             "value" : "100000",
             "secretKey" : "45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8",

state.cpp

@@ -248,6 +248,39 @@ BOOST_AUTO_TEST_CASE(stCreateTest)
 	}
 }
 
+BOOST_AUTO_TEST_CASE(stRandom)
+{
+	string testPath = dev::test::getTestPath();
+	testPath += "/StateTests/RandomTests";
+
+	vector<boost::filesystem::path> testFiles;
+	boost::filesystem::directory_iterator iterator(testPath);
+	for(; iterator != boost::filesystem::directory_iterator(); ++iterator)
+		if (boost::filesystem::is_regular_file(iterator->path()) && iterator->path().extension() == ".json")
+			testFiles.push_back(iterator->path());
+
+	for (auto& path: testFiles)
+	{
+		try
+		{
+			cnote << "Testing ..." << path.filename();
+			json_spirit::mValue v;
+			string s = asString(dev::contents(path.string()));
+			BOOST_REQUIRE_MESSAGE(s.length() > 0, "Content of " + path.string() + " is empty. Have you cloned the 'tests' repo branch develop and set ETHEREUM_TEST_PATH to its path?");
+			json_spirit::read_string(s, v);
+			dev::test::doStateTests(v, false);
+		}
+		catch (Exception const& _e)
+		{
+			BOOST_ERROR("Failed test with Exception: " << diagnostic_information(_e));
+		}
+		catch (std::exception const& _e)
+		{
+			BOOST_ERROR("Failed test with Exception: " << _e.what());
+		}
+	}
+}
+
 BOOST_AUTO_TEST_CASE(userDefinedFileState)
 {
 	dev::test::userDefinedTest("--singletest", dev::test::doStateTests);