testeth: optional VM tracing (--vmtrace), command line options refactoring.

Closes ethereum/cpp-ethereum#1280

TestHelper.cpp

@@ -439,8 +439,6 @@ void userDefinedTest(string testTypeFlag, std::function<void(json_spirit::mValue
 			}
 			g_logVerbosity = currentVerbosity;
 		}
-		else
-			continue;
 	}
 }
 
@@ -449,32 +447,27 @@ void executeTests(const string& _name, const string& _testPathAppendix, std::fun
 	string testPath = getTestPath();
 	testPath += _testPathAppendix;
 
-	for (int i = 1; i < boost::unit_test::framework::master_test_suite().argc; ++i)
+	if (Options::get().fillTests)
 	{
-		string arg = boost::unit_test::framework::master_test_suite().argv[i];
+		try
-		if (arg == "--filltests")
 		{
-			try
+			cnote << "Populating tests...";
-			{
+			json_spirit::mValue v;
-				cnote << "Populating tests...";
+			boost::filesystem::path p(__FILE__);
-				json_spirit::mValue v;
+			boost::filesystem::path dir = p.parent_path();
-				boost::filesystem::path p(__FILE__);
+			string s = asString(dev::contents(dir.string() + "/" + _name + "Filler.json"));
-				boost::filesystem::path dir = p.parent_path();
+			BOOST_REQUIRE_MESSAGE(s.length() > 0, "Contents of " + dir.string() + "/" + _name + "Filler.json is empty.");
-				string s = asString(dev::contents(dir.string() + "/" + _name + "Filler.json"));
+			json_spirit::read_string(s, v);
-				BOOST_REQUIRE_MESSAGE(s.length() > 0, "Contents of " + dir.string() + "/" + _name + "Filler.json is empty.");
+			doTests(v, true);
-				json_spirit::read_string(s, v);
+			writeFile(testPath + "/" + _name + ".json", asBytes(json_spirit::write_string(v, true)));
-				doTests(v, true);
+		}
-				writeFile(testPath + "/" + _name + ".json", asBytes(json_spirit::write_string(v, true)));
+		catch (Exception const& _e)
-			}
+		{
-			catch (Exception const& _e)
+			BOOST_ERROR("Failed filling test with Exception: " << diagnostic_information(_e));
-			{
+		}
-				BOOST_ERROR("Failed filling test with Exception: " << diagnostic_information(_e));
+		catch (std::exception const& _e)
-			}
+		{
-			catch (std::exception const& _e)
+			BOOST_ERROR("Failed filling test with Exception: " << _e.what());
-			{
-				BOOST_ERROR("Failed filling test with Exception: " << _e.what());
-			}
-			break;
 		}
 	}
 
@@ -541,21 +534,48 @@ RLPStream createRLPStreamFromTransactionFields(json_spirit::mObject& _tObj)
 	return rlpStream;
 }
 
-void processCommandLineOptions()
+Options::Options()
 {
 	auto argc = boost::unit_test::framework::master_test_suite().argc;
 	auto argv = boost::unit_test::framework::master_test_suite().argv;
 
-	for (auto i =  0; i < argc; ++i)
+	for (auto i = 0; i < argc; ++i)
 	{
-		if (std::string(argv[i]) == "--jit")
+		auto arg = std::string{argv[i]};
+		if (arg == "--jit")
 		{
+			jit = true;
 			eth::VMFactory::setKind(eth::VMKind::JIT);
-			break;
+		}
+		else if (arg == "--vmtrace")
+			vmtrace = true;
+		else if (arg == "--performance")
+			performance = true;
+		else if (arg == "--quadratic")
+			quadratic = true;
+		else if (arg == "--memory")
+			memory = true;
+		else if (arg == "--inputlimits")
+			inputLimits = true;
+		else if (arg == "--bigdata")
+			bigData = true;
+		else if (arg == "--all")
+		{
+			performance = true;
+			quadratic = true;
+			memory = true;
+			inputLimits = true;
+			bigData = true;
 		}
 	}
 }
 
+Options const& Options::get()
+{
+	static Options instance;
+	return instance;
+}
+
 LastHashes lastHashes(u256 _currentBlockNumber)
 {
 	LastHashes ret;

TestHelper.h

@@ -141,7 +141,6 @@ void executeTests(const std::string& _name, const std::string& _testPathAppendix
 std::string getTestPath();
 void userDefinedTest(std::string testTypeFlag, std::function<void(json_spirit::mValue&, bool)> doTests);
 RLPStream createRLPStreamFromTransactionFields(json_spirit::mObject& _tObj);
-void processCommandLineOptions();
 eth::LastHashes lastHashes(u256 _currentBlockNumber);
 json_spirit::mObject fillJsonWithState(eth::State _state);
 
@@ -158,5 +157,31 @@ void checkAddresses(mapType& _expectedAddrs, mapType& _resultAddrs)
 	BOOST_CHECK(_expectedAddrs == _resultAddrs);
 }
 
+class Options
+{
+public:
+	bool jit = false;		///< Use JIT
+	bool vmtrace = false;	///< Create EVM execution tracer // TODO: Link with log verbosity?
+	bool showTimes = false;	///< Print test groups execution times
+	bool fillTests = false; ///< Create JSON test files from execution results
+
+	/// Test selection
+	/// @{
+	bool performance = false;
+	bool quadratic = false;
+	bool memory = false;
+	bool inputLimits = false;
+	bool bigData = false;
+	/// @}
+
+	/// Get reference to options
+	/// The first time used, options are parsed
+	static Options const& get();
+
+private:
+	Options();
+	Options(Options const&) = delete;
+};
+
 }
 }

state.cpp

@@ -41,7 +41,7 @@ namespace dev {  namespace test {
 
 void doStateTests(json_spirit::mValue& v, bool _fillin)
 {
-	processCommandLineOptions();
+	Options::get(); // process command line options
 
 	for (auto& i: v.get_obj())
 	{
@@ -172,48 +172,40 @@ BOOST_AUTO_TEST_CASE(stBlockHashTest)
 
 BOOST_AUTO_TEST_CASE(stQuadraticComplexityTest)
 {
-	   for (int i = 1; i < boost::unit_test::framework::master_test_suite().argc; ++i)
+	if (test::Options::get().quadratic)
-	   {
+	{
-			   string arg = boost::unit_test::framework::master_test_suite().argv[i];
+		auto start = chrono::steady_clock::now();
-			   if (arg == "--quadratic" || arg == "--all")
-			   {
-					   auto start = chrono::steady_clock::now();
 
-					   dev::test::executeTests("stQuadraticComplexityTest", "/StateTests", dev::test::doStateTests);
+		dev::test::executeTests("stQuadraticComplexityTest", "/StateTests", dev::test::doStateTests);
 
-					   auto end = chrono::steady_clock::now();
+		auto end = chrono::steady_clock::now();
-					   auto duration(chrono::duration_cast<chrono::milliseconds>(end - start));
+		auto duration(chrono::duration_cast<chrono::milliseconds>(end - start));
-					   cnote << "test duration: " << duration.count() << " milliseconds.\n";
+		cnote << "test duration: " << duration.count() << " milliseconds.\n";
-			   }
+	}
-	   }
 }
 
 BOOST_AUTO_TEST_CASE(stMemoryStressTest)
 {
-	   for (int i = 1; i < boost::unit_test::framework::master_test_suite().argc; ++i)
+	if (test::Options::get().memory)
-	   {
+	{
-			   string arg = boost::unit_test::framework::master_test_suite().argv[i];
+		auto start = chrono::steady_clock::now();
-			   if (arg == "--memory" || arg == "--all")
-			   {
-					   auto start = chrono::steady_clock::now();
 
-					   dev::test::executeTests("stMemoryStressTest", "/StateTests", dev::test::doStateTests);
+		dev::test::executeTests("stMemoryStressTest", "/StateTests", dev::test::doStateTests);
 
-					   auto end = chrono::steady_clock::now();
+		auto end = chrono::steady_clock::now();
-					   auto duration(chrono::duration_cast<chrono::milliseconds>(end - start));
+		auto duration(chrono::duration_cast<chrono::milliseconds>(end - start));
-					   cnote << "test duration: " << duration.count() << " milliseconds.\n";
+		cnote << "test duration: " << duration.count() << " milliseconds.\n";
-			   }
+	}
-	   }
 }
 
- BOOST_AUTO_TEST_CASE(stSolidityTest)
+BOOST_AUTO_TEST_CASE(stSolidityTest)
- {
+{
-		dev::test::executeTests("stSolidityTest", "/StateTests", dev::test::doStateTests);
+	dev::test::executeTests("stSolidityTest", "/StateTests", dev::test::doStateTests);
- }
+}
 
 BOOST_AUTO_TEST_CASE(stMemoryTest)
 {
-	   dev::test::executeTests("stMemoryTest", "/StateTests", dev::test::doStateTests);
+	dev::test::executeTests("stMemoryTest", "/StateTests", dev::test::doStateTests);
 }
 
 

transaction.cpp

@@ -116,21 +116,16 @@ BOOST_AUTO_TEST_CASE(ttWrongRLPTransaction)
 
 BOOST_AUTO_TEST_CASE(tt10mbDataField)
 {
-	for (int i = 1; i < boost::unit_test::framework::master_test_suite().argc; ++i)
+	if (test::Options::get().bigData)
 	{
-		string arg = boost::unit_test::framework::master_test_suite().argv[i];
+		auto start = chrono::steady_clock::now();
-		if (arg == "--bigdata" || arg == "--all")
-		{
-			auto start = chrono::steady_clock::now();
 
-			dev::test::executeTests("tt10mbDataField", "/TransactionTests", dev::test::doTransactionTests);
+		dev::test::executeTests("tt10mbDataField", "/TransactionTests", dev::test::doTransactionTests);
 
-			auto end = chrono::steady_clock::now();
+		auto end = chrono::steady_clock::now();
-			auto duration(chrono::duration_cast<chrono::milliseconds>(end - start));
+		auto duration(chrono::duration_cast<chrono::milliseconds>(end - start));
-			cnote << "test duration: " << duration.count() << " milliseconds.\n";
+		cnote << "test duration: " << duration.count() << " milliseconds.\n";
-		}
 	}
-
 }
 
 BOOST_AUTO_TEST_CASE(ttCreateTest)

vm.cpp

@@ -312,7 +312,7 @@ namespace dev { namespace test {
 
 void doVMTests(json_spirit::mValue& v, bool _fillin)
 {
-	processCommandLineOptions();
+	Options::get(); // process command line options // TODO: We need to control the main() function
 
 	for (auto& i: v.get_obj())
 	{
@@ -344,7 +344,8 @@ void doVMTests(json_spirit::mValue& v, bool _fillin)
 		try
 		{
 			auto vm = eth::VMFactory::create(fev.gas);
-			output = vm->go(fev, fev.simpleTrace()).toBytes();
+			auto vmtrace = Options::get().vmtrace ? fev.simpleTrace() : OnOpFunc{};
+			output = vm->go(fev, vmtrace).toBytes();
 			gas = vm->gas();
 		}
 		catch (VMException const&)
@@ -364,18 +365,12 @@ void doVMTests(json_spirit::mValue& v, bool _fillin)
 		}
 
 		auto endTime = std::chrono::high_resolution_clock::now();
-		auto argc = boost::unit_test::framework::master_test_suite().argc;
+		if (Options::get().showTimes)
-		auto argv = boost::unit_test::framework::master_test_suite().argv;
-		for (auto i = 0; i < argc; ++i)
 		{
-			if (std::string(argv[i]) == "--show-times")
+			auto testDuration = endTime - startTime;
-			{
+			cnote << "Execution time: "
-				auto testDuration = endTime - startTime;
+				  << std::chrono::duration_cast<std::chrono::milliseconds>(testDuration).count()
-				cnote << "Execution time: "
+				  << " ms";
-				      << std::chrono::duration_cast<std::chrono::milliseconds>(testDuration).count()
-				      << " ms";
-				break;
-			}
 		}
 
 		// delete null entries in storage for the sake of comparison
@@ -517,58 +512,42 @@ BOOST_AUTO_TEST_CASE(vmSystemOperationsTest)
 
 BOOST_AUTO_TEST_CASE(vmPerformanceTest)
 {
-	for (int i = 1; i < boost::unit_test::framework::master_test_suite().argc; ++i)
+	if (test::Options::get().performance)
 	{
-		string arg = boost::unit_test::framework::master_test_suite().argv[i];
+		auto start = chrono::steady_clock::now();
-		if (arg == "--performance" || arg == "--all")
-		{
-			auto start = chrono::steady_clock::now();
 
-			dev::test::executeTests("vmPerformanceTest", "/VMTests", dev::test::doVMTests);
+		dev::test::executeTests("vmPerformanceTest", "/VMTests", dev::test::doVMTests);
 
-			auto end = chrono::steady_clock::now();
+		auto end = chrono::steady_clock::now();
-			auto duration(chrono::duration_cast<chrono::milliseconds>(end - start));
+		auto duration(chrono::duration_cast<chrono::milliseconds>(end - start));
-			cnote << "test duration: " << duration.count() << " milliseconds.\n";
+		cnote << "test duration: " << duration.count() << " milliseconds.\n";
-		}
 	}
 }
 
 BOOST_AUTO_TEST_CASE(vmInputLimitsTest1)
 {
-	for (int i = 1; i < boost::unit_test::framework::master_test_suite().argc; ++i)
+	if (test::Options::get().inputLimits)
 	{
-		string arg = boost::unit_test::framework::master_test_suite().argv[i];
+		auto start = chrono::steady_clock::now();
-		if (arg == "--inputlimits" || arg == "--all")
-		{
-			auto start = chrono::steady_clock::now();
 
-			dev::test::executeTests("vmInputLimits1", "/VMTests", dev::test::doVMTests);
+		dev::test::executeTests("vmInputLimits1", "/VMTests", dev::test::doVMTests);
 
-			auto end = chrono::steady_clock::now();
+		auto end = chrono::steady_clock::now();
-			auto duration(chrono::duration_cast<chrono::milliseconds>(end - start));
+		auto duration(chrono::duration_cast<chrono::milliseconds>(end - start));
-			cnote << "test duration: " << duration.count() << " milliseconds.\n";
+		cnote << "test duration: " << duration.count() << " milliseconds.\n";
-		}
 	}
 }
 
 BOOST_AUTO_TEST_CASE(vmInputLimitsTest2)
 {
-	for (int i = 1; i < boost::unit_test::framework::master_test_suite().argc; ++i)
+	if (test::Options::get().inputLimits)
-	{
+		dev::test::executeTests("vmInputLimits2", "/VMTests", dev::test::doVMTests);
-		string arg = boost::unit_test::framework::master_test_suite().argv[i];
-		if (arg == "--inputlimits" || arg == "--all")
-			dev::test::executeTests("vmInputLimits2", "/VMTests", dev::test::doVMTests);
-	}
 }
 
 BOOST_AUTO_TEST_CASE(vmInputLimitsLightTest)
 {
-	for (int i = 1; i < boost::unit_test::framework::master_test_suite().argc; ++i)
+	if (test::Options::get().inputLimits)
-	{
+		dev::test::executeTests("vmInputLimitsLight", "/VMTests", dev::test::doVMTests);
-		string arg = boost::unit_test::framework::master_test_suite().argv[i];
-		if (arg == "--inputlimits" || arg == "--all")
-			dev::test::executeTests("vmInputLimitsLight", "/VMTests", dev::test::doVMTests);
-	}
 }
 
 BOOST_AUTO_TEST_CASE(vmRandom)