Merge pull request #2604 from winsvega/verbosity

Nice Error Report

TestHelper.cpp

@@ -253,12 +253,16 @@ void ImportTest::importTransaction(json_spirit::mObject const& o_tr)
 	importTransaction(o_tr, m_transaction);
 }
 
-void ImportTest::compareStates(State const& _stateExpect, State const& _statePost, AccountMaskMap const _expectedStateOptions, WhenError _throw)
+int ImportTest::compareStates(State const& _stateExpect, State const& _statePost, AccountMaskMap const _expectedStateOptions, WhenError _throw)
 {
 	#define CHECK(a,b)						\
 		{									\
 			if (_throw == WhenError::Throw) \
-				{TBOOST_CHECK_MESSAGE(a,b);}\
+			{								\
+				TBOOST_CHECK_MESSAGE(a, b);	\
+				if (!a)						\
+					return 1;				\
+			}								\
 			else							\
 				{TBOOST_WARN_MESSAGE(a,b);}	\
 		}
@@ -309,12 +313,13 @@ void ImportTest::compareStates(State const& _stateExpect, State const& _statePos
 						"Check State: " << a.first <<  ": incorrect code '" << toHex(_statePost.code(a.first)) << "', expected '" << toHex(_stateExpect.code(a.first)) << "'");
 		}
 	}
+	return 0;
 }
 
-void ImportTest::exportTest(bytes const& _output)
+int ImportTest::exportTest(bytes const& _output)
 {
+	int err = 0;
 	// export output
-
 	m_testObject["out"] = (_output.size() > 4096 && !Options::get().fulloutput) ? "#" + toString(_output.size()) : toHex(_output, 2, HexPrefix::Add);
 
 	// compare expected output with post output
@@ -322,7 +327,12 @@ void ImportTest::exportTest(bytes const& _output)
 	{
 		std::string warning = "Check State: Error! Unexpected output: " + m_testObject["out"].get_str() + " Expected: " + m_testObject["expectOut"].get_str();
 		if (Options::get().checkState)
-			{TBOOST_CHECK_MESSAGE((m_testObject["out"].get_str() == m_testObject["expectOut"].get_str()), warning);}
+		{
+			bool statement = (m_testObject["out"].get_str() == m_testObject["expectOut"].get_str());
+			TBOOST_CHECK_MESSAGE(statement, warning);
+			if (!statement)
+				err = 1;
+		}
 		else
 			TBOOST_WARN_MESSAGE((m_testObject["out"].get_str() == m_testObject["expectOut"].get_str()), warning);
 
@@ -350,6 +360,7 @@ void ImportTest::exportTest(bytes const& _output)
 	m_testObject["pre"] = fillJsonWithState(m_statePre);
 	m_testObject["env"] = makeAllFieldsHex(m_testObject["env"].get_obj());
 	m_testObject["transaction"] = makeAllFieldsHex(m_testObject["transaction"].get_obj());
+	return err;
 }
 
 json_spirit::mObject fillJsonWithTransaction(Transaction _txn)
@@ -566,7 +577,10 @@ void userDefinedTest(std::function<void(json_spirit::mValue&, bool)> doTests)
 
 	auto& filename = Options::get().singleTestFile;
 	auto& testname = Options::get().singleTestName;
-	VerbosityHolder sentinel(12);
+
+	if (g_logVerbosity != -1)
+		VerbosityHolder sentinel(12);
+
 	try
 	{
 		cnote << "Testing user defined test: " << filename;
@@ -631,7 +645,7 @@ void executeTests(const string& _name, const string& _testPathAppendix, const bo
 
 	try
 	{
-		std::cout << "TEST " << _name << ":\n";
+		cnote << "TEST " << _name << ":";
 		json_spirit::mValue v;
 		string s = asString(dev::contents(testPath + "/" + _name + ".json"));
 		TBOOST_REQUIRE_MESSAGE((s.length() > 0), "Contents of " + testPath + "/" + _name + ".json is empty. Have you cloned the 'tests' repo branch develop and set ETHEREUM_TEST_PATH to its path?");
@@ -773,7 +787,26 @@ Options::Options()
 		}
 		else if (arg == "--fulloutput")
 			fulloutput = true;
+		else if (arg == "--verbosity" && i + 1 < argc)
+		{
+			static std::ostringstream strCout; //static string to redirect logs to
+			std::string indentLevel = std::string{argv[i + 1]};
+			if (indentLevel == "0")
+			{
+				logVerbosity = Verbosity::None;
+				std::cout.rdbuf(strCout.rdbuf());
+				std::cerr.rdbuf(strCout.rdbuf());
+			}
+			else if (indentLevel == "1")
+				logVerbosity = Verbosity::NiceReport;
+			else
+				logVerbosity = Verbosity::Full;
+		}
 	}
+
+	//Default option
+	if (logVerbosity == Verbosity::NiceReport)
+		g_logVerbosity = -1;	//disable cnote but leave cerr and cout
 }
 
 Options const& Options::get()

TestHelper.h

@@ -22,7 +22,6 @@
 #pragma once
 
 #include <functional>
-
 #include <boost/test/unit_test.hpp>
 #include <boost/filesystem.hpp>
 
@@ -143,8 +142,8 @@ public:
 	static json_spirit::mObject& makeAllFieldsHex(json_spirit::mObject& _o);
 
 	bytes executeTest();
-	void exportTest(bytes const& _output);
-	static void compareStates(eth::State const& _stateExpect, eth::State const& _statePost, eth::AccountMaskMap const _expectedStateOptions = eth::AccountMaskMap(), WhenError _throw = WhenError::Throw);
+	int exportTest(bytes const& _output);
+	static int compareStates(eth::State const& _stateExpect, eth::State const& _statePost, eth::AccountMaskMap const _expectedStateOptions = eth::AccountMaskMap(), WhenError _throw = WhenError::Throw);
 
 	eth::State m_statePre;
 	eth::State m_statePost;
@@ -218,6 +217,13 @@ void checkAddresses(mapType& _expectedAddrs, mapType& _resultAddrs)
 	TBOOST_CHECK((_expectedAddrs == _resultAddrs));
 }*/
 
+enum class Verbosity
+{
+	Full,
+	NiceReport,
+	None
+};
+
 class Options
 {
 public:
@@ -227,6 +233,7 @@ public:
 	std::string statsOutFile; ///< Stats output file. "out" for standard output
 	bool checkState = false;///< Throw error when checking test states
 	bool fulloutput = false;///< Replace large output to just it's length
+	Verbosity logVerbosity = Verbosity::NiceReport;
 
 	/// Test selection
 	/// @{

boostTest.cpp

@@ -18,11 +18,75 @@
  * @author Marko Simovic <markobarko@gmail.com>
  * @date 2014
  * Stub for generating main boost.test module.
+ * Original code taken from boost sources.
  */
 
 #define BOOST_TEST_MODULE EthereumTests
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wunused-parameter"
-#define BOOST_DISABLE_WIN32 //disables SEH warning
+//#define BOOST_DISABLE_WIN32 //disables SEH warning
+#define BOOST_TEST_NO_MAIN
 #include <boost/test/included/unit_test.hpp>
 #pragma GCC diagnostic pop
+
+#include <test/TestHelper.h>
+using namespace boost::unit_test;
+
+//Custom Boost Initialization
+test_suite* init_func( int argc, char* argv[] )
+{
+	if (argc == 0)
+		argv[1]=(char*)"a";
+
+	dev::test::Options::get();
+
+	return 0;
+}
+
+//Custom Boost Unit Test Main
+int main( int argc, char* argv[] )
+{
+	try
+	{
+		framework::init( init_func, argc, argv );
+
+		if( !runtime_config::test_to_run().is_empty() )
+		{
+			test_case_filter filter( runtime_config::test_to_run() );
+
+			traverse_test_tree( framework::master_test_suite().p_id, filter );
+		}
+
+		framework::run();
+
+		results_reporter::make_report();
+
+		return runtime_config::no_result_code()
+					? boost::exit_success
+					: results_collector.results( framework::master_test_suite().p_id ).result_code();
+	}
+	catch (framework::nothing_to_test const&)
+	{
+		return boost::exit_success;
+	}
+	catch (framework::internal_error const& ex)
+	{
+		results_reporter::get_stream() << "Boost.Test framework internal error: " << ex.what() << std::endl;
+
+		return boost::exit_exception_failure;
+	}
+	catch (framework::setup_error const& ex)
+	{
+		results_reporter::get_stream() << "Test setup error: " << ex.what() << std::endl;
+
+		return boost::exit_exception_failure;
+	}
+	catch (...)
+	{
+		results_reporter::get_stream() << "Boost.Test framework internal error: unknown reason" << std::endl;
+
+		return boost::exit_exception_failure;
+	}
+
+	return 0;
+}

libsolidity/solidityExecutionFramework.h

@@ -44,7 +44,9 @@ class ExecutionFramework
 public:
 	ExecutionFramework()
 	{
-		g_logVerbosity = 0;
+		if (g_logVerbosity != -1)
+			g_logVerbosity = 0;
+		//m_state.resetCurrent();
 	}
 
 	bytes const& compileAndRunWithoutCheck(