From 2fb9eafc9d0b6cf01a8e7e64ab13f06433c5920f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Mon, 27 Jul 2015 14:25:28 +0200 Subject: [PATCH 01/10] Test dynamic jumps. From e9e6bc67804f952e74dca73c0ab7839737df5757 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Mon, 27 Jul 2015 18:48:58 +0200 Subject: [PATCH 02/10] Add test for global stack underflow. From b12507c55e2ab3cb2d709f998d9229546a345238 Mon Sep 17 00:00:00 2001 From: Dimitry Date: Mon, 27 Jul 2015 21:14:55 +0300 Subject: [PATCH 03/10] Nice Error Report --- TestHelper.cpp | 26 +++++++++++++++++++++++++- TestHelper.h | 8 ++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/TestHelper.cpp b/TestHelper.cpp index 4a3de4dc6..094521a68 100644 --- a/TestHelper.cpp +++ b/TestHelper.cpp @@ -258,7 +258,11 @@ void ImportTest::compareStates(State const& _stateExpect, State const& _statePos #define CHECK(a,b) \ { \ if (_throw == WhenError::Throw) \ - {TBOOST_CHECK_MESSAGE(a,b);}\ + { \ + TBOOST_CHECK(a); \ + if (!a) \ + std::cerr << b << std::endl;\ + } \ else \ {TBOOST_WARN_MESSAGE(a,b);} \ } @@ -773,7 +777,27 @@ Options::Options() } else if (arg == "--fulloutput") fulloutput = true; + else if (arg == "--verbosity" && i + 1 < argc) + { + static std::ostringstream strCout; + std::string depthLevel = std::string{argv[i + 1]}; + if (depthLevel == "0") + { + logVerbosity = Verbosity::None; + std::cout.rdbuf( strCout.rdbuf() ); + std::cerr.rdbuf( strCout.rdbuf() ); + } + else + if (depthLevel == "1") + logVerbosity = Verbosity::NiceReport; + else + logVerbosity = Verbosity::Full; + } } + + //Default option + if (logVerbosity == Verbosity::NiceReport) + g_logVerbosity = -1; //disable cnote but not the cerr } Options const& Options::get() diff --git a/TestHelper.h b/TestHelper.h index 5f2f8d6d5..a5f50f807 100644 --- a/TestHelper.h +++ b/TestHelper.h @@ -218,6 +218,13 @@ void checkAddresses(mapType& _expectedAddrs, mapType& _resultAddrs) TBOOST_CHECK((_expectedAddrs == _resultAddrs)); }*/ +enum class Verbosity +{ + Full, + NiceReport, + None +}; + class Options { public: @@ -227,6 +234,7 @@ public: std::string statsOutFile; ///< Stats output file. "out" for standard output bool checkState = false;///< Throw error when checking test states bool fulloutput = false;///< Replace large output to just it's length + Verbosity logVerbosity = Verbosity::NiceReport; /// Test selection /// @{ From d6d8951baf6f36e3d73727a9e6a24860c659caf8 Mon Sep 17 00:00:00 2001 From: Dimitry Date: Tue, 28 Jul 2015 23:32:11 +0300 Subject: [PATCH 04/10] Error logs --- TestHelper.cpp | 44 +++++++++++------- TestHelper.h | 5 +-- boostTest.cpp | 57 ++++++++++++++++++++++++ libsolidity/solidityExecutionFramework.h | 4 +- 4 files changed, 89 insertions(+), 21 deletions(-) diff --git a/TestHelper.cpp b/TestHelper.cpp index 094521a68..80b4214cf 100644 --- a/TestHelper.cpp +++ b/TestHelper.cpp @@ -253,15 +253,15 @@ void ImportTest::importTransaction(json_spirit::mObject const& o_tr) importTransaction(o_tr, m_transaction); } -void ImportTest::compareStates(State const& _stateExpect, State const& _statePost, AccountMaskMap const _expectedStateOptions, WhenError _throw) +int ImportTest::compareStates(State const& _stateExpect, State const& _statePost, AccountMaskMap const _expectedStateOptions, WhenError _throw) { #define CHECK(a,b) \ { \ if (_throw == WhenError::Throw) \ { \ - TBOOST_CHECK(a); \ + TBOOST_CHECK_MESSAGE(a, b); \ if (!a) \ - std::cerr << b << std::endl;\ + return 1; \ } \ else \ {TBOOST_WARN_MESSAGE(a,b);} \ @@ -313,20 +313,26 @@ void ImportTest::compareStates(State const& _stateExpect, State const& _statePos "Check State: " << a.first << ": incorrect code '" << toHex(_statePost.code(a.first)) << "', expected '" << toHex(_stateExpect.code(a.first)) << "'"); } } + return 0; } -void ImportTest::exportTest(bytes const& _output) +int ImportTest::exportTest(bytes const& _output) { - // export output - - m_testObject["out"] = (_output.size() > 4096 && !Options::get().fulloutput) ? "#" + toString(_output.size()) : toHex(_output, 2, HexPrefix::Add); + int err = 0; + // export output + m_TestObject["out"] = (_output.size() > 4096 && !Options::get().fulloutput) ? "#" + toString(_output.size()) : toHex(_output, 2, HexPrefix::Add); // compare expected output with post output if (m_testObject.count("expectOut") > 0) { std::string warning = "Check State: Error! Unexpected output: " + m_testObject["out"].get_str() + " Expected: " + m_testObject["expectOut"].get_str(); if (Options::get().checkState) - {TBOOST_CHECK_MESSAGE((m_testObject["out"].get_str() == m_testObject["expectOut"].get_str()), warning);} + { + bool statement = (m_testObject["out"].get_str() == m_testObject["expectOut"].get_str()); + TBOOST_CHECK_MESSAGE(statement, warning); + if (!statement) + err = 1; + } else TBOOST_WARN_MESSAGE((m_testObject["out"].get_str() == m_testObject["expectOut"].get_str()), warning); @@ -354,6 +360,7 @@ void ImportTest::exportTest(bytes const& _output) m_testObject["pre"] = fillJsonWithState(m_statePre); m_testObject["env"] = makeAllFieldsHex(m_testObject["env"].get_obj()); m_testObject["transaction"] = makeAllFieldsHex(m_testObject["transaction"].get_obj()); + return err; } json_spirit::mObject fillJsonWithTransaction(Transaction _txn) @@ -570,7 +577,10 @@ void userDefinedTest(std::function doTests) auto& filename = Options::get().singleTestFile; auto& testname = Options::get().singleTestName; - VerbosityHolder sentinel(12); + + if (g_logVerbosity != -1) + VerbosityHolder sentinel(12); + try { cnote << "Testing user defined test: " << filename; @@ -635,7 +645,7 @@ void executeTests(const string& _name, const string& _testPathAppendix, const bo try { - std::cout << "TEST " << _name << ":\n"; + cnote << "TEST " << _name << ":"; json_spirit::mValue v; string s = asString(dev::contents(testPath + "/" + _name + ".json")); TBOOST_REQUIRE_MESSAGE((s.length() > 0), "Contents of " + testPath + "/" + _name + ".json is empty. Have you cloned the 'tests' repo branch develop and set ETHEREUM_TEST_PATH to its path?"); @@ -779,16 +789,16 @@ Options::Options() fulloutput = true; else if (arg == "--verbosity" && i + 1 < argc) { - static std::ostringstream strCout; - std::string depthLevel = std::string{argv[i + 1]}; - if (depthLevel == "0") + static std::ostringstream strCout; //static string to redirect logs to + std::string indentLevel = std::string{argv[i + 1]}; + if (indentLevel == "0") { logVerbosity = Verbosity::None; - std::cout.rdbuf( strCout.rdbuf() ); - std::cerr.rdbuf( strCout.rdbuf() ); + std::cout.rdbuf(strCout.rdbuf()); + std::cerr.rdbuf(strCout.rdbuf()); } else - if (depthLevel == "1") + if (indentLevel == "1") logVerbosity = Verbosity::NiceReport; else logVerbosity = Verbosity::Full; @@ -797,7 +807,7 @@ Options::Options() //Default option if (logVerbosity == Verbosity::NiceReport) - g_logVerbosity = -1; //disable cnote but not the cerr + g_logVerbosity = -1; //disable cnote but leave cerr and cout } Options const& Options::get() diff --git a/TestHelper.h b/TestHelper.h index a5f50f807..e5cf13233 100644 --- a/TestHelper.h +++ b/TestHelper.h @@ -22,7 +22,6 @@ #pragma once #include - #include #include @@ -143,8 +142,8 @@ public: static json_spirit::mObject& makeAllFieldsHex(json_spirit::mObject& _o); bytes executeTest(); - void exportTest(bytes const& _output); - static void compareStates(eth::State const& _stateExpect, eth::State const& _statePost, eth::AccountMaskMap const _expectedStateOptions = eth::AccountMaskMap(), WhenError _throw = WhenError::Throw); + int exportTest(bytes const& _output); + static int compareStates(eth::State const& _stateExpect, eth::State const& _statePost, eth::AccountMaskMap const _expectedStateOptions = eth::AccountMaskMap(), WhenError _throw = WhenError::Throw); eth::State m_statePre; eth::State m_statePost; diff --git a/boostTest.cpp b/boostTest.cpp index 1523a7a11..af546c3a3 100644 --- a/boostTest.cpp +++ b/boostTest.cpp @@ -24,5 +24,62 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" #define BOOST_DISABLE_WIN32 //disables SEH warning +#define BOOST_TEST_NO_MAIN #include #pragma GCC diagnostic pop + +#include +using namespace boost::unit_test; + +//Custom Boost Initialization +test_suite* init_func( int argc, char* argv[] ) +{ + if (argc == 0) + argv[1]=(char*)"a"; + + dev::test::Options::get(); + + return 0; +} + +//Custom Boost Unit Test Main +int main( int argc, char* argv[] ) +{ + try { + framework::init( init_func, argc, argv ); + + if( !runtime_config::test_to_run().is_empty() ) { + test_case_filter filter( runtime_config::test_to_run() ); + + traverse_test_tree( framework::master_test_suite().p_id, filter ); + } + + framework::run(); + + results_reporter::make_report(); + + return runtime_config::no_result_code() + ? boost::exit_success + : results_collector.results( framework::master_test_suite().p_id ).result_code(); + } + catch( framework::nothing_to_test const& ) { + return boost::exit_success; + } + catch( framework::internal_error const& ex ) { + results_reporter::get_stream() << "Boost.Test framework internal error: " << ex.what() << std::endl; + + return boost::exit_exception_failure; + } + catch( framework::setup_error const& ex ) { + results_reporter::get_stream() << "Test setup error: " << ex.what() << std::endl; + + return boost::exit_exception_failure; + } + catch( ... ) { + results_reporter::get_stream() << "Boost.Test framework internal error: unknown reason" << std::endl; + + return boost::exit_exception_failure; + } + + return 0; +} diff --git a/libsolidity/solidityExecutionFramework.h b/libsolidity/solidityExecutionFramework.h index 98241b2ff..aac033b50 100644 --- a/libsolidity/solidityExecutionFramework.h +++ b/libsolidity/solidityExecutionFramework.h @@ -44,7 +44,9 @@ class ExecutionFramework public: ExecutionFramework() { - g_logVerbosity = 0; + if (g_logVerbosity != -1) + g_logVerbosity = 0; + m_state.resetCurrent(); } bytes const& compileAndRunWithoutCheck( From c0aa9f67267f7eb151b4e8322d313f846cffc244 Mon Sep 17 00:00:00 2001 From: Dimitry Date: Tue, 11 Aug 2015 14:30:35 +0300 Subject: [PATCH 05/10] error report: merge with latesd develop --- TestHelper.cpp | 2 +- libsolidity/solidityExecutionFramework.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/TestHelper.cpp b/TestHelper.cpp index 80b4214cf..3869613c9 100644 --- a/TestHelper.cpp +++ b/TestHelper.cpp @@ -320,7 +320,7 @@ int ImportTest::exportTest(bytes const& _output) { int err = 0; // export output - m_TestObject["out"] = (_output.size() > 4096 && !Options::get().fulloutput) ? "#" + toString(_output.size()) : toHex(_output, 2, HexPrefix::Add); + m_testObject["out"] = (_output.size() > 4096 && !Options::get().fulloutput) ? "#" + toString(_output.size()) : toHex(_output, 2, HexPrefix::Add); // compare expected output with post output if (m_testObject.count("expectOut") > 0) diff --git a/libsolidity/solidityExecutionFramework.h b/libsolidity/solidityExecutionFramework.h index aac033b50..05f93bb60 100644 --- a/libsolidity/solidityExecutionFramework.h +++ b/libsolidity/solidityExecutionFramework.h @@ -46,7 +46,7 @@ public: { if (g_logVerbosity != -1) g_logVerbosity = 0; - m_state.resetCurrent(); + //m_state.resetCurrent(); } bytes const& compileAndRunWithoutCheck( From cbbf99da648a588c0044480908532d4c3f0014a9 Mon Sep 17 00:00:00 2001 From: Dimitry Date: Wed, 12 Aug 2015 11:12:12 +0300 Subject: [PATCH 06/10] nice error report merge From 4de035e5badf8bf4658b39c27813e042afb87f9e Mon Sep 17 00:00:00 2001 From: Dimitry Date: Wed, 12 Aug 2015 18:53:46 +0300 Subject: [PATCH 07/10] boost issue --- boostTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boostTest.cpp b/boostTest.cpp index af546c3a3..79ed27abc 100644 --- a/boostTest.cpp +++ b/boostTest.cpp @@ -23,7 +23,7 @@ #define BOOST_TEST_MODULE EthereumTests #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" -#define BOOST_DISABLE_WIN32 //disables SEH warning +//#define BOOST_DISABLE_WIN32 //disables SEH warning #define BOOST_TEST_NO_MAIN #include #pragma GCC diagnostic pop From 5cfacc2e7af9219064d2e3a306d4b74156d8e390 Mon Sep 17 00:00:00 2001 From: Dimitry Date: Thu, 13 Aug 2015 12:18:29 +0300 Subject: [PATCH 08/10] Error report: style issues --- TestHelper.cpp | 5 ++--- boostTest.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/TestHelper.cpp b/TestHelper.cpp index 3869613c9..15a5d5633 100644 --- a/TestHelper.cpp +++ b/TestHelper.cpp @@ -319,7 +319,7 @@ int ImportTest::compareStates(State const& _stateExpect, State const& _statePost int ImportTest::exportTest(bytes const& _output) { int err = 0; - // export output + // export output m_testObject["out"] = (_output.size() > 4096 && !Options::get().fulloutput) ? "#" + toString(_output.size()) : toHex(_output, 2, HexPrefix::Add); // compare expected output with post output @@ -797,8 +797,7 @@ Options::Options() std::cout.rdbuf(strCout.rdbuf()); std::cerr.rdbuf(strCout.rdbuf()); } - else - if (indentLevel == "1") + else if (indentLevel == "1") logVerbosity = Verbosity::NiceReport; else logVerbosity = Verbosity::Full; diff --git a/boostTest.cpp b/boostTest.cpp index 79ed27abc..824a8e8e5 100644 --- a/boostTest.cpp +++ b/boostTest.cpp @@ -62,20 +62,20 @@ int main( int argc, char* argv[] ) ? boost::exit_success : results_collector.results( framework::master_test_suite().p_id ).result_code(); } - catch( framework::nothing_to_test const& ) { + catch (framework::nothing_to_test const&) { return boost::exit_success; } - catch( framework::internal_error const& ex ) { + catch (framework::internal_error const& ex) { results_reporter::get_stream() << "Boost.Test framework internal error: " << ex.what() << std::endl; return boost::exit_exception_failure; } - catch( framework::setup_error const& ex ) { + catch (framework::setup_error const& ex) { results_reporter::get_stream() << "Test setup error: " << ex.what() << std::endl; return boost::exit_exception_failure; } - catch( ... ) { + catch (...) { results_reporter::get_stream() << "Boost.Test framework internal error: unknown reason" << std::endl; return boost::exit_exception_failure; From 30f37815296f61d2c236b493b3e66870cdaccf56 Mon Sep 17 00:00:00 2001 From: Dimitry Date: Thu, 13 Aug 2015 13:27:47 +0300 Subject: [PATCH 09/10] error report: style --- boostTest.cpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/boostTest.cpp b/boostTest.cpp index 824a8e8e5..f448c48e3 100644 --- a/boostTest.cpp +++ b/boostTest.cpp @@ -18,6 +18,7 @@ * @author Marko Simovic * @date 2014 * Stub for generating main boost.test module. + * Original code taken from boost sources. */ #define BOOST_TEST_MODULE EthereumTests @@ -45,10 +46,12 @@ test_suite* init_func( int argc, char* argv[] ) //Custom Boost Unit Test Main int main( int argc, char* argv[] ) { - try { + try + { framework::init( init_func, argc, argv ); - if( !runtime_config::test_to_run().is_empty() ) { + if( !runtime_config::test_to_run().is_empty() ) + { test_case_filter filter( runtime_config::test_to_run() ); traverse_test_tree( framework::master_test_suite().p_id, filter ); @@ -62,20 +65,24 @@ int main( int argc, char* argv[] ) ? boost::exit_success : results_collector.results( framework::master_test_suite().p_id ).result_code(); } - catch (framework::nothing_to_test const&) { + catch (framework::nothing_to_test const&) + { return boost::exit_success; } - catch (framework::internal_error const& ex) { + catch (framework::internal_error const& ex) + { results_reporter::get_stream() << "Boost.Test framework internal error: " << ex.what() << std::endl; return boost::exit_exception_failure; } - catch (framework::setup_error const& ex) { + catch (framework::setup_error const& ex) + { results_reporter::get_stream() << "Test setup error: " << ex.what() << std::endl; return boost::exit_exception_failure; } - catch (...) { + catch (...) + { results_reporter::get_stream() << "Boost.Test framework internal error: unknown reason" << std::endl; return boost::exit_exception_failure; From 1e137c41f9423c31c5bb686d60eaaf8b60901581 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 13 Aug 2015 13:13:59 +0200 Subject: [PATCH 10/10] NameReg integration. Minor other alterations.