diff --git a/TestHelper.cpp b/TestHelper.cpp index 9ac64107d..fcca46308 100644 --- a/TestHelper.cpp +++ b/TestHelper.cpp @@ -439,8 +439,6 @@ void userDefinedTest(string testTypeFlag, std::function 0, "Contents of " + dir.string() + "/" + _name + "Filler.json is empty."); - json_spirit::read_string(s, v); - doTests(v, true); - writeFile(testPath + "/" + _name + ".json", asBytes(json_spirit::write_string(v, true))); - } - catch (Exception const& _e) - { - BOOST_ERROR("Failed filling test with Exception: " << diagnostic_information(_e)); - } - catch (std::exception const& _e) - { - BOOST_ERROR("Failed filling test with Exception: " << _e.what()); - } - break; + cnote << "Populating tests..."; + json_spirit::mValue v; + boost::filesystem::path p(__FILE__); + boost::filesystem::path dir = p.parent_path(); + string s = asString(dev::contents(dir.string() + "/" + _name + "Filler.json")); + BOOST_REQUIRE_MESSAGE(s.length() > 0, "Contents of " + dir.string() + "/" + _name + "Filler.json is empty."); + json_spirit::read_string(s, v); + doTests(v, true); + writeFile(testPath + "/" + _name + ".json", asBytes(json_spirit::write_string(v, true))); + } + catch (Exception const& _e) + { + BOOST_ERROR("Failed filling test with Exception: " << diagnostic_information(_e)); + } + catch (std::exception const& _e) + { + BOOST_ERROR("Failed filling test with Exception: " << _e.what()); } } @@ -541,21 +534,48 @@ RLPStream createRLPStreamFromTransactionFields(json_spirit::mObject& _tObj) return rlpStream; } -void processCommandLineOptions() +Options::Options() { auto argc = boost::unit_test::framework::master_test_suite().argc; auto argv = boost::unit_test::framework::master_test_suite().argv; - for (auto i = 0; i < argc; ++i) + for (auto i = 0; i < argc; ++i) { - if (std::string(argv[i]) == "--jit") + auto arg = std::string{argv[i]}; + if (arg == "--jit") { + jit = true; eth::VMFactory::setKind(eth::VMKind::JIT); - break; + } + else if (arg == "--vmtrace") + vmtrace = true; + else if (arg == "--performance") + performance = true; + else if (arg == "--quadratic") + quadratic = true; + else if (arg == "--memory") + memory = true; + else if (arg == "--inputlimits") + inputLimits = true; + else if (arg == "--bigdata") + bigData = true; + else if (arg == "--all") + { + performance = true; + quadratic = true; + memory = true; + inputLimits = true; + bigData = true; } } } +Options const& Options::get() +{ + static Options instance; + return instance; +} + LastHashes lastHashes(u256 _currentBlockNumber) { LastHashes ret; diff --git a/TestHelper.h b/TestHelper.h index d378848a2..9efed0fa6 100644 --- a/TestHelper.h +++ b/TestHelper.h @@ -141,7 +141,6 @@ void executeTests(const std::string& _name, const std::string& _testPathAppendix std::string getTestPath(); void userDefinedTest(std::string testTypeFlag, std::function doTests); RLPStream createRLPStreamFromTransactionFields(json_spirit::mObject& _tObj); -void processCommandLineOptions(); eth::LastHashes lastHashes(u256 _currentBlockNumber); json_spirit::mObject fillJsonWithState(eth::State _state); @@ -158,5 +157,31 @@ void checkAddresses(mapType& _expectedAddrs, mapType& _resultAddrs) BOOST_CHECK(_expectedAddrs == _resultAddrs); } +class Options +{ +public: + bool jit = false; ///< Use JIT + bool vmtrace = false; ///< Create EVM execution tracer // TODO: Link with log verbosity? + bool showTimes = false; ///< Print test groups execution times + bool fillTests = false; ///< Create JSON test files from execution results + + /// Test selection + /// @{ + bool performance = false; + bool quadratic = false; + bool memory = false; + bool inputLimits = false; + bool bigData = false; + /// @} + + /// Get reference to options + /// The first time used, options are parsed + static Options const& get(); + +private: + Options(); + Options(Options const&) = delete; +}; + } } diff --git a/state.cpp b/state.cpp index 5202aff22..7fd51d409 100644 --- a/state.cpp +++ b/state.cpp @@ -41,7 +41,7 @@ namespace dev { namespace test { void doStateTests(json_spirit::mValue& v, bool _fillin) { - processCommandLineOptions(); + Options::get(); // process command line options for (auto& i: v.get_obj()) { @@ -172,48 +172,40 @@ BOOST_AUTO_TEST_CASE(stBlockHashTest) BOOST_AUTO_TEST_CASE(stQuadraticComplexityTest) { - for (int i = 1; i < boost::unit_test::framework::master_test_suite().argc; ++i) - { - string arg = boost::unit_test::framework::master_test_suite().argv[i]; - if (arg == "--quadratic" || arg == "--all") - { - auto start = chrono::steady_clock::now(); + if (test::Options::get().quadratic) + { + auto start = chrono::steady_clock::now(); - dev::test::executeTests("stQuadraticComplexityTest", "/StateTests", dev::test::doStateTests); + dev::test::executeTests("stQuadraticComplexityTest", "/StateTests", dev::test::doStateTests); - auto end = chrono::steady_clock::now(); - auto duration(chrono::duration_cast(end - start)); - cnote << "test duration: " << duration.count() << " milliseconds.\n"; - } - } + auto end = chrono::steady_clock::now(); + auto duration(chrono::duration_cast(end - start)); + cnote << "test duration: " << duration.count() << " milliseconds.\n"; + } } BOOST_AUTO_TEST_CASE(stMemoryStressTest) { - for (int i = 1; i < boost::unit_test::framework::master_test_suite().argc; ++i) - { - string arg = boost::unit_test::framework::master_test_suite().argv[i]; - if (arg == "--memory" || arg == "--all") - { - auto start = chrono::steady_clock::now(); + if (test::Options::get().memory) + { + auto start = chrono::steady_clock::now(); - dev::test::executeTests("stMemoryStressTest", "/StateTests", dev::test::doStateTests); + dev::test::executeTests("stMemoryStressTest", "/StateTests", dev::test::doStateTests); - auto end = chrono::steady_clock::now(); - auto duration(chrono::duration_cast(end - start)); - cnote << "test duration: " << duration.count() << " milliseconds.\n"; - } - } + auto end = chrono::steady_clock::now(); + auto duration(chrono::duration_cast(end - start)); + cnote << "test duration: " << duration.count() << " milliseconds.\n"; + } } - BOOST_AUTO_TEST_CASE(stSolidityTest) - { - dev::test::executeTests("stSolidityTest", "/StateTests", dev::test::doStateTests); - } +BOOST_AUTO_TEST_CASE(stSolidityTest) +{ + dev::test::executeTests("stSolidityTest", "/StateTests", dev::test::doStateTests); +} BOOST_AUTO_TEST_CASE(stMemoryTest) { - dev::test::executeTests("stMemoryTest", "/StateTests", dev::test::doStateTests); + dev::test::executeTests("stMemoryTest", "/StateTests", dev::test::doStateTests); } diff --git a/transaction.cpp b/transaction.cpp index 83fd8da4d..77f6ecdaf 100644 --- a/transaction.cpp +++ b/transaction.cpp @@ -116,21 +116,16 @@ BOOST_AUTO_TEST_CASE(ttWrongRLPTransaction) BOOST_AUTO_TEST_CASE(tt10mbDataField) { - for (int i = 1; i < boost::unit_test::framework::master_test_suite().argc; ++i) + if (test::Options::get().bigData) { - string arg = boost::unit_test::framework::master_test_suite().argv[i]; - if (arg == "--bigdata" || arg == "--all") - { - auto start = chrono::steady_clock::now(); + auto start = chrono::steady_clock::now(); - dev::test::executeTests("tt10mbDataField", "/TransactionTests", dev::test::doTransactionTests); + dev::test::executeTests("tt10mbDataField", "/TransactionTests", dev::test::doTransactionTests); - auto end = chrono::steady_clock::now(); - auto duration(chrono::duration_cast(end - start)); - cnote << "test duration: " << duration.count() << " milliseconds.\n"; - } + auto end = chrono::steady_clock::now(); + auto duration(chrono::duration_cast(end - start)); + cnote << "test duration: " << duration.count() << " milliseconds.\n"; } - } BOOST_AUTO_TEST_CASE(ttCreateTest) diff --git a/vm.cpp b/vm.cpp index a39f0005f..4433a60ee 100644 --- a/vm.cpp +++ b/vm.cpp @@ -312,7 +312,7 @@ namespace dev { namespace test { void doVMTests(json_spirit::mValue& v, bool _fillin) { - processCommandLineOptions(); + Options::get(); // process command line options // TODO: We need to control the main() function for (auto& i: v.get_obj()) { @@ -344,7 +344,8 @@ void doVMTests(json_spirit::mValue& v, bool _fillin) try { auto vm = eth::VMFactory::create(fev.gas); - output = vm->go(fev, fev.simpleTrace()).toBytes(); + auto vmtrace = Options::get().vmtrace ? fev.simpleTrace() : OnOpFunc{}; + output = vm->go(fev, vmtrace).toBytes(); gas = vm->gas(); } catch (VMException const&) @@ -364,18 +365,12 @@ void doVMTests(json_spirit::mValue& v, bool _fillin) } auto endTime = std::chrono::high_resolution_clock::now(); - auto argc = boost::unit_test::framework::master_test_suite().argc; - auto argv = boost::unit_test::framework::master_test_suite().argv; - for (auto i = 0; i < argc; ++i) + if (Options::get().showTimes) { - if (std::string(argv[i]) == "--show-times") - { - auto testDuration = endTime - startTime; - cnote << "Execution time: " - << std::chrono::duration_cast(testDuration).count() - << " ms"; - break; - } + auto testDuration = endTime - startTime; + cnote << "Execution time: " + << std::chrono::duration_cast(testDuration).count() + << " ms"; } // delete null entries in storage for the sake of comparison @@ -517,58 +512,42 @@ BOOST_AUTO_TEST_CASE(vmSystemOperationsTest) BOOST_AUTO_TEST_CASE(vmPerformanceTest) { - for (int i = 1; i < boost::unit_test::framework::master_test_suite().argc; ++i) + if (test::Options::get().performance) { - string arg = boost::unit_test::framework::master_test_suite().argv[i]; - if (arg == "--performance" || arg == "--all") - { - auto start = chrono::steady_clock::now(); + auto start = chrono::steady_clock::now(); - dev::test::executeTests("vmPerformanceTest", "/VMTests", dev::test::doVMTests); + dev::test::executeTests("vmPerformanceTest", "/VMTests", dev::test::doVMTests); - auto end = chrono::steady_clock::now(); - auto duration(chrono::duration_cast(end - start)); - cnote << "test duration: " << duration.count() << " milliseconds.\n"; - } + auto end = chrono::steady_clock::now(); + auto duration(chrono::duration_cast(end - start)); + cnote << "test duration: " << duration.count() << " milliseconds.\n"; } } BOOST_AUTO_TEST_CASE(vmInputLimitsTest1) { - for (int i = 1; i < boost::unit_test::framework::master_test_suite().argc; ++i) + if (test::Options::get().inputLimits) { - string arg = boost::unit_test::framework::master_test_suite().argv[i]; - if (arg == "--inputlimits" || arg == "--all") - { - auto start = chrono::steady_clock::now(); + auto start = chrono::steady_clock::now(); - dev::test::executeTests("vmInputLimits1", "/VMTests", dev::test::doVMTests); + dev::test::executeTests("vmInputLimits1", "/VMTests", dev::test::doVMTests); - auto end = chrono::steady_clock::now(); - auto duration(chrono::duration_cast(end - start)); - cnote << "test duration: " << duration.count() << " milliseconds.\n"; - } + auto end = chrono::steady_clock::now(); + auto duration(chrono::duration_cast(end - start)); + cnote << "test duration: " << duration.count() << " milliseconds.\n"; } } BOOST_AUTO_TEST_CASE(vmInputLimitsTest2) { - for (int i = 1; i < boost::unit_test::framework::master_test_suite().argc; ++i) - { - string arg = boost::unit_test::framework::master_test_suite().argv[i]; - if (arg == "--inputlimits" || arg == "--all") - dev::test::executeTests("vmInputLimits2", "/VMTests", dev::test::doVMTests); - } + if (test::Options::get().inputLimits) + dev::test::executeTests("vmInputLimits2", "/VMTests", dev::test::doVMTests); } BOOST_AUTO_TEST_CASE(vmInputLimitsLightTest) { - for (int i = 1; i < boost::unit_test::framework::master_test_suite().argc; ++i) - { - string arg = boost::unit_test::framework::master_test_suite().argv[i]; - if (arg == "--inputlimits" || arg == "--all") - dev::test::executeTests("vmInputLimitsLight", "/VMTests", dev::test::doVMTests); - } + if (test::Options::get().inputLimits) + dev::test::executeTests("vmInputLimitsLight", "/VMTests", dev::test::doVMTests); } BOOST_AUTO_TEST_CASE(vmRandom)