diff --git a/.circleci/config.yml b/.circleci/config.yml index ac23af569..e28725192 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -719,6 +719,9 @@ jobs: EVM: constantinople OPTIMIZE: 0 TERM: xterm + # For Archlinux we do not have prebuilt docker images and we would need to build evmone from source, + # thus we forgo semantics tests to speed things up. + SOLTEST_FLAGS: --no-semantic-tests steps: - run: name: Install runtime dependencies diff --git a/test/Common.cpp b/test/Common.cpp index df836288a..9b2cfbb96 100644 --- a/test/Common.cpp +++ b/test/Common.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -62,15 +63,10 @@ boost::filesystem::path testPath() return {}; } -std::string envOrDefaultPath(std::string const& env_name, std::string const& lib_name) +std::optional findInDefaultPath(std::string const& lib_name) { - if (auto path = getenv(env_name.c_str())) - return path; - auto const searchPath = { - fs::path("/usr/local/lib"), - fs::path("/usr/lib"), fs::current_path() / "deps", fs::current_path() / "deps" / "lib", fs::current_path() / ".." / "deps", @@ -83,9 +79,9 @@ std::string envOrDefaultPath(std::string const& env_name, std::string const& lib { fs::path p = basePath / lib_name; if (fs::exists(p)) - return p.string(); + return p; } - return {}; + return std::nullopt; } } @@ -101,6 +97,7 @@ CommonOptions::CommonOptions(std::string _caption): ("testpath", po::value(&this->testPath)->default_value(solidity::test::testPath()), "path to test files") ("vm", po::value>(&vmPaths), "path to evmc library, can be supplied multiple times.") ("ewasm", po::bool_switch(&ewasm), "tries to automatically find an ewasm vm and enable ewasm test-execution.") + ("no-semantic-tests", po::bool_switch(&disableSemanticTests), "disable semantic tests") ("no-smt", po::bool_switch(&disableSMT), "disable SMT checker") ("optimize", po::bool_switch(&optimize), "enables optimization") ("enforce-via-yul", po::bool_switch(&enforceViaYul), "Enforce compiling all tests via yul to see if additional tests can be activated.") @@ -166,28 +163,19 @@ bool CommonOptions::parse(int argc, char const* const* argv) if (vmPaths.empty()) { - std::string evmone = envOrDefaultPath("ETH_EVMONE", evmoneFilename); - if (!evmone.empty()) - vmPaths.emplace_back(evmone); + if (auto envPath = getenv("ETH_EVMONE")) + vmPaths.emplace_back(envPath); + else if (auto repoPath = findInDefaultPath(evmoneFilename)) + vmPaths.emplace_back(*repoPath); else - { - std::cout << "Unable to find " << solidity::test::evmoneFilename - << ". Please provide the path using --vm ." << std::endl; - std::cout << "You can download it at" << std::endl; - std::cout << solidity::test::evmoneDownloadLink << std::endl; - } - } - - if (ewasm) { - std::string hera = envOrDefaultPath("ETH_HERA", heraFilename); - if (!hera.empty()) - vmPaths.emplace_back(hera); - else { - std::cout << "Unable to find " << solidity::test::heraFilename - << ". Please provide the path using --vm ." << std::endl; - std::cout << "You can download it at" << std::endl; - std::cout << solidity::test::heraDownloadLink << std::endl; - std::cout << "Ewasm tests disabled." << std::endl; + vmPaths.emplace_back(evmoneFilename); + if (ewasm) { + if (auto envPath = getenv("ETH_HERA")) + vmPaths.emplace_back(envPath); + else if (auto repoPath = findInDefaultPath(heraFilename)) + vmPaths.emplace_back(*repoPath); + else + vmPaths.emplace_back(heraFilename); } } @@ -239,4 +227,29 @@ bool isValidSemanticTestPath(boost::filesystem::path const& _testPath) return true; } +bool loadVMs(CommonOptions const& _options) +{ + if (_options.disableSemanticTests && !_options.ewasm) + return true; + + auto [evmSupported, ewasmSupported] = solidity::test::EVMHost::checkVmPaths(_options.vmPaths); + if (!_options.disableSemanticTests && !evmSupported) + { + std::cerr << "Unable to find " << solidity::test::evmoneFilename; + std::cerr << ". Please disable semantics tests with --no-semantic-tests or provide a path using --vm ." << std::endl; + std::cerr << "You can download it at" << std::endl; + std::cerr << solidity::test::evmoneDownloadLink << std::endl; + return false; + } + if (_options.ewasm && !ewasmSupported) + { + std::cerr << "Unable to find " << solidity::test::heraFilename; + std::cerr << ". To be able to enable ewasm tests, please provide the path using --vm ." << std::endl; + std::cerr << "You can download it at" << std::endl; + std::cerr << solidity::test::heraDownloadLink << std::endl; + return false; + } + return true; +} + } diff --git a/test/Common.h b/test/Common.h index 332dbe4b9..5448c043c 100644 --- a/test/Common.h +++ b/test/Common.h @@ -62,6 +62,7 @@ struct CommonOptions bool enforceCompileToEwasm = false; bool enforceGasTest = false; u256 enforceGasTestMinValue = 100000; + bool disableSemanticTests = false; bool disableSMT = false; bool useABIEncoderV1 = false; bool showMessages = false; @@ -92,4 +93,6 @@ private: /// Note: @p _testPath can be relative but must include at least the `/test/libsolidity/semanticTests/` part bool isValidSemanticTestPath(boost::filesystem::path const& _testPath); +bool loadVMs(CommonOptions const& _options); + } diff --git a/test/EVMHost.cpp b/test/EVMHost.cpp index 157f4e4d3..4aad3c994 100644 --- a/test/EVMHost.cpp +++ b/test/EVMHost.cpp @@ -69,7 +69,7 @@ evmc::VM& EVMHost::getVM(string const& _path) return NullVM; } -bool EVMHost::checkVmPaths(vector const& _vmPaths) +std::tuple EVMHost::checkVmPaths(vector const& _vmPaths) { bool evmVmFound = false; bool ewasmVmFound = false; @@ -77,7 +77,7 @@ bool EVMHost::checkVmPaths(vector const& _vmPaths) { evmc::VM& vm = EVMHost::getVM(path.string()); if (!vm) - return false; + continue; if (vm.has_capability(EVMC_CAPABILITY_EVM1)) { @@ -93,7 +93,7 @@ bool EVMHost::checkVmPaths(vector const& _vmPaths) ewasmVmFound = true; } } - return evmVmFound; + return {evmVmFound, ewasmVmFound}; } EVMHost::EVMHost(langutil::EVMVersion _evmVersion, evmc::VM& _vm): diff --git a/test/EVMHost.h b/test/EVMHost.h index c258c6dd0..b82d3e255 100644 --- a/test/EVMHost.h +++ b/test/EVMHost.h @@ -49,8 +49,9 @@ public: /// Tries to load all defined evmc vm shared libraries. /// @param _vmPaths paths to multiple evmc shared libraries. /// @throw Exception if multiple evm1 or multiple ewasm evmc vms where loaded. - /// @returns true, if an evmc vm was supporting evm1 loaded properly. - static bool checkVmPaths(std::vector const& _vmPaths); + /// @returns A pair of booleans, the first element being true, if an evmc vm supporting evm1 was loaded properly, + /// the second being true, if an evmc vm supporting ewasm was loaded properly. + static std::tuple checkVmPaths(std::vector const& _vmPaths); explicit EVMHost(langutil::EVMVersion _evmVersion, evmc::VM& _vm); diff --git a/test/boostTest.cpp b/test/boostTest.cpp index fb8d421f9..f0a702812 100644 --- a/test/boostTest.cpp +++ b/test/boostTest.cpp @@ -172,17 +172,10 @@ test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] ) initializeOptions(); - bool disableSemantics = true; - try - { - disableSemantics = !solidity::test::EVMHost::checkVmPaths(solidity::test::CommonOptions::get().vmPaths); - } - catch (std::runtime_error const& _exception) - { - cerr << "Error: " << _exception.what() << endl; + if (!solidity::test::loadVMs(solidity::test::CommonOptions::get())) exit(1); - } - if (disableSemantics) + + if (solidity::test::CommonOptions::get().disableSemanticTests) cout << endl << "--- SKIPPING ALL SEMANTICS TESTS ---" << endl << endl; // Include the interactive tests in the automatic tests as well @@ -193,7 +186,7 @@ test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] ) if (ts.smt && options.disableSMT) continue; - if (ts.needsVM && disableSemantics) + if (ts.needsVM && solidity::test::CommonOptions::get().disableSemanticTests) continue; solAssert(registerTests( @@ -207,7 +200,7 @@ test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] ) ) > 0, std::string("no ") + ts.title + " tests found"); } - if (disableSemantics) + if (solidity::test::CommonOptions::get().disableSemanticTests) { for (auto suite: { "ABIDecoderTest", diff --git a/test/tools/isoltest.cpp b/test/tools/isoltest.cpp index 0f1ee1fac..fa7b914aa 100644 --- a/test/tools/isoltest.cpp +++ b/test/tools/isoltest.cpp @@ -420,82 +420,74 @@ std::optional runTestSuite( int main(int argc, char const *argv[]) { - setupTerminal(); - + try { - auto options = std::make_unique(&TestTool::editor); + setupTerminal(); - try { + auto options = std::make_unique(&TestTool::editor); + if (!options->parse(argc, argv)) return -1; options->validate(); solidity::test::CommonOptions::setSingleton(std::move(options)); } - catch (std::exception const& _exception) - { - cerr << _exception.what() << endl; + + auto& options = dynamic_cast(solidity::test::CommonOptions::get()); + + if (!solidity::test::loadVMs(options)) return 1; + + if (options.disableSemanticTests) + cout << endl << "--- SKIPPING ALL SEMANTICS TESTS ---" << endl << endl; + + TestStats global_stats{0, 0}; + cout << "Running tests..." << endl << endl; + + // Actually run the tests. + // Interactive tests are added in InteractiveTests.h + for (auto const& ts: g_interactiveTestsuites) + { + if (ts.needsVM && options.disableSemanticTests) + continue; + + if (ts.smt && options.disableSMT) + continue; + + auto stats = runTestSuite( + ts.testCaseCreator, + options, + options.testPath / ts.path, + ts.subpath, + ts.title + ); + if (stats) + global_stats += *stats; + else + return 1; } - } - auto& options = dynamic_cast(solidity::test::CommonOptions::get()); + cout << endl << "Summary: "; + AnsiColorized(cout, !options.noColor, {BOLD, global_stats ? GREEN : RED}) << + global_stats.successCount << "/" << global_stats.testCount; + cout << " tests successful"; + if (global_stats.skippedCount > 0) + { + cout << " ("; + AnsiColorized(cout, !options.noColor, {BOLD, YELLOW}) << global_stats.skippedCount; + cout << " tests skipped)"; + } + cout << "." << endl; - bool disableSemantics = true; - try - { - disableSemantics = !solidity::test::EVMHost::checkVmPaths(options.vmPaths); + if (options.disableSemanticTests) + cout << "\nNOTE: Skipped semantics tests.\n" << endl; + + return global_stats ? 0 : 1; } - catch (std::runtime_error const& _exception) + catch (std::exception const& _exception) { - cerr << "Error: " << _exception.what() << endl; + cerr << _exception.what() << endl; return 1; } - - if (disableSemantics) - cout << endl << "--- SKIPPING ALL SEMANTICS TESTS ---" << endl << endl; - - TestStats global_stats{0, 0}; - cout << "Running tests..." << endl << endl; - - // Actually run the tests. - // Interactive tests are added in InteractiveTests.h - for (auto const& ts: g_interactiveTestsuites) - { - if (ts.needsVM && disableSemantics) - continue; - - if (ts.smt && options.disableSMT) - continue; - - auto stats = runTestSuite( - ts.testCaseCreator, - options, - options.testPath / ts.path, - ts.subpath, - ts.title - ); - if (stats) - global_stats += *stats; - else - return 1; - } - - cout << endl << "Summary: "; - AnsiColorized(cout, !options.noColor, {BOLD, global_stats ? GREEN : RED}) << - global_stats.successCount << "/" << global_stats.testCount; - cout << " tests successful"; - if (global_stats.skippedCount > 0) - { - cout << " ("; - AnsiColorized(cout, !options.noColor, {BOLD, YELLOW}) << global_stats.skippedCount; - cout << " tests skipped)"; - } - cout << "." << endl; - - if (disableSemantics) - cout << "\nNOTE: Skipped semantics tests because no evmc vm could be found.\n" << endl; - - return global_stats ? 0 : 1; }