From da38149f5780f8c29fc81c290d71b6e472d4b55c Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Thu, 19 Nov 2020 16:32:35 -0500 Subject: [PATCH] [test] Add support for --enforce-compile-to-ewasm. --- test/Common.cpp | 1 + test/Common.h | 1 + test/TestCase.h | 1 + test/boostTest.cpp | 6 +- test/libsolidity/SemanticTest.cpp | 154 +++++++++++++++++++----------- test/libsolidity/SemanticTest.h | 14 ++- test/tools/isoltest.cpp | 1 + 7 files changed, 115 insertions(+), 63 deletions(-) diff --git a/test/Common.cpp b/test/Common.cpp index 8406727d3..10883e872 100644 --- a/test/Common.cpp +++ b/test/Common.cpp @@ -103,6 +103,7 @@ CommonOptions::CommonOptions(std::string _caption): ("no-smt", po::bool_switch(&disableSMT), "disable SMT checker") ("optimize", po::bool_switch(&optimize), "enables optimization") ("enforce-via-yul", po::bool_switch(&enforceViaYul), "Enforce compiling all tests via yul to see if additional tests can be activated.") + ("enforce-compile-to-ewasm", po::bool_switch(&enforceCompileToEwasm), "Enforce compiling all tests to Ewasm to see if additional tests can be activated.") ("enforce-gas-cost", po::bool_switch(&enforceGasTest), "Enforce checking gas cost in semantic tests.") ("enforce-gas-cost-min-value", po::value(&enforceGasTestMinValue), "Threshold value to enforce adding gas checks to a test.") ("abiencoderv1", po::bool_switch(&useABIEncoderV1), "enables abi encoder v1") diff --git a/test/Common.h b/test/Common.h index 2434e49ce..047d0b8ae 100644 --- a/test/Common.h +++ b/test/Common.h @@ -59,6 +59,7 @@ struct CommonOptions bool ewasm = false; bool optimize = false; bool enforceViaYul = false; + bool enforceCompileToEwasm = false; bool enforceGasTest = false; u256 enforceGasTestMinValue = 100000; bool disableSMT = false; diff --git a/test/TestCase.h b/test/TestCase.h index f4699db39..5d4394b11 100644 --- a/test/TestCase.h +++ b/test/TestCase.h @@ -41,6 +41,7 @@ public: langutil::EVMVersion evmVersion; std::vector vmPaths; bool enforceCompileViaYul = false; + bool enforceCompileToEwasm = false; bool enforceGasCost = false; u256 enforceGasCostMinValue; }; diff --git a/test/boostTest.cpp b/test/boostTest.cpp index 0162cd593..86d2e0a66 100644 --- a/test/boostTest.cpp +++ b/test/boostTest.cpp @@ -65,6 +65,7 @@ int registerTests( boost::filesystem::path const& _basepath, boost::filesystem::path const& _path, bool _enforceViaYul, + bool _enforceCompileToEwasm, vector const& _labels, TestCase::TestCaseCreator _testCaseCreator ) @@ -76,8 +77,9 @@ int registerTests( solidity::test::CommonOptions::get().evmVersion(), solidity::test::CommonOptions::get().vmPaths, _enforceViaYul, + _enforceCompileToEwasm, solidity::test::CommonOptions::get().enforceGasTest, - solidity::test::CommonOptions::get().enforceGasTestMinValue + solidity::test::CommonOptions::get().enforceGasTestMinValue, }; if (fs::is_directory(fullpath)) { @@ -91,6 +93,7 @@ int registerTests( *sub_suite, _basepath, _path / entry.path().filename(), _enforceViaYul, + _enforceCompileToEwasm, _labels, _testCaseCreator ); @@ -195,6 +198,7 @@ test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] ) options.testPath / ts.path, ts.subpath, options.enforceViaYul, + options.enforceCompileToEwasm, ts.labels, ts.testCaseCreator ) > 0, std::string("no ") + ts.title + " tests found"); diff --git a/test/libsolidity/SemanticTest.cpp b/test/libsolidity/SemanticTest.cpp index 7a06f0ba1..0307abedc 100644 --- a/test/libsolidity/SemanticTest.cpp +++ b/test/libsolidity/SemanticTest.cpp @@ -50,6 +50,7 @@ SemanticTest::SemanticTest( langutil::EVMVersion _evmVersion, vector const& _vmPaths, bool _enforceViaYul, + bool _enforceCompileToEwasm, bool _enforceGasCost, u256 _enforceGasCostMinValue ): @@ -58,51 +59,43 @@ SemanticTest::SemanticTest( m_sources(m_reader.sources()), m_lineOffset(m_reader.lineNumber()), m_enforceViaYul(_enforceViaYul), + m_enforceCompileToEwasm(_enforceCompileToEwasm), m_enforceGasCost(_enforceGasCost), - m_enforceGasCostMinValue(_enforceGasCostMinValue) + m_enforceGasCostMinValue(std::move(_enforceGasCostMinValue)) { initializeBuiltins(); - string choice = m_reader.stringSetting("compileViaYul", "default"); - if (choice == "also") + static set const compileViaYulAllowedValues{"also", "true", "false", "default"}; + static set const yulRunTriggers{"also", "true"}; + static set const legacyRunTriggers{"also", "false", "default"}; + + string compileViaYul = m_reader.stringSetting("compileViaYul", "default"); + if (!contains(compileViaYulAllowedValues, compileViaYul)) + BOOST_THROW_EXCEPTION(runtime_error("Invalid compileViaYul value: " + compileViaYul + ".")); + m_testCaseWantsYulRun = contains(yulRunTriggers, compileViaYul); + m_testCaseWantsLegacyRun = contains(legacyRunTriggers, compileViaYul); + + // Do not enforce via yul and ewasm, if via yul was explicitly denied. + if (compileViaYul == "false") { - m_runWithYul = true; - m_runWithoutYul = true; - } - else if (choice == "true") - { - m_runWithYul = true; - m_runWithoutYul = false; - } - else if (choice == "false") - { - m_runWithYul = false; - m_runWithoutYul = true; - // Do not try to run via yul if explicitly denied. m_enforceViaYul = false; + m_enforceCompileToEwasm = false; } - else if (choice == "default") - { - m_runWithYul = false; - m_runWithoutYul = true; - } - else - BOOST_THROW_EXCEPTION(runtime_error("Invalid compileViaYul value: " + choice + ".")); string compileToEwasm = m_reader.stringSetting("compileToEwasm", "false"); if (compileToEwasm == "also") - m_runWithEwasm = true; + m_testCaseWantsEwasmRun = true; else if (compileToEwasm == "false") - m_runWithEwasm = false; + m_testCaseWantsEwasmRun = false; else BOOST_THROW_EXCEPTION(runtime_error("Invalid compileToEwasm value: " + compileToEwasm + ".")); - if (m_runWithEwasm && !m_runWithYul) + if (m_testCaseWantsEwasmRun && !m_testCaseWantsYulRun) BOOST_THROW_EXCEPTION(runtime_error("Invalid compileToEwasm value: " + compileToEwasm + ", compileViaYul need to be enabled.")); - // run ewasm tests only, if an ewasm evmc vm was defined - if (m_runWithEwasm && !m_supportsEwasm) - m_runWithEwasm = false; + // run ewasm tests only if an ewasm evmc vm was defined + if (m_testCaseWantsEwasmRun && !m_supportsEwasm) + m_testCaseWantsEwasmRun = false; m_runWithABIEncoderV1Only = m_reader.boolSetting("ABIEncoderV1Only", false); if (m_runWithABIEncoderV1Only && !solidity::test::CommonOptions::get().useABIEncoderV1) @@ -136,17 +129,26 @@ void SemanticTest::initializeBuiltins() TestCase::TestResult SemanticTest::run(ostream& _stream, string const& _linePrefix, bool _formatted) { TestResult result = TestResult::Success; - bool compileViaYul = m_runWithYul || m_enforceViaYul; - if (m_runWithoutYul) + if (m_testCaseWantsLegacyRun) result = runTest(_stream, _linePrefix, _formatted, false, false); - if (compileViaYul && result == TestResult::Success) + if ((m_testCaseWantsYulRun || m_enforceViaYul) && result == TestResult::Success) result = runTest(_stream, _linePrefix, _formatted, true, false); - if (m_runWithEwasm && result == TestResult::Success) - result = runTest(_stream, _linePrefix, _formatted, true, true); - + if ((m_testCaseWantsEwasmRun || m_enforceCompileToEwasm) && result == TestResult::Success) + { + // TODO: Once we have full Ewasm support, we could remove try/catch here. + try + { + result = runTest(_stream, _linePrefix, _formatted, true, true); + } + catch (...) + { + if (!m_enforceCompileToEwasm) + throw; + } + } return result; } @@ -154,31 +156,34 @@ TestCase::TestResult SemanticTest::runTest( ostream& _stream, string const& _linePrefix, bool _formatted, - bool _compileViaYul, - bool _compileToEwasm -) + bool _isYulRun, + bool _isEwasmRun) { bool success = true; m_gasCostFailure = false; - if (_compileViaYul && _compileToEwasm) + if (_isEwasmRun) + { + soltestAssert(_isYulRun, ""); selectVM(evmc_capabilities::EVMC_CAPABILITY_EWASM); + } else selectVM(evmc_capabilities::EVMC_CAPABILITY_EVM1); reset(); - m_compileViaYul = _compileViaYul; - if (_compileToEwasm) + m_compileViaYul = _isYulRun; + if (_isEwasmRun) { soltestAssert(m_compileViaYul, ""); - m_compileToEwasm = _compileToEwasm; + m_compileToEwasm = _isEwasmRun; } - m_compileViaYulCanBeSet = false; + m_canEnableYulRun = false; + m_canEnableEwasmRun = false; - if (_compileViaYul) - AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Running via Yul:" << endl; + if (_isYulRun) + AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Running via Yul" << (_isEwasmRun ? " (ewasm):" : ":") << endl; for (TestFunctionCall& test: m_tests) test.reset(); @@ -230,7 +235,7 @@ TestCase::TestResult SemanticTest::runTest( { if (m_transactionSuccessful == test.call().expectations.failure) success = false; - if (success && !checkGasCostExpectation(test, _compileViaYul)) + if (success && !checkGasCostExpectation(test, _isYulRun)) m_gasCostFailure = true; test.setFailure(!m_transactionSuccessful); @@ -268,7 +273,7 @@ TestCase::TestResult SemanticTest::runTest( } bool outputMismatch = (output != test.call().expectations.rawBytes()); - if (!outputMismatch && !checkGasCostExpectation(test, _compileViaYul)) + if (!outputMismatch && !checkGasCostExpectation(test, _isYulRun)) { success = false; m_gasCostFailure = true; @@ -287,9 +292,9 @@ TestCase::TestResult SemanticTest::runTest( } } - if (!m_runWithYul && _compileViaYul) + if (!m_testCaseWantsYulRun && _isYulRun) { - m_compileViaYulCanBeSet = success; + m_canEnableYulRun = success; string message = success ? "Test can pass via Yul, but marked with \"compileViaYul: false.\"" : "Test compiles via Yul, but it gives different test results."; @@ -299,8 +304,31 @@ TestCase::TestResult SemanticTest::runTest( return TestResult::Failure; } - if (!success && (m_runWithYul || !_compileViaYul)) + // Right now we have sometimes different test results in Yul vs. Ewasm. + // The main reason is that Ewasm just returns a failure in some cases. + // TODO: If Ewasm support got fully implemented, we could implement this in the same way as above. + if (success && !m_testCaseWantsEwasmRun && _isEwasmRun) { + // TODO: There is something missing in Ewasm to support other types of revert strings: + // for now, we just ignore test-cases that do not use RevertStrings::Default. + if (m_revertStrings != RevertStrings::Default) + return TestResult::Success; + + m_canEnableEwasmRun = true; + AnsiColorized(_stream, _formatted, {BOLD, YELLOW}) << + _linePrefix << endl << + _linePrefix << "Test can pass via Yul (Ewasm), but marked with \"compileToEwasm: false.\"" << endl; + return TestResult::Failure; + } + + if (!success) + { + // Ignore failing tests that can't yet get compiled to Ewasm: + // if the test run was not successful and enforce compiling to ewasm was set, + // but the test case did not want to get run with Ewasm, we just ignore this failure. + if (m_enforceCompileToEwasm && !m_testCaseWantsEwasmRun) + return TestResult::Success; + AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result:" << endl; for (TestFunctionCall const& test: m_tests) { @@ -331,13 +359,13 @@ TestCase::TestResult SemanticTest::runTest( AnsiColorized(_stream, _formatted, {BOLD, RED}) << _linePrefix << endl << _linePrefix << "Attention: Updates on the test will apply the detected format displayed." << endl; - if (_compileViaYul && m_runWithoutYul) + if (_isYulRun && m_testCaseWantsLegacyRun) { _stream << _linePrefix << endl << _linePrefix; AnsiColorized(_stream, _formatted, {RED_BACKGROUND}) << "Note that the test passed without Yul."; _stream << endl; } - else if (!_compileViaYul && m_runWithYul) + else if (!_isYulRun && m_testCaseWantsYulRun) AnsiColorized(_stream, _formatted, {BOLD, YELLOW}) << _linePrefix << endl << _linePrefix << "Note that the test also has to pass via Yul." << endl; @@ -431,15 +459,27 @@ void SemanticTest::printUpdatedExpectations(ostream& _stream, string const&) con void SemanticTest::printUpdatedSettings(ostream& _stream, string const& _linePrefix) { auto& settings = m_reader.settings(); - if (settings.empty() && !m_compileViaYulCanBeSet) + if (settings.empty() && !m_canEnableYulRun) return; _stream << _linePrefix << "// ====" << endl; - if (m_compileViaYulCanBeSet) + if (m_canEnableEwasmRun) + { + soltestAssert(m_canEnableYulRun || m_testCaseWantsYulRun, ""); + string compileViaYul = m_reader.stringSetting("compileViaYul", ""); + if (!compileViaYul.empty()) + _stream << _linePrefix << "// compileViaYul: " << compileViaYul << "\n"; + _stream << _linePrefix << "// compileToEwasm: also\n"; + } + else if (m_canEnableYulRun) _stream << _linePrefix << "// compileViaYul: also\n"; - for (auto const& setting: settings) - if (!m_compileViaYulCanBeSet || setting.first != "compileViaYul") - _stream << _linePrefix << "// " << setting.first << ": " << setting.second << endl; + + for (auto const& [settingName, settingValue]: settings) + if ( + !(settingName == "compileToEwasm" && m_canEnableEwasmRun) && + !(settingName == "compileViaYul" && (m_canEnableYulRun || m_canEnableEwasmRun)) + ) + _stream << _linePrefix << "// " << settingName << ": " << settingValue<< endl; } void SemanticTest::parseExpectations(istream& _stream) diff --git a/test/libsolidity/SemanticTest.h b/test/libsolidity/SemanticTest.h index 2bba341f4..873357f38 100644 --- a/test/libsolidity/SemanticTest.h +++ b/test/libsolidity/SemanticTest.h @@ -46,6 +46,7 @@ public: _options.evmVersion, _options.vmPaths, _options.enforceCompileViaYul, + _options.enforceCompileToEwasm, _options.enforceGasCost, _options.enforceGasCostMinValue ); @@ -56,6 +57,7 @@ public: langutil::EVMVersion _evmVersion, std::vector const& _vmPaths, bool _enforceViaYul = false, + bool _enforceCompileToEwasm = false, bool _enforceGasCost = false, u256 _enforceGasCostMinValue = 100000 ); @@ -76,19 +78,21 @@ public: bool deploy(std::string const& _contractName, u256 const& _value, bytes const& _arguments, std::map const& _libraries = {}); private: - TestResult runTest(std::ostream& _stream, std::string const& _linePrefix, bool _formatted, bool _compileViaYul, bool _compileToEwasm); + TestResult runTest(std::ostream& _stream, std::string const& _linePrefix, bool _formatted, bool _isYulRun, bool _isEwasmRun); bool checkGasCostExpectation(TestFunctionCall& io_test, bool _compileViaYul) const; void initializeBuiltins(); SourceMap m_sources; std::size_t m_lineOffset; std::vector m_tests; - bool m_runWithYul = false; - bool m_runWithEwasm = false; - bool m_runWithoutYul = true; + bool m_testCaseWantsYulRun = false; + bool m_testCaseWantsEwasmRun = false; + bool m_testCaseWantsLegacyRun = true; bool m_enforceViaYul = false; + bool m_enforceCompileToEwasm = false; bool m_runWithABIEncoderV1Only = false; bool m_allowNonExistingFunctions = false; - bool m_compileViaYulCanBeSet = false; + bool m_canEnableYulRun = false; + bool m_canEnableEwasmRun = false; std::map m_builtins{}; bool m_gasCostFailure = false; diff --git a/test/tools/isoltest.cpp b/test/tools/isoltest.cpp index b05cbe84a..8aa93d6cf 100644 --- a/test/tools/isoltest.cpp +++ b/test/tools/isoltest.cpp @@ -163,6 +163,7 @@ TestTool::Result TestTool::process() m_options.evmVersion(), m_options.vmPaths, m_options.enforceViaYul, + m_options.enforceCompileToEwasm, m_options.enforceGasTest, m_options.enforceGasTestMinValue });