diff --git a/test/libsolidity/AnalysisFramework.cpp b/test/libsolidity/AnalysisFramework.cpp index 7e4c32eae..6839efe1f 100644 --- a/test/libsolidity/AnalysisFramework.cpp +++ b/test/libsolidity/AnalysisFramework.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -66,11 +67,60 @@ AnalysisFramework::parseAnalyseAndReturnError( return make_pair(&compiler().ast(""), std::move(errors)); } +bool AnalysisFramework::runFramework(StringMap _sources, PipelineStage _targetStage) +{ + resetFramework(); + m_targetStage = _targetStage; + soltestAssert(m_compiler); + + m_compiler->setSources(std::move(_sources)); + setupCompiler(*m_compiler); + executeCompilationPipeline(); + return pipelineSuccessful(); +} + +void AnalysisFramework::resetFramework() +{ + compiler().reset(); + m_targetStage = PipelineStage::Compilation; +} + std::unique_ptr AnalysisFramework::createStack() const { return std::make_unique(); } +void AnalysisFramework::setupCompiler(CompilerStack& _compiler) +{ + _compiler.setEVMVersion(solidity::test::CommonOptions::get().evmVersion()); +} + +void AnalysisFramework::executeCompilationPipeline() +{ + soltestAssert(m_compiler); + + // If you add a new stage, remember to handle it below. + soltestAssert( + m_targetStage == PipelineStage::Parsing || + m_targetStage == PipelineStage::Analysis || + m_targetStage == PipelineStage::Compilation + ); + + bool parsingSuccessful = m_compiler->parse(); + soltestAssert(parsingSuccessful || !filteredErrors(false /* _includeWarningsAndInfos */).empty()); + if (!parsingSuccessful || stageSuccessful(m_targetStage)) + return; + + bool analysisSuccessful = m_compiler->analyze(); + soltestAssert(analysisSuccessful || !filteredErrors(false /* _includeWarningsAndInfos */).empty()); + if (!analysisSuccessful || stageSuccessful(m_targetStage)) + return; + + bool compilationSuccessful = m_compiler->compile(); + soltestAssert(compilationSuccessful || !filteredErrors(false /* _includeWarningsAndInfos */).empty()); + soltestAssert(stageSuccessful(m_targetStage) == compilationSuccessful); +} + ErrorList AnalysisFramework::filterErrors(ErrorList const& _errorList, bool _includeWarningsAndInfos) const { ErrorList errors; @@ -113,6 +163,16 @@ ErrorList AnalysisFramework::filterErrors(ErrorList const& _errorList, bool _inc return errors; } +bool AnalysisFramework::stageSuccessful(PipelineStage _stage) const +{ + switch (_stage) { + case PipelineStage::Parsing: return compiler().state() >= CompilerStack::Parsed; + case PipelineStage::Analysis: return compiler().state() >= CompilerStack::AnalysisSuccessful; + case PipelineStage::Compilation: return compiler().state() >= CompilerStack::CompilationSuccessful; + } + unreachable(); +} + SourceUnit const* AnalysisFramework::parseAndAnalyse(std::string const& _source) { auto sourceAndError = parseAnalyseAndReturnError(_source); diff --git a/test/libsolidity/AnalysisFramework.h b/test/libsolidity/AnalysisFramework.h index 65b6940f6..902f4369f 100644 --- a/test/libsolidity/AnalysisFramework.h +++ b/test/libsolidity/AnalysisFramework.h @@ -39,6 +39,12 @@ using FunctionTypePointer = FunctionType const*; namespace solidity::frontend::test { +enum class PipelineStage { + Parsing, + Analysis, + Compilation, +}; + class AnalysisFramework { @@ -56,6 +62,25 @@ protected: bool success(std::string const& _source); langutil::ErrorList expectError(std::string const& _source, bool _warning = false, bool _allowMultiple = false); +public: + /// Runs the full compiler pipeline on specified sources. This is the main function of the + /// framework. Resets the stack, configures it and runs either until the first failed stage or + /// until the @p _targetStage is reached. + /// Afterwards the caller can inspect the stack via @p compiler(). The framework provides a few + /// convenience helpers to check the state and error list, in general the caller can freely + /// access the stack, including generating outputs if the compilation succeeded. + bool runFramework(StringMap _sources, PipelineStage _targetStage = PipelineStage::Compilation); + bool runFramework(std::string _source, PipelineStage _targetStage = PipelineStage::Compilation) + { + return runFramework({{"", std::move(_source)}}, _targetStage); + } + + void resetFramework(); + + PipelineStage targetStage() const { return m_targetStage; } + bool pipelineSuccessful() const { return stageSuccessful(m_targetStage); } + bool stageSuccessful(PipelineStage _stage) const; + std::string formatErrors( langutil::ErrorList const& _errors, bool _colored = false, @@ -80,9 +105,6 @@ protected: return filterErrors(compiler().errors(), _includeWarningsAndInfos); } - std::vector m_warningsToFilter = {"This is a pre-release compiler version"}; - std::vector m_messagesToCut = {"Source file requires different compiler version (current compiler is"}; - /// @returns reference to lazy-instantiated CompilerStack. solidity::frontend::CompilerStack& compiler() { @@ -99,12 +121,25 @@ protected: return *m_compiler; } +protected: /// Creates a new instance of @p CompilerStack. Override if your test case needs to pass in /// custom constructor arguments. virtual std::unique_ptr createStack() const; + /// Configures @p CompilerStack. The default implementation sets basic parameters based on + /// CLI options. Override if your test case needs extra configuration. + virtual void setupCompiler(CompilerStack& _compiler); + + /// Executes the requested pipeline stages until @p m_targetStage is reached. + /// Stops at the first failed stage. + void executeCompilationPipeline(); + + std::vector m_warningsToFilter = {"This is a pre-release compiler version"}; + std::vector m_messagesToCut = {"Source file requires different compiler version (current compiler is"}; + private: mutable std::unique_ptr m_compiler; + PipelineStage m_targetStage = PipelineStage::Compilation; }; // Asserts that the compilation down to typechecking diff --git a/test/libsolidity/GasTest.cpp b/test/libsolidity/GasTest.cpp index 086060588..87f67dcff 100644 --- a/test/libsolidity/GasTest.cpp +++ b/test/libsolidity/GasTest.cpp @@ -97,13 +97,13 @@ void GasTest::printUpdatedExpectations(std::ostream& _stream, std::string const& } } -TestCase::TestResult GasTest::run(std::ostream& _stream, std::string const& _linePrefix, bool _formatted) +void GasTest::setupCompiler(CompilerStack& _compiler) { - compiler().reset(); + // Prerelease CBOR metadata varies in size due to changing version numbers and build dates. // This leads to volatile creation cost estimates. Therefore we force the compiler to // release mode for testing gas estimates. - compiler().setMetadataFormat(CompilerStack::MetadataFormat::NoMetadata); + _compiler.setMetadataFormat(CompilerStack::MetadataFormat::NoMetadata); OptimiserSettings settings = m_optimise ? OptimiserSettings::standard() : OptimiserSettings::minimal(); if (m_optimiseYul) { @@ -111,10 +111,12 @@ TestCase::TestResult GasTest::run(std::ostream& _stream, std::string const& _lin settings.optimizeStackAllocation = m_optimise; } settings.expectedExecutionsPerDeployment = m_optimiseRuns; - compiler().setOptimiserSettings(settings); - compiler().setSources({{"", withPreamble(m_source)}}); + _compiler.setOptimiserSettings(settings); +} - if (!compiler().parseAndAnalyze() || !compiler().compile()) +TestCase::TestResult GasTest::run(std::ostream& _stream, std::string const& _linePrefix, bool _formatted) +{ + if (!runFramework(withPreamble(m_source), PipelineStage::Compilation)) { _stream << formatErrors(filteredErrors(), _formatted); return TestResult::FatalError; diff --git a/test/libsolidity/GasTest.h b/test/libsolidity/GasTest.h index 46619c471..a04976cd5 100644 --- a/test/libsolidity/GasTest.h +++ b/test/libsolidity/GasTest.h @@ -44,6 +44,9 @@ public: void printSource(std::ostream &_stream, std::string const &_linePrefix = "", bool _formatted = false) const override; void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override; +protected: + void setupCompiler(CompilerStack& _compiler) override; + private: void parseExpectations(std::istream& _stream); diff --git a/test/libsolidity/MemoryGuardTest.cpp b/test/libsolidity/MemoryGuardTest.cpp index c8475469d..d2e5d3d09 100644 --- a/test/libsolidity/MemoryGuardTest.cpp +++ b/test/libsolidity/MemoryGuardTest.cpp @@ -36,13 +36,15 @@ using namespace solidity::frontend; using namespace solidity::frontend::test; using namespace yul; +void MemoryGuardTest::setupCompiler(CompilerStack& _compiler) +{ + _compiler.setViaIR(true); + _compiler.setOptimiserSettings(OptimiserSettings::none()); +} + TestCase::TestResult MemoryGuardTest::run(std::ostream& _stream, std::string const& _linePrefix, bool _formatted) { - compiler().reset(); - compiler().setSources(StringMap{{"", m_source}}); - compiler().setViaIR(true); - compiler().setOptimiserSettings(OptimiserSettings::none()); - if (!compiler().compile()) + if (!runFramework(m_source, PipelineStage::Compilation)) { _stream << formatErrors(filteredErrors(), _formatted); return TestResult::FatalError; diff --git a/test/libsolidity/MemoryGuardTest.h b/test/libsolidity/MemoryGuardTest.h index e2ca8a6e5..ec6fe5e5b 100644 --- a/test/libsolidity/MemoryGuardTest.h +++ b/test/libsolidity/MemoryGuardTest.h @@ -48,6 +48,9 @@ public: } TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override; + +protected: + void setupCompiler(CompilerStack& _compiler) override; }; } diff --git a/test/libsolidity/SMTCheckerTest.cpp b/test/libsolidity/SMTCheckerTest.cpp index 6368bfa4f..893ccb0f1 100644 --- a/test/libsolidity/SMTCheckerTest.cpp +++ b/test/libsolidity/SMTCheckerTest.cpp @@ -126,11 +126,11 @@ SMTCheckerTest::SMTCheckerTest(std::string const& _filename): SyntaxTest(_filena m_modelCheckerSettings.bmcLoopIterations = std::optional{bmcLoopIterations}; } -void SMTCheckerTest::setupCompiler() +void SMTCheckerTest::setupCompiler(CompilerStack& _compiler) { - SyntaxTest::setupCompiler(); + SyntaxTest::setupCompiler(_compiler); - compiler().setModelCheckerSettings(m_modelCheckerSettings); + _compiler.setModelCheckerSettings(m_modelCheckerSettings); } void SMTCheckerTest::filterObtainedErrors() diff --git a/test/libsolidity/SMTCheckerTest.h b/test/libsolidity/SMTCheckerTest.h index 3e8d8238b..d42572dc5 100644 --- a/test/libsolidity/SMTCheckerTest.h +++ b/test/libsolidity/SMTCheckerTest.h @@ -38,7 +38,7 @@ public: } SMTCheckerTest(std::string const& _filename); - void setupCompiler() override; + void setupCompiler(CompilerStack& _compiler) override; void filterObtainedErrors() override; void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override; diff --git a/test/libsolidity/SolidityCompiler.cpp b/test/libsolidity/SolidityCompiler.cpp index 9252cac16..1d4f6851a 100644 --- a/test/libsolidity/SolidityCompiler.cpp +++ b/test/libsolidity/SolidityCompiler.cpp @@ -28,7 +28,17 @@ namespace solidity::frontend::test { -BOOST_FIXTURE_TEST_SUITE(SolidityCompiler, AnalysisFramework) +class SolidityCompilerFixture: protected AnalysisFramework +{ + void setupCompiler(CompilerStack& _compiler) override + { + // FIXME: This test was probably supposed to respect CommonOptions::get().optimize but + // due to a bug it was always running with optimizer disabled and it does not pass with it. + _compiler.setOptimiserSettings(false); + } +}; + +BOOST_FIXTURE_TEST_SUITE(SolidityCompiler, SolidityCompilerFixture) BOOST_AUTO_TEST_CASE(does_not_include_creation_time_only_internal_functions) { @@ -39,12 +49,13 @@ BOOST_AUTO_TEST_CASE(does_not_include_creation_time_only_internal_functions) function f() internal { unchecked { for (uint i = 0; i < 10; ++i) x += 3 + i; } } } )"; - compiler().setOptimiserSettings(solidity::test::CommonOptions::get().optimize); - BOOST_REQUIRE(success(sourceCode)); + + runFramework(sourceCode, PipelineStage::Compilation); BOOST_REQUIRE_MESSAGE( - compiler().compile(), + pipelineSuccessful(), "Contract compilation failed:\n" + formatErrors(filteredErrors(), true /* _colored */) ); + bytes const& creationBytecode = solidity::test::bytecodeSansMetadata(compiler().object("C").bytecode); bytes const& runtimeBytecode = solidity::test::bytecodeSansMetadata(compiler().runtimeObject("C").bytecode); BOOST_CHECK(creationBytecode.size() >= 90); diff --git a/test/libsolidity/SyntaxTest.cpp b/test/libsolidity/SyntaxTest.cpp index ad53c2b26..906391e01 100644 --- a/test/libsolidity/SyntaxTest.cpp +++ b/test/libsolidity/SyntaxTest.cpp @@ -49,53 +49,51 @@ SyntaxTest::SyntaxTest( m_optimiseYul = m_reader.boolSetting("optimize-yul", true); } -void SyntaxTest::setupCompiler() +void SyntaxTest::setupCompiler(CompilerStack& _compiler) { - compiler().reset(); - compiler().setSources(withPreamble(m_sources.sources)); - compiler().setEVMVersion(m_evmVersion); - compiler().setOptimiserSettings( + AnalysisFramework::setupCompiler(_compiler); + + _compiler.setEVMVersion(m_evmVersion); + _compiler.setOptimiserSettings( m_optimiseYul ? OptimiserSettings::full() : OptimiserSettings::minimal() ); - compiler().setMetadataFormat(CompilerStack::MetadataFormat::NoMetadata); - compiler().setMetadataHash(CompilerStack::MetadataHash::None); + _compiler.setMetadataFormat(CompilerStack::MetadataFormat::NoMetadata); + _compiler.setMetadataHash(CompilerStack::MetadataHash::None); } void SyntaxTest::parseAndAnalyze() { - setupCompiler(); - - if (compiler().parse() && compiler().analyze()) - try + try + { + runFramework(withPreamble(m_sources.sources), PipelineStage::Compilation); + if (!pipelineSuccessful() && stageSuccessful(PipelineStage::Analysis)) { - if (!compiler().compile()) - { - ErrorList const& errors = compiler().errors(); - auto codeGeneretionErrorCount = count_if(errors.cbegin(), errors.cend(), [](auto const& error) { - return error->type() == Error::Type::CodeGenerationError; - }); - auto errorCount = count_if(errors.cbegin(), errors.cend(), [](auto const& error) { - return Error::isError(error->type()); - }); - // failing compilation after successful analysis is a rare case, - // it assumes that errors contain exactly one error, and the error is of type Error::Type::CodeGenerationError - if (codeGeneretionErrorCount != 1 || errorCount != 1) - BOOST_THROW_EXCEPTION(runtime_error("Compilation failed even though analysis was successful.")); - } - } - catch (UnimplementedFeatureError const& _e) - { - m_errorList.emplace_back(SyntaxTestError{ - "UnimplementedFeatureError", - nullopt, - errorMessage(_e), - "", - -1, - -1 + ErrorList const& errors = compiler().errors(); + auto codeGeneretionErrorCount = count_if(errors.cbegin(), errors.cend(), [](auto const& error) { + return error->type() == Error::Type::CodeGenerationError; }); + auto errorCount = count_if(errors.cbegin(), errors.cend(), [](auto const& error) { + return Error::isError(error->type()); + }); + // failing compilation after successful analysis is a rare case, + // it assumes that errors contain exactly one error, and the error is of type Error::Type::CodeGenerationError + if (codeGeneretionErrorCount != 1 || errorCount != 1) + BOOST_THROW_EXCEPTION(runtime_error("Compilation failed even though analysis was successful.")); } + } + catch (UnimplementedFeatureError const& _e) + { + m_errorList.emplace_back(SyntaxTestError{ + "UnimplementedFeatureError", + nullopt, + errorMessage(_e), + "", + -1, + -1 + }); + } filterObtainedErrors(); } @@ -146,4 +144,3 @@ void SyntaxTest::filterObtainedErrors() }); } } - diff --git a/test/libsolidity/SyntaxTest.h b/test/libsolidity/SyntaxTest.h index a85e1f53f..a3f8f4e5d 100644 --- a/test/libsolidity/SyntaxTest.h +++ b/test/libsolidity/SyntaxTest.h @@ -48,7 +48,7 @@ public: ); protected: - virtual void setupCompiler(); + void setupCompiler(CompilerStack& _compiler) override; void parseAndAnalyze() override; virtual void filterObtainedErrors();