Refactor AnalysisFramework to share more features between test cases that depend on it

test/libsolidity/AnalysisFramework.cpp

@@ -22,6 +22,7 @@
 #include <test/libsolidity/AnalysisFramework.h>
 
 #include <test/libsolidity/util/Common.h>
+#include <test/libsolidity/util/SoltestErrors.h>
 #include <test/Common.h>
 
 #include <libsolidity/interface/CompilerStack.h>
@@ -66,11 +67,60 @@ AnalysisFramework::parseAnalyseAndReturnError(
 	return make_pair(&compiler().ast(""), std::move(errors));
 }
 
+bool AnalysisFramework::runFramework(StringMap _sources, PipelineStage _targetStage)
+{
+	resetFramework();
+	m_targetStage = _targetStage;
+	soltestAssert(m_compiler);
+
+	m_compiler->setSources(std::move(_sources));
+	setupCompiler(*m_compiler);
+	executeCompilationPipeline();
+	return pipelineSuccessful();
+}
+
+void AnalysisFramework::resetFramework()
+{
+	compiler().reset();
+	m_targetStage = PipelineStage::Compilation;
+}
+
 std::unique_ptr<CompilerStack> AnalysisFramework::createStack() const
 {
 	return std::make_unique<CompilerStack>();
 }
 
+void AnalysisFramework::setupCompiler(CompilerStack& _compiler)
+{
+	_compiler.setEVMVersion(solidity::test::CommonOptions::get().evmVersion());
+}
+
+void AnalysisFramework::executeCompilationPipeline()
+{
+	soltestAssert(m_compiler);
+
+	// If you add a new stage, remember to handle it below.
+	soltestAssert(
+		m_targetStage == PipelineStage::Parsing ||
+		m_targetStage == PipelineStage::Analysis ||
+		m_targetStage == PipelineStage::Compilation
+	);
+
+	bool parsingSuccessful = m_compiler->parse();
+	soltestAssert(parsingSuccessful || !filteredErrors(false /* _includeWarningsAndInfos */).empty());
+	if (!parsingSuccessful || stageSuccessful(m_targetStage))
+		return;
+
+	bool analysisSuccessful = m_compiler->analyze();
+	soltestAssert(analysisSuccessful || !filteredErrors(false /* _includeWarningsAndInfos */).empty());
+	if (!analysisSuccessful || stageSuccessful(m_targetStage))
+		return;
+
+	bool compilationSuccessful = m_compiler->compile();
+	soltestAssert(compilationSuccessful || !filteredErrors(false /* _includeWarningsAndInfos */).empty());
+	soltestAssert(stageSuccessful(m_targetStage) == compilationSuccessful);
+}
+
 ErrorList AnalysisFramework::filterErrors(ErrorList const& _errorList, bool _includeWarningsAndInfos) const
 {
 	ErrorList errors;
@@ -113,6 +163,16 @@ ErrorList AnalysisFramework::filterErrors(ErrorList const& _errorList, bool _inc
 	return errors;
 }
 
+bool AnalysisFramework::stageSuccessful(PipelineStage _stage) const
+{
+	switch (_stage) {
+		case PipelineStage::Parsing: return compiler().state() >= CompilerStack::Parsed;
+		case PipelineStage::Analysis: return compiler().state() >= CompilerStack::AnalysisSuccessful;
+		case PipelineStage::Compilation: return compiler().state() >= CompilerStack::CompilationSuccessful;
+	}
+	unreachable();
+}
+
 SourceUnit const* AnalysisFramework::parseAndAnalyse(std::string const& _source)
 {
 	auto sourceAndError = parseAnalyseAndReturnError(_source);

test/libsolidity/AnalysisFramework.h

@@ -39,6 +39,12 @@ using FunctionTypePointer = FunctionType const*;
 namespace solidity::frontend::test
 {
 
+enum class PipelineStage {
+	Parsing,
+	Analysis,
+	Compilation,
+};
+
 class AnalysisFramework
 {
 
@@ -56,6 +62,25 @@ protected:
 	bool success(std::string const& _source);
 	langutil::ErrorList expectError(std::string const& _source, bool _warning = false, bool _allowMultiple = false);
 
+public:
+	/// Runs the full compiler pipeline on specified sources. This is the main function of the
+	/// framework. Resets the stack, configures it and runs either until the first failed stage or
+	/// until the @p _targetStage is reached.
+	/// Afterwards the caller can inspect the stack via @p compiler(). The framework provides a few
+	/// convenience helpers to check the state and error list, in general the caller can freely
+	/// access the stack, including generating outputs if the compilation succeeded.
+	bool runFramework(StringMap _sources, PipelineStage _targetStage = PipelineStage::Compilation);
+	bool runFramework(std::string _source, PipelineStage _targetStage = PipelineStage::Compilation)
+	{
+		return runFramework({{"", std::move(_source)}}, _targetStage);
+	}
+
+	void resetFramework();
+
+	PipelineStage targetStage() const { return m_targetStage; }
+	bool pipelineSuccessful() const { return stageSuccessful(m_targetStage); }
+	bool stageSuccessful(PipelineStage _stage) const;
+
 	std::string formatErrors(
 		langutil::ErrorList const& _errors,
 		bool _colored = false,
@@ -80,9 +105,6 @@ protected:
 		return filterErrors(compiler().errors(), _includeWarningsAndInfos);
 	}
 
-	std::vector<std::string> m_warningsToFilter = {"This is a pre-release compiler version"};
-	std::vector<std::string> m_messagesToCut = {"Source file requires different compiler version (current compiler is"};
-
 	/// @returns reference to lazy-instantiated CompilerStack.
 	solidity::frontend::CompilerStack& compiler()
 	{
@@ -99,12 +121,25 @@ protected:
 		return *m_compiler;
 	}
 
+protected:
 	/// Creates a new instance of @p CompilerStack. Override if your test case needs to pass in
 	/// custom constructor arguments.
 	virtual std::unique_ptr<CompilerStack> createStack() const;
 
+	/// Configures @p CompilerStack. The default implementation sets basic parameters based on
+	/// CLI options. Override if your test case needs extra configuration.
+	virtual void setupCompiler(CompilerStack& _compiler);
+
+	/// Executes the requested pipeline stages until @p m_targetStage is reached.
+	/// Stops at the first failed stage.
+	void executeCompilationPipeline();
+
+	std::vector<std::string> m_warningsToFilter = {"This is a pre-release compiler version"};
+	std::vector<std::string> m_messagesToCut = {"Source file requires different compiler version (current compiler is"};
+
 private:
 	mutable std::unique_ptr<solidity::frontend::CompilerStack> m_compiler;
+	PipelineStage m_targetStage = PipelineStage::Compilation;
 };
 
 // Asserts that the compilation down to typechecking

test/libsolidity/GasTest.cpp

@@ -97,13 +97,13 @@ void GasTest::printUpdatedExpectations(std::ostream& _stream, std::string const&
 	}
 }
 
-TestCase::TestResult GasTest::run(std::ostream& _stream, std::string const& _linePrefix, bool _formatted)
+void GasTest::setupCompiler(CompilerStack& _compiler)
 {
-	compiler().reset();
+
 	// Prerelease CBOR metadata varies in size due to changing version numbers and build dates.
 	// This leads to volatile creation cost estimates. Therefore we force the compiler to
 	// release mode for testing gas estimates.
-	compiler().setMetadataFormat(CompilerStack::MetadataFormat::NoMetadata);
+	_compiler.setMetadataFormat(CompilerStack::MetadataFormat::NoMetadata);
 	OptimiserSettings settings = m_optimise ? OptimiserSettings::standard() : OptimiserSettings::minimal();
 	if (m_optimiseYul)
 	{
@@ -111,10 +111,12 @@ TestCase::TestResult GasTest::run(std::ostream& _stream, std::string const& _lin
 		settings.optimizeStackAllocation = m_optimise;
 	}
 	settings.expectedExecutionsPerDeployment = m_optimiseRuns;
-	compiler().setOptimiserSettings(settings);
-	compiler().setSources({{"", withPreamble(m_source)}});
+	_compiler.setOptimiserSettings(settings);
+}
 
-	if (!compiler().parseAndAnalyze() || !compiler().compile())
+TestCase::TestResult GasTest::run(std::ostream& _stream, std::string const& _linePrefix, bool _formatted)
+{
+	if (!runFramework(withPreamble(m_source), PipelineStage::Compilation))
 	{
 		_stream << formatErrors(filteredErrors(), _formatted);
 		return TestResult::FatalError;

test/libsolidity/GasTest.h

@@ -44,6 +44,9 @@ public:
 	void printSource(std::ostream &_stream, std::string const &_linePrefix = "", bool _formatted = false) const override;
 	void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override;
 
+protected:
+	void setupCompiler(CompilerStack& _compiler) override;
+
 private:
 	void parseExpectations(std::istream& _stream);
 

test/libsolidity/MemoryGuardTest.cpp

@@ -36,13 +36,15 @@ using namespace solidity::frontend;
 using namespace solidity::frontend::test;
 using namespace yul;
 
+void MemoryGuardTest::setupCompiler(CompilerStack& _compiler)
+{
+	_compiler.setViaIR(true);
+	_compiler.setOptimiserSettings(OptimiserSettings::none());
+}
+
 TestCase::TestResult MemoryGuardTest::run(std::ostream& _stream, std::string const& _linePrefix, bool _formatted)
 {
-	compiler().reset();
-	compiler().setSources(StringMap{{"", m_source}});
-	compiler().setViaIR(true);
-	compiler().setOptimiserSettings(OptimiserSettings::none());
-	if (!compiler().compile())
+	if (!runFramework(m_source, PipelineStage::Compilation))
 	{
 		_stream << formatErrors(filteredErrors(), _formatted);
 		return TestResult::FatalError;

test/libsolidity/MemoryGuardTest.h

@@ -48,6 +48,9 @@ public:
 	}
 
 	TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
+
+protected:
+	void setupCompiler(CompilerStack& _compiler) override;
 };
 
 }

test/libsolidity/SMTCheckerTest.cpp

@@ -126,11 +126,11 @@ SMTCheckerTest::SMTCheckerTest(std::string const& _filename): SyntaxTest(_filena
 	m_modelCheckerSettings.bmcLoopIterations = std::optional<unsigned>{bmcLoopIterations};
 }
 
-void SMTCheckerTest::setupCompiler()
+void SMTCheckerTest::setupCompiler(CompilerStack& _compiler)
 {
-	SyntaxTest::setupCompiler();
+	SyntaxTest::setupCompiler(_compiler);
 
-	compiler().setModelCheckerSettings(m_modelCheckerSettings);
+	_compiler.setModelCheckerSettings(m_modelCheckerSettings);
 }
 
 void SMTCheckerTest::filterObtainedErrors()

test/libsolidity/SMTCheckerTest.h

@@ -38,7 +38,7 @@ public:
 	}
 	SMTCheckerTest(std::string const& _filename);
 
-	void setupCompiler() override;
+	void setupCompiler(CompilerStack& _compiler) override;
 	void filterObtainedErrors() override;
 
 	void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override;

test/libsolidity/SolidityCompiler.cpp

@@ -28,7 +28,17 @@
 namespace solidity::frontend::test
 {
 
-BOOST_FIXTURE_TEST_SUITE(SolidityCompiler, AnalysisFramework)
+class SolidityCompilerFixture: protected AnalysisFramework
+{
+	void setupCompiler(CompilerStack& _compiler) override
+	{
+		// FIXME: This test was probably supposed to respect CommonOptions::get().optimize but
+		// due to a bug it was always running with optimizer disabled and it does not pass with it.
+		_compiler.setOptimiserSettings(false);
+	}
+};
+
+BOOST_FIXTURE_TEST_SUITE(SolidityCompiler, SolidityCompilerFixture)
 
 BOOST_AUTO_TEST_CASE(does_not_include_creation_time_only_internal_functions)
 {
@@ -39,12 +49,13 @@ BOOST_AUTO_TEST_CASE(does_not_include_creation_time_only_internal_functions)
 			function f() internal { unchecked { for (uint i = 0; i < 10; ++i) x += 3 + i; } }
 		}
 	)";
-	compiler().setOptimiserSettings(solidity::test::CommonOptions::get().optimize);
-	BOOST_REQUIRE(success(sourceCode));
+
+	runFramework(sourceCode, PipelineStage::Compilation);
 	BOOST_REQUIRE_MESSAGE(
-		compiler().compile(),
+		pipelineSuccessful(),
 		"Contract compilation failed:\n" + formatErrors(filteredErrors(), true /* _colored */)
 	);
+
 	bytes const& creationBytecode = solidity::test::bytecodeSansMetadata(compiler().object("C").bytecode);
 	bytes const& runtimeBytecode = solidity::test::bytecodeSansMetadata(compiler().runtimeObject("C").bytecode);
 	BOOST_CHECK(creationBytecode.size() >= 90);

test/libsolidity/SyntaxTest.cpp

@@ -49,53 +49,51 @@ SyntaxTest::SyntaxTest(
 	m_optimiseYul = m_reader.boolSetting("optimize-yul", true);
 }
 
-void SyntaxTest::setupCompiler()
+void SyntaxTest::setupCompiler(CompilerStack& _compiler)
 {
-	compiler().reset();
-	compiler().setSources(withPreamble(m_sources.sources));
-	compiler().setEVMVersion(m_evmVersion);
-	compiler().setOptimiserSettings(
+	AnalysisFramework::setupCompiler(_compiler);
+
+	_compiler.setEVMVersion(m_evmVersion);
+	_compiler.setOptimiserSettings(
 		m_optimiseYul ?
 		OptimiserSettings::full() :
 		OptimiserSettings::minimal()
 	);
-	compiler().setMetadataFormat(CompilerStack::MetadataFormat::NoMetadata);
-	compiler().setMetadataHash(CompilerStack::MetadataHash::None);
+	_compiler.setMetadataFormat(CompilerStack::MetadataFormat::NoMetadata);
+	_compiler.setMetadataHash(CompilerStack::MetadataHash::None);
 }
 
 void SyntaxTest::parseAndAnalyze()
 {
-	setupCompiler();
-
-	if (compiler().parse() && compiler().analyze())
-		try
+	try
+	{
+		runFramework(withPreamble(m_sources.sources), PipelineStage::Compilation);
+		if (!pipelineSuccessful() && stageSuccessful(PipelineStage::Analysis))
 		{
-			if (!compiler().compile())
-			{
-				ErrorList const& errors = compiler().errors();
-				auto codeGeneretionErrorCount = count_if(errors.cbegin(), errors.cend(), [](auto const& error) {
-					return error->type() == Error::Type::CodeGenerationError;
-				});
-				auto errorCount = count_if(errors.cbegin(), errors.cend(), [](auto const& error) {
-					return Error::isError(error->type());
-				});
-				// failing compilation after successful analysis is a rare case,
-				// it assumes that errors contain exactly one error, and the error is of type Error::Type::CodeGenerationError
-				if (codeGeneretionErrorCount != 1 || errorCount != 1)
-					BOOST_THROW_EXCEPTION(runtime_error("Compilation failed even though analysis was successful."));
-			}
-		}
-		catch (UnimplementedFeatureError const& _e)
-		{
-			m_errorList.emplace_back(SyntaxTestError{
-				"UnimplementedFeatureError",
-				nullopt,
-				errorMessage(_e),
-				"",
-				-1,
-				-1
+			ErrorList const& errors = compiler().errors();
+			auto codeGeneretionErrorCount = count_if(errors.cbegin(), errors.cend(), [](auto const& error) {
+				return error->type() == Error::Type::CodeGenerationError;
 			});
+			auto errorCount = count_if(errors.cbegin(), errors.cend(), [](auto const& error) {
+				return Error::isError(error->type());
+			});
+			// failing compilation after successful analysis is a rare case,
+			// it assumes that errors contain exactly one error, and the error is of type Error::Type::CodeGenerationError
+			if (codeGeneretionErrorCount != 1 || errorCount != 1)
+				BOOST_THROW_EXCEPTION(runtime_error("Compilation failed even though analysis was successful."));
 		}
+	}
+	catch (UnimplementedFeatureError const& _e)
+	{
+		m_errorList.emplace_back(SyntaxTestError{
+			"UnimplementedFeatureError",
+			nullopt,
+			errorMessage(_e),
+			"",
+			-1,
+			-1
+		});
+	}
 
 	filterObtainedErrors();
 }
@@ -146,4 +144,3 @@ void SyntaxTest::filterObtainedErrors()
 		});
 	}
 }
-

test/libsolidity/SyntaxTest.h

@@ -48,7 +48,7 @@ public:
 	);
 
 protected:
-	virtual void setupCompiler();
+	void setupCompiler(CompilerStack& _compiler) override;
 	void parseAndAnalyze() override;
 	virtual void filterObtainedErrors();