Refactor ASTJSON tests to allow easier addition of variations.

test/libsolidity/ASTJSONTest.cpp

@@ -16,17 +16,20 @@
 */
 // SPDX-License-Identifier: GPL-3.0
 
-#include <boost/algorithm/string/replace.hpp>
-#include <test/libsolidity/ASTJSONTest.h>
-#include <test/Common.h>
-#include <libsolutil/AnsiColorized.h>
 #include <liblangutil/SourceReferenceFormatter.h>
 #include <libsolidity/ast/ASTJsonConverter.h>
-#include <libsolidity/interface/CompilerStack.h>
+#include <libsolutil/AnsiColorized.h>
+#include <libsolutil/CommonIO.h>
+
+#include <test/Common.h>
+#include <test/libsolidity/ASTJSONTest.h>
+
 #include <boost/algorithm/string.hpp>
 #include <boost/algorithm/string/predicate.hpp>
-#include <boost/throw_exception.hpp>
+#include <boost/algorithm/string/replace.hpp>
 #include <boost/test/unit_test.hpp>
+#include <boost/throw_exception.hpp>
+
 #include <fstream>
 #include <memory>
 #include <stdexcept>
@@ -72,8 +75,13 @@ ASTJSONTest::ASTJSONTest(string const& _filename)
 	if (!boost::algorithm::ends_with(_filename, ".sol"))
 		BOOST_THROW_EXCEPTION(runtime_error("Invalid test contract file name: \"" + _filename + "\"."));
 
-	m_astFilename = _filename.substr(0, _filename.size() - 4) + ".json";
+	string_view baseName = _filename;
-	m_astParseOnlyFilename = _filename.substr(0, _filename.size() - 4) + "_parseOnly.json";
+	baseName.remove_suffix(4);
+
+	m_variants = {
+		TestVariant(baseName, CompilerStack::State::Parsed),
+		TestVariant(baseName, CompilerStack::State::AnalysisPerformed),
+	};
 
 	ifstream file(_filename);
 	if (!file)
@@ -102,26 +110,13 @@ ASTJSONTest::ASTJSONTest(string const& _filename)
 	}
 
 	m_sources.emplace_back(sourceName.empty() ? "a" : sourceName, source);
-
 	file.close();
-	file.open(m_astFilename);
+
-	if (file)
+	for (TestVariant& variant: m_variants)
 	{
-		string line;
+		variant.expectation = readFileAsString(variant.astFilename());
-		while (getline(file, line))
+		boost::replace_all(variant.expectation, "\r\n", "\n");
-			m_expectation += line + "\n";
 	}
-
-	file.close();
-	file.open(m_astParseOnlyFilename);
-	if (file)
-	{
-		string line;
-		while (getline(file, line))
-			m_expectationParseOnly += line + "\n";
-	}
-
-	file.close();
 }
 
 TestCase::TestResult ASTJSONTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
@@ -135,97 +130,76 @@ TestCase::TestResult ASTJSONTest::run(ostream& _stream, string const& _linePrefi
 		sources[m_sources[i].first] = m_sources[i].second;
 		sourceIndices[m_sources[i].first] = static_cast<unsigned>(i + 1);
 	}
-	c.setSources(sources);
-	c.setEVMVersion(solidity::test::CommonOptions::get().evmVersion());
 
+	bool resultsMatch = true;
 
-	if (!c.compile(CompilerStack::State::Parsed))
+	for (TestVariant& variant: m_variants)
 	{
-		SourceReferenceFormatter formatter(_stream, c, _formatted, false);
+		c.reset();
-		formatter.printErrorInformation(c.errors());
+		c.setSources(sources);
-		return TestResult::FatalError;
+		c.setEVMVersion(solidity::test::CommonOptions::get().evmVersion());
+
+		if (!c.parseAndAnalyze(variant.stopAfter))
+		{
+			// Ignore non-fatal analysis errors, we only want to export.
+			if (c.state() > CompilerStack::State::Parsed)
+				continue;
+
+			SourceReferenceFormatter formatter(_stream, c, _formatted, false);
+			formatter.printErrorInformation(c.errors());
+			return TestResult::FatalError;
+		}
+
+		resultsMatch = resultsMatch && runTest(
+			variant,
+			sourceIndices,
+			c,
+			_stream,
+			_linePrefix,
+			_formatted
+		);
 	}
 
-	bool resultsMatch = runTest(
-		m_expectationParseOnly,
-		m_resultParseOnly,
-		sourceIndices,
-		c,
-		"parseOnly",
-		_stream,
-		_linePrefix,
-		_formatted
-	);
-
-	c.reset();
-	c.setSources(sources);
-	c.setEVMVersion(solidity::test::CommonOptions::get().evmVersion());
-	if (!c.parse())
-	{
-		// Empty Expectations means we expect failure
-		if (m_expectation.empty())
-			return resultsMatch ? TestResult::Success : TestResult::Failure;
-
-		SourceReferenceFormatter{_stream, c, _formatted, false}
-			.printErrorInformation(c.errors());
-		return TestResult::FatalError;
-	}
-
-	c.analyze();
-
-	resultsMatch = runTest(
-		m_expectation,
-		m_result,
-		sourceIndices,
-		c,
-		"",
-		_stream,
-		_linePrefix,
-		_formatted
-	) && resultsMatch;
-
 	return resultsMatch ? TestResult::Success : TestResult::Failure;
 }
 
 bool ASTJSONTest::runTest(
-	string& _expectation,
+	TestVariant& _variant,
-	string& _result,
 	map<string, unsigned> const& _sourceIndices,
 	CompilerStack& _compiler,
-	string const& _variation,
 	ostream& _stream,
 	string const& _linePrefix,
 	bool const _formatted
 )
 {
 	if (m_sources.size() > 1)
-		_result += "[\n";
+		_variant.result += "[\n";
 
 	for (size_t i = 0; i < m_sources.size(); i++)
 	{
 		ostringstream result;
 		ASTJsonConverter(_compiler.state(), _sourceIndices).print(result, _compiler.ast(m_sources[i].first));
-		_result += result.str();
+		_variant.result += result.str();
 		if (i != m_sources.size() - 1)
-			_result += ",";
+			_variant.result += ",";
-		_result += "\n";
+		_variant.result += "\n";
 	}
 
 	if (m_sources.size() > 1)
-		_result += "]\n";
+		_variant.result += "]\n";
 
-	replaceTagWithVersion(_expectation);
+	replaceTagWithVersion(_variant.expectation);
 
-	if (_expectation != _result)
+	if (_variant.expectation != _variant.result)
 	{
 		string nextIndentLevel = _linePrefix + "  ";
 		AnsiColorized(_stream, _formatted, {BOLD, CYAN}) <<
 			_linePrefix <<
 			"Expected result" <<
-			(!_variation.empty() ? " (" + _variation + "):" : ":") <<
+			(!_variant.name().empty() ? " (" + _variant.name() + "):" : ":") <<
 			endl;
 		{
-			istringstream stream(_expectation);
+			istringstream stream(_variant.expectation);
 			string line;
 			while (getline(stream, line))
 				_stream << nextIndentLevel << line << endl;
@@ -235,10 +209,10 @@ bool ASTJSONTest::runTest(
 		AnsiColorized(_stream, _formatted, {BOLD, CYAN}) <<
 			_linePrefix <<
 			"Obtained result" <<
-			(!_variation.empty() ? " (" + _variation + "):" : ":") <<
+			(!_variant.name().empty() ? " (" + _variant.name() + "):" : ":") <<
 			endl;
 		{
-			istringstream stream(_result);
+			istringstream stream(_variant.result);
 			string line;
 			while (getline(stream, line))
 				_stream << nextIndentLevel << line << endl;
@@ -266,14 +240,18 @@ void ASTJSONTest::printSource(ostream& _stream, string const& _linePrefix, bool
 
 void ASTJSONTest::printUpdatedExpectations(std::ostream&, std::string const&) const
 {
-	updateExpectation(m_astFilename, m_result, "");
+	for (TestVariant const& variant: m_variants)
-	updateExpectation(m_astParseOnlyFilename, m_resultParseOnly, "parseOnly ");
+		updateExpectation(
+			variant.astFilename(),
+			variant.result,
+			variant.name().empty() ? "" : variant.name() + " "
+		);
 }
 
-void ASTJSONTest::updateExpectation(string const& _filename, string const& _expectation, string const& _variation) const
+void ASTJSONTest::updateExpectation(string const& _filename, string const& _expectation, string const& _variant) const
 {
 	ofstream file(_filename.c_str());
-	if (!file) BOOST_THROW_EXCEPTION(runtime_error("Cannot write " + _variation + "AST expectation to \"" + _filename + "\"."));
+	if (!file) BOOST_THROW_EXCEPTION(runtime_error("Cannot write " + _variant + "AST expectation to \"" + _filename + "\"."));
 	file.exceptions(ios::badbit);
 
 	string replacedResult = _expectation;

test/libsolidity/ASTJSONTest.h

@@ -19,6 +19,7 @@
 #pragma once
 
 #include <libsolutil/AnsiColorized.h>
+#include <libsolidity/interface/CompilerStack.h>
 #include <test/TestCase.h>
 
 #include <iosfwd>
@@ -37,6 +38,32 @@ namespace solidity::frontend::test
 class ASTJSONTest: public TestCase
 {
 public:
+	struct TestVariant
+	{
+		TestVariant(std::string_view _baseName, CompilerStack::State _stopAfter):
+			baseName(_baseName),
+			stopAfter(_stopAfter)
+		{}
+
+		std::string name() const
+		{
+			return stopAfter == CompilerStack::State::Parsed ? "parseOnly" : "";
+		}
+
+		std::string astFilename() const
+		{
+			return std::string(baseName) +
+				(name().empty() ? "" : "_") +
+				name() +
+				".json";
+		}
+
+		std::string baseName;
+		CompilerStack::State stopAfter;
+		std::string result;
+		std::string expectation;
+	};
+
 	static std::unique_ptr<TestCase> create(Config const& _config)
 	{
 		return std::make_unique<ASTJSONTest>(_config.filename);
@@ -49,11 +76,9 @@ public:
 	void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override;
 private:
 	bool runTest(
-		std::string& _expectation,
+		TestVariant& _testVariant,
-		std::string& _result,
 		std::map<std::string, unsigned> const& _sourceIndices,
 		CompilerStack& _compiler,
-		std::string const& _variation,
 		std::ostream& _stream,
 		std::string const& _linePrefix = "",
 		bool const _formatted = false
@@ -61,15 +86,12 @@ private:
 	void updateExpectation(
 		std::string const& _filename,
 		std::string const& _expectation,
-		std::string const& _variation
+		std::string const& _variant
 	) const;
 
+	std::vector<TestVariant> m_variants;
+
 	std::vector<std::pair<std::string, std::string>> m_sources;
-	std::string m_expectationParseOnly;
-	std::string m_astFilename;
-	std::string m_astParseOnlyFilename;
-	std::string m_result;
-	std::string m_resultParseOnly;
 };
 
 }