Store generated sources in standard-json output.

libsolidity/codegen/Compiler.cpp

@@ -51,8 +51,8 @@ void Compiler::compileContract(
 
 	m_context.optimise(m_optimiserSettings);
 
-	solAssert(m_context.requestedYulFunctionsRan(), "requestedYulFunctions() was not called.");
-	solAssert(m_runtimeContext.requestedYulFunctionsRan(), "requestedYulFunctions() was not called.");
+	solAssert(m_context.appendYulUtilityFunctionsRan(), "appendYulUtilityFunctions() was not called.");
+	solAssert(m_runtimeContext.appendYulUtilityFunctionsRan(), "appendYulUtilityFunctions() was not called.");
 }
 
 std::shared_ptr<evmasm::Assembly> Compiler::runtimeAssemblyPtr() const

libsolidity/codegen/Compiler.h

@@ -74,6 +74,9 @@ public:
 	/// @returns Assembly items of the runtime compiler context
 	evmasm::AssemblyItems const& runtimeAssemblyItems() const { return m_context.assembly().sub(m_runtimeSub).items(); }
 
+	std::string generatedYulUtilityCode() const { return m_context.generatedYulUtilityCode(); }
+	std::string runtimeGeneratedYulUtilityCode() const { return m_runtimeContext.generatedYulUtilityCode(); }
+
 	/// @returns the entry label of the given function. Might return an AssemblyItem of type
 	/// UndefinedItem if it does not exist yet.
 	evmasm::AssemblyItem functionEntryLabel(FunctionDefinition const& _function) const;

libsolidity/codegen/CompilerContext.cpp

@@ -37,6 +37,7 @@
 #include <libyul/optimiser/Suite.h>
 #include <libyul/Object.h>
 #include <libyul/YulString.h>
+#include <libyul/Utilities.h>
 
 #include <libsolutil/Whiskers.h>
 
@@ -191,14 +192,25 @@ void CompilerContext::appendMissingLowLevelFunctions()
 	}
 }
 
-pair<string, set<string>> CompilerContext::requestedYulFunctions()
+void CompilerContext::appendYulUtilityFunctions(OptimiserSettings const& _optimiserSettings)
 {
-	solAssert(!m_requestedYulFunctionsRan, "requestedYulFunctions called more than once.");
-	m_requestedYulFunctionsRan = true;
+	solAssert(!m_appendYulUtilityFunctionsRan, "requestedYulFunctions called more than once.");
+	m_appendYulUtilityFunctionsRan = true;
 
-	set<string> empty;
-	swap(empty, m_externallyUsedYulFunctions);
-	return {m_yulFunctionCollector.requestedFunctions(), std::move(empty)};
+	string code = m_yulFunctionCollector.requestedFunctions();
+	if (!code.empty())
+	{
+		m_generatedYulUtilityCode = yul::reindent("{\n" + move(code) + "\n}");
+
+		appendInlineAssembly(
+			m_generatedYulUtilityCode,
+			{},
+			m_externallyUsedYulFunctions,
+			true,
+			_optimiserSettings,
+			yulUtilityFileName()
+		);
+	}
 }
 
 void CompilerContext::addVariable(
@@ -369,7 +381,8 @@ void CompilerContext::appendInlineAssembly(
 	vector<string> const& _localVariables,
 	set<string> const& _externallyUsedFunctions,
 	bool _system,
-	OptimiserSettings const& _optimiserSettings
+	OptimiserSettings const& _optimiserSettings,
+	string _sourceName
 )
 {
 	unsigned startStackHeight = stackHeight();
@@ -420,7 +433,7 @@ void CompilerContext::appendInlineAssembly(
 
 	ErrorList errors;
 	ErrorReporter errorReporter(errors);
-	auto scanner = make_shared<langutil::Scanner>(langutil::CharStream(_assembly, "--CODEGEN--"));
+	auto scanner = make_shared<langutil::Scanner>(langutil::CharStream(_assembly, _sourceName));
 	yul::EVMDialect const& dialect = yul::EVMDialect::strictAssemblyForEVM(m_evmVersion);
 	optional<langutil::SourceLocation> locationOverride;
 	if (!_system)

libsolidity/codegen/CompilerContext.h

@@ -163,12 +163,14 @@ public:
 	void appendMissingLowLevelFunctions();
 	ABIFunctions& abiFunctions() { return m_abiFunctions; }
 	YulUtilFunctions& utilFunctions() { return m_yulUtilFunctions; }
-	/// @returns concatenation of all generated functions and a set of the
-	/// externally used functions.
-	/// Clears the internal list, i.e. calling it again will result in an
-	/// empty return value.
-	std::pair<std::string, std::set<std::string>> requestedYulFunctions();
-	bool requestedYulFunctionsRan() const { return m_requestedYulFunctionsRan; }
+
+	/// Appends concatenation of all generated Yul functions to the bytecode
+	/// and stores the Yul source code to be returned by @a generatedYulUtilityCode.
+	/// Should be called exactly once on each context.
+	void appendYulUtilityFunctions(OptimiserSettings const& _optimiserSettings);
+	bool appendYulUtilityFunctionsRan() const { return m_appendYulUtilityFunctionsRan; }
+	std::string const& generatedYulUtilityCode() const { return m_generatedYulUtilityCode; }
+	static std::string yulUtilityFileName() { return "#utility.yul"; }
 
 	/// Returns the distance of the given local variable from the bottom of the stack (of the current function).
 	unsigned baseStackOffsetOfVariable(Declaration const& _declaration) const;
@@ -246,17 +248,21 @@ public:
 	CompilerContext& operator<<(u256 const& _value) { m_asm->append(_value); return *this; }
 	CompilerContext& operator<<(bytes const& _data) { m_asm->append(_data); return *this; }
 
-	/// Appends inline assembly (strict mode).
-	/// @a _replacements are string-matching replacements that are performed prior to parsing the inline assembly.
+	/// Appends inline assembly (strict-EVM dialect for the current version).
+	/// @param _assembly the assembly text, should be a block.
 	/// @param _localVariables assigns stack positions to variables with the last one being the stack top
 	/// @param _externallyUsedFunctions a set of function names that are not to be renamed or removed.
-	/// @param _system if true, this is a "system-level" assembly where all functions use named labels.
+	/// @param _system if true, this is a "system-level" assembly where all functions use named labels
+	///                and the code is marked to be exported as "compiler-generated assembly utility file".
+	/// @param _optimiserSettings settings for the Yul optimiser, which is run in this function already.
+	/// @param _sourceName the name of the assembly file to be used for source locations
 	void appendInlineAssembly(
 		std::string const& _assembly,
 		std::vector<std::string> const& _localVariables = std::vector<std::string>(),
 		std::set<std::string> const& _externallyUsedFunctions = std::set<std::string>(),
 		bool _system = false,
-		OptimiserSettings const& _optimiserSettings = OptimiserSettings::none()
+		OptimiserSettings const& _optimiserSettings = OptimiserSettings::none(),
+		std::string _sourceName = "--CODEGEN--"
 	);
 
 	/// If m_revertStrings is debug, @returns inline assembly code that
@@ -385,14 +391,17 @@ private:
 	MultiUseYulFunctionCollector m_yulFunctionCollector;
 	/// Set of externally used yul functions.
 	std::set<std::string> m_externallyUsedYulFunctions;
+	/// Generated Yul code used as utility. Source references from the bytecode can point here.
+	/// Produced from @a m_yulFunctionCollector.
+	std::string m_generatedYulUtilityCode;
 	/// Container for ABI functions to be generated.
 	ABIFunctions m_abiFunctions;
 	/// Container for Yul Util functions to be generated.
 	YulUtilFunctions m_yulUtilFunctions;
 	/// The queue of low-level functions to generate.
 	std::queue<std::tuple<std::string, unsigned, unsigned, std::function<void(CompilerContext&)>>> m_lowLevelFunctionGenerationQueue;
-	/// Flag to check that requestedYulFunctions() was called exactly once
-	bool m_requestedYulFunctionsRan = false;
+	/// Flag to check that appendYulUtilityFunctions() was called exactly once
+	bool m_appendYulUtilityFunctionsRan = false;
 };
 
 }

libsolidity/codegen/ContractCompiler.cpp

@@ -1272,15 +1272,7 @@ void ContractCompiler::appendMissingFunctions()
 		solAssert(m_context.nextFunctionToCompile() != function, "Compiled the wrong function?");
 	}
 	m_context.appendMissingLowLevelFunctions();
-	auto [yulFunctions, externallyUsedYulFunctions] = m_context.requestedYulFunctions();
-	if (!yulFunctions.empty())
-		m_context.appendInlineAssembly(
-			"{" + move(yulFunctions) + "}",
-			{},
-			externallyUsedYulFunctions,
-			true,
-			m_optimiserSettings
-		);
+	m_context.appendYulUtilityFunctions(m_optimiserSettings);
 }
 
 void ContractCompiler::appendModifierOrFunctionCode()

libsolidity/codegen/ContractCompiler.h

@@ -143,6 +143,7 @@ private:
 	/// Pointer to the runtime compiler in case this is a creation compiler.
 	ContractCompiler* m_runtimeCompiler = nullptr;
 	CompilerContext& m_context;
+
 	/// Tag to jump to for a "break" statement and the stack height after freeing the local loop variables.
 	std::vector<std::pair<evmasm::AssemblyItem, unsigned>> m_breakTags;
 	/// Tag to jump to for a "continue" statement and the stack height after freeing the local loop variables.

libsolidity/interface/CompilerStack.cpp

@@ -57,7 +57,9 @@
 
 #include <libyul/YulString.h>
 #include <libyul/AsmPrinter.h>
+#include <libyul/AsmJsonConverter.h>
 #include <libyul/AssemblyStack.h>
+#include <libyul/AsmParser.h>
 
 #include <liblangutil/Scanner.h>
 #include <liblangutil/SemVerHandler.h>
@@ -591,6 +593,48 @@ evmasm::AssemblyItems const* CompilerStack::runtimeAssemblyItems(string const& _
 	return currentContract.compiler ? &contract(_contractName).compiler->runtimeAssemblyItems() : nullptr;
 }
 
+Json::Value CompilerStack::generatedSources(string const& _contractName, bool _runtime) const
+{
+	if (m_stackState != CompilationSuccessful)
+		BOOST_THROW_EXCEPTION(CompilerError() << errinfo_comment("Compilation was not successful."));
+
+	Contract const& c = contract(_contractName);
+	util::LazyInit<Json::Value const> const& sources =
+		_runtime ?
+		c.runtimeGeneratedSources :
+		c.generatedSources;
+	return sources.init([&]{
+		Json::Value sources{Json::arrayValue};
+		// If there is no compiler, then no bytecode was generated and thus no
+		// sources were generated.
+		if (c.compiler)
+		{
+			string source =
+				_runtime ?
+				c.compiler->runtimeGeneratedYulUtilityCode() :
+				c.compiler->generatedYulUtilityCode();
+			if (!source.empty())
+			{
+				string sourceName = CompilerContext::yulUtilityFileName();
+				unsigned sourceIndex = sourceIndices()[sourceName];
+				ErrorList errors;
+				ErrorReporter errorReporter(errors);
+				auto scanner = make_shared<langutil::Scanner>(langutil::CharStream(source, sourceName));
+				yul::EVMDialect const& dialect = yul::EVMDialect::strictAssemblyForEVM(m_evmVersion);
+				shared_ptr<yul::Block> parserResult = yul::Parser{errorReporter, dialect}.parse(scanner, false);
+				solAssert(parserResult, "");
+				sources[0]["ast"] = yul::AsmJsonConverter{sourceIndex}(*parserResult);
+				sources[0]["name"] = sourceName;
+				sources[0]["id"] = sourceIndex;
+				sources[0]["language"] = "Yul";
+				sources[0]["contents"] = move(source);
+
+			}
+		}
+		return sources;
+	});
+}
+
 string const* CompilerStack::sourceMapping(string const& _contractName) const
 {
 	if (m_stackState != CompilationSuccessful)
@@ -733,6 +777,8 @@ map<string, unsigned> CompilerStack::sourceIndices() const
 	unsigned index = 0;
 	for (auto const& s: m_sources)
 		indices[s.first] = index++;
+	solAssert(!indices.count(CompilerContext::yulUtilityFileName()), "");
+	indices[CompilerContext::yulUtilityFileName()] = index++;
 	return indices;
 }
 

libsolidity/interface/CompilerStack.h

@@ -277,6 +277,10 @@ public:
 	/// @returns runtime contract assembly items
 	evmasm::AssemblyItems const* runtimeAssemblyItems(std::string const& _contractName) const;
 
+	/// @returns an array containing all utility sources generated during compilation.
+	/// Format: [ { name: string, id: number, language: "Yul", contents: string }, ... ]
+	Json::Value generatedSources(std::string const& _contractName, bool _runtime = false) const;
+
 	/// @returns the string that provides a mapping between bytecode and sourcecode or a nullptr
 	/// if the contract does not (yet) have bytecode.
 	std::string const* sourceMapping(std::string const& _contractName) const;
@@ -356,6 +360,8 @@ private:
 		util::LazyInit<Json::Value const> storageLayout;
 		util::LazyInit<Json::Value const> userDocumentation;
 		util::LazyInit<Json::Value const> devDocumentation;
+		util::LazyInit<Json::Value const> generatedSources;
+		util::LazyInit<Json::Value const> runtimeGeneratedSources;
 		mutable std::optional<std::string const> sourceMapping;
 		mutable std::optional<std::string const> runtimeSourceMapping;
 	};

libsolidity/interface/StandardCompiler.cpp

@@ -241,10 +241,12 @@ bool isBinaryRequested(Json::Value const& _outputSelection)
 		"*",
 		"ir", "irOptimized",
 		"wast", "wasm", "ewasm.wast", "ewasm.wasm",
-		"evm.deployedBytecode", "evm.deployedBytecode.object", "evm.deployedBytecode.opcodes",
+		"evm.deployedBytecode", "evm.deployedBytecode.generatedSources",
+		"evm.deployedBytecode.object", "evm.deployedBytecode.opcodes",
 		"evm.deployedBytecode.sourceMap", "evm.deployedBytecode.linkReferences",
 		"evm.deployedBytecode.immutableReferences",
-		"evm.bytecode", "evm.bytecode.object", "evm.bytecode.opcodes", "evm.bytecode.sourceMap",
+		"evm.bytecode", "evm.bytecode.generatedSources",
+		"evm.bytecode.object", "evm.bytecode.opcodes", "evm.bytecode.sourceMap",
 		"evm.bytecode.linkReferences",
 		"evm.gasEstimates", "evm.legacyAssembly", "evm.assembly"
 	};
@@ -364,7 +366,12 @@ Json::Value formatImmutableReferences(map<u256, pair<string, vector<size_t>>> co
 	return ret;
 }
 
-Json::Value collectEVMObject(evmasm::LinkerObject const& _object, string const* _sourceMap, bool _runtimeObject)
+Json::Value collectEVMObject(
+	evmasm::LinkerObject const& _object,
+	string const* _sourceMap,
+	Json::Value _generatedSources,
+	bool _runtimeObject
+)
 {
 	Json::Value output = Json::objectValue;
 	output["object"] = _object.toHex();
@@ -373,6 +380,7 @@ Json::Value collectEVMObject(evmasm::LinkerObject const& _object, string const*
 	output["linkReferences"] = formatLinkReferences(_object.linkReferences);
 	if (_runtimeObject)
 		output["immutableReferences"] = formatImmutableReferences(_object.immutableReferences);
+	output["generatedSources"] = move(_generatedSources);
 	return output;
 }
 
@@ -1074,12 +1082,13 @@ Json::Value StandardCompiler::compileSolidity(StandardCompiler::InputsAndSetting
 			_inputsAndSettings.outputSelection,
 			file,
 			name,
-			{ "evm.bytecode", "evm.bytecode.object", "evm.bytecode.opcodes", "evm.bytecode.sourceMap", "evm.bytecode.linkReferences" },
+			{ "evm.bytecode", "evm.bytecode.object", "evm.bytecode.opcodes", "evm.bytecode.generatedSources", "evm.bytecode.sourceMap", "evm.bytecode.linkReferences" },
 			wildcardMatchesExperimental
 		))
 			evmData["bytecode"] = collectEVMObject(
 				compilerStack.object(contractName),
 				compilerStack.sourceMapping(contractName),
+				compilerStack.generatedSources(contractName),
 				false
 			);
 
@@ -1087,12 +1096,13 @@ Json::Value StandardCompiler::compileSolidity(StandardCompiler::InputsAndSetting
 			_inputsAndSettings.outputSelection,
 			file,
 			name,
-			{ "evm.deployedBytecode", "evm.deployedBytecode.object", "evm.deployedBytecode.opcodes", "evm.deployedBytecode.sourceMap", "evm.deployedBytecode.linkReferences", "evm.deployedBytecode.immutableReferences" },
+			{ "evm.deployedBytecode", "evm.deployedBytecode.object", "evm.deployedBytecode.opcodes", "evm.deployedBytecode.generatedSources", "evm.deployedBytecode.sourceMap", "evm.deployedBytecode.linkReferences", "evm.deployedBytecode.immutableReferences" },
 			wildcardMatchesExperimental
 		))
 			evmData["deployedBytecode"] = collectEVMObject(
 				compilerStack.runtimeObject(contractName),
 				compilerStack.runtimeSourceMapping(contractName),
+				compilerStack.generatedSources(contractName, true),
 				true
 			);
 
@@ -1178,7 +1188,7 @@ Json::Value StandardCompiler::compileYul(InputsAndSettings _inputsAndSettings)
 	for (string const& objectKind: vector<string>{"bytecode", "deployedBytecode"})
 	{
 		auto artifacts = util::applyMap(
-			vector<string>{"", ".object", ".opcodes", ".sourceMap", ".linkReferences"},
+			vector<string>{"", ".object", ".opcodes", ".sourceMap", ".generatedSources", ".linkReferences"},
 			[&](auto const& _s) { return "evm." + objectKind + _s; }
 		);
 		if (isArtifactRequested(
@@ -1191,7 +1201,8 @@ Json::Value StandardCompiler::compileYul(InputsAndSettings _inputsAndSettings)
 		{
 			MachineAssemblyObject const& o = objectKind == "bytecode" ? object : runtimeObject;
 			if (o.bytecode)
-				output["contracts"][sourceName][contractName]["evm"][objectKind] = collectEVMObject(*o.bytecode, o.sourceMappings.get(), false);
+				output["contracts"][sourceName][contractName]["evm"][objectKind] =
+					collectEVMObject(*o.bytecode, o.sourceMappings.get(), Json::arrayValue, false);
 		}
 	}
 

solc/CommandLineInterface.cpp

@@ -127,6 +127,8 @@ static string const g_strEVM = "evm";
 static string const g_strEVM15 = "evm15";
 static string const g_strEVMVersion = "evm-version";
 static string const g_strEwasm = "ewasm";
+static string const g_strGeneratedSources = "generated-sources";
+static string const g_strGeneratedSourcesRuntime = "generated-sources-runtime";
 static string const g_strGas = "gas";
 static string const g_strHelp = "help";
 static string const g_strImportAst = "import-ast";
@@ -238,6 +240,8 @@ static set<string> const g_combinedJsonArgs
 	g_strBinary,
 	g_strBinaryRuntime,
 	g_strCompactJSON,
+	g_strGeneratedSources,
+	g_strGeneratedSourcesRuntime,
 	g_strInterface,
 	g_strMetadata,
 	g_strNatspecUser,
@@ -1523,6 +1527,10 @@ void CommandLineInterface::handleCombinedJSON()
 			contractData[g_strAsm] = m_compiler->assemblyJSON(contractName);
 		if (requests.count(g_strStorageLayout) && m_compiler->compilationSuccessful())
 			contractData[g_strStorageLayout] = jsonCompactPrint(m_compiler->storageLayout(contractName));
+		if (requests.count(g_strGeneratedSources) && m_compiler->compilationSuccessful())
+			contractData[g_strGeneratedSources] = m_compiler->generatedSources(contractName, false);
+		if (requests.count(g_strGeneratedSourcesRuntime) && m_compiler->compilationSuccessful())
+			contractData[g_strGeneratedSourcesRuntime] = m_compiler->generatedSources(contractName, true);
 		if (requests.count(g_strSrcMap) && m_compiler->compilationSuccessful())
 		{
 			auto map = m_compiler->sourceMapping(contractName);