Implement lazy function generation using function generation queue

libsolidity/codegen/MultiUseYulFunctionCollector.cpp

@@ -34,6 +34,7 @@ string MultiUseYulFunctionCollector::requestedFunctions()
 {
 	string result;
 	for (auto const& f: m_requestedFunctions)
+		// std::map guarantees ascending order when iterating through its keys.
 		result += f.second;
 	m_requestedFunctions.clear();
 	return result;

libsolidity/codegen/MultiUseYulFunctionCollector.h

@@ -41,10 +41,15 @@ public:
 	std::string createFunction(std::string const& _name, std::function<std::string()> const& _creator);
 
 	/// @returns concatenation of all generated functions.
+	/// Guarantees that the order of functions in the generated code is deterministic and
+	/// platform-independent.
 	/// Clears the internal list, i.e. calling it again will result in an
 	/// empty return value.
 	std::string requestedFunctions();
 
+	/// @returns true IFF a function with the specified name has already been collected.
+	bool contains(std::string const& _name) const { return m_requestedFunctions.count(_name) > 0; }
+
 private:
 	/// Map from function name to code for a multi-use function.
 	std::map<std::string, std::string> m_requestedFunctions;

libsolidity/codegen/ir/IRGenerationContext.cpp

@@ -32,6 +32,25 @@ using namespace solidity;
 using namespace solidity::util;
 using namespace solidity::frontend;
 
+string IRGenerationContext::enqueueFunctionForCodeGeneration(FunctionDefinition const& _function)
+{
+	string name = functionName(_function);
+
+	if (!m_functions.contains(name))
+		m_functionGenerationQueue.insert(&_function);
+
+	return name;
+}
+
+FunctionDefinition const* IRGenerationContext::dequeueFunctionForCodeGeneration()
+{
+	solAssert(!m_functionGenerationQueue.empty(), "");
+
+	FunctionDefinition const* result = *m_functionGenerationQueue.begin();
+	m_functionGenerationQueue.erase(m_functionGenerationQueue.begin());
+	return result;
+}
+
 ContractDefinition const& IRGenerationContext::mostDerivedContract() const
 {
 	solAssert(m_mostDerivedContract, "Most derived contract requested but not set.");
@@ -77,11 +96,6 @@ string IRGenerationContext::functionName(VariableDeclaration const& _varDecl)
 	return "getter_fun_" + _varDecl.name() + "_" + to_string(_varDecl.id());
 }
 
-string IRGenerationContext::virtualFunctionName(FunctionDefinition const& _functionDeclaration)
-{
-	return functionName(_functionDeclaration.resolveVirtual(mostDerivedContract()));
-}
-
 string IRGenerationContext::newYulVariable()
 {
 	return "_" + to_string(++m_varCounter);
@@ -137,6 +151,8 @@ string IRGenerationContext::internalDispatch(size_t _in, size_t _out)
 						{ "funID", to_string(function->id()) },
 						{ "name", functionName(*function)}
 					});
+
+					enqueueFunctionForCodeGeneration(*function);
 				}
 		templ("cases", move(functions));
 		return templ.render();

libsolidity/codegen/ir/IRGenerationContext.h

@@ -30,6 +30,7 @@
 
 #include <libsolutil/Common.h>
 
+#include <set>
 #include <string>
 #include <memory>
 #include <vector>
@@ -61,6 +62,15 @@ public:
 
 	MultiUseYulFunctionCollector& functionCollector() { return m_functions; }
 
+	/// Adds a Solidity function to the function generation queue and returns the name of the
+	/// corresponding Yul function.
+	std::string enqueueFunctionForCodeGeneration(FunctionDefinition const& _function);
+
+	/// Pops one item from the function generation queue. Must not be called if the queue is empty.
+	FunctionDefinition const* dequeueFunctionForCodeGeneration();
+
+	bool functionGenerationQueueEmpty() { return m_functionGenerationQueue.empty(); }
+
 	/// Sets the most derived contract (the one currently being compiled)>
 	void setMostDerivedContract(ContractDefinition const& _mostDerivedContract)
 	{
@@ -82,7 +92,6 @@ public:
 
 	std::string functionName(FunctionDefinition const& _function);
 	std::string functionName(VariableDeclaration const& _varDecl);
-	std::string virtualFunctionName(FunctionDefinition const& _functionDeclaration);
 
 	std::string newYulVariable();
 
@@ -113,6 +122,15 @@ private:
 	std::map<VariableDeclaration const*, std::pair<u256, unsigned>> m_stateVariables;
 	MultiUseYulFunctionCollector m_functions;
 	size_t m_varCounter = 0;
+
+	/// Function definitions queued for code generation. They're the Solidity functions whose calls
+	/// were discovered by the IR generator during AST traversal.
+	/// Note that the queue gets filled in a lazy way - new definitions can be added while the
+	/// collected ones get removed and traversed.
+	/// The order and duplicates are irrelevant here (hence std::set rather than std::queue) as
+	/// long as the order of Yul functions in the generated code is deterministic and the same on
+	/// all platforms - which is a property guaranteed by MultiUseYulFunctionCollector.
+	std::set<FunctionDefinition const*> m_functionGenerationQueue;
 };
 
 }

libsolidity/codegen/ir/IRGenerator.cpp

@@ -101,20 +101,13 @@ string IRGenerator::generate(ContractDefinition const& _contract)
 	t("memoryInit", memoryInit());
 	t("constructor", constructorCode(_contract));
 	t("deploy", deployCode(_contract));
-	// We generate code for all functions and rely on the optimizer to remove them again
+	generateQueuedFunctions();
-	// TODO it would probably be better to only generate functions when internalDispatch or
-	// virtualFunctionName is called - same below.
-	for (auto const* contract: _contract.annotation().linearizedBaseContracts)
-		for (auto const* fun: contract->definedFunctions())
-			generateFunction(*fun);
 	t("functions", m_context.functionCollector().requestedFunctions());
 
 	resetContext(_contract);
 	t("RuntimeObject", runtimeObjectName(_contract));
 	t("dispatch", dispatchRoutine(_contract));
-	for (auto const* contract: _contract.annotation().linearizedBaseContracts)
+	generateQueuedFunctions();
-		for (auto const* fun: contract->definedFunctions())
-			generateFunction(*fun);
 	t("runtimeFunctions", m_context.functionCollector().requestedFunctions());
 	return t.render();
 }
@@ -126,6 +119,13 @@ string IRGenerator::generate(Block const& _block)
 	return generator.code();
 }
 
+void IRGenerator::generateQueuedFunctions()
+{
+	while (!m_context.functionGenerationQueueEmpty())
+		// NOTE: generateFunction() may modify function generation queue
+		generateFunction(*m_context.dequeueFunctionForCodeGeneration());
+}
+
 string IRGenerator::generateFunction(FunctionDefinition const& _function)
 {
 	string functionName = m_context.functionName(_function);
@@ -290,7 +290,7 @@ string IRGenerator::constructorCode(ContractDefinition const& _contract)
 		t("assignToParams", paramVars == 0 ? "" : "let " + suffixedVariableNameList("param_", 0, paramVars) + " := ");
 		t("params", suffixedVariableNameList("param_", 0, paramVars));
 		t("abiDecode", abiFunctions.tupleDecoder(constructor->functionType(false)->parameterTypes(), true));
-		t("constructorName", m_context.functionName(*constructor));
+		t("constructorName", m_context.enqueueFunctionForCodeGeneration(*constructor));
 
 		out << t.render();
 	}
@@ -369,7 +369,7 @@ string IRGenerator::dispatchRoutine(ContractDefinition const& _contract)
 		templ["retParams"] = suffixedVariableNameList("ret_", retVars, 0);
 
 		if (FunctionDefinition const* funDef = dynamic_cast<FunctionDefinition const*>(&type->declaration()))
-			templ["function"] = generateFunction(*funDef);
+			templ["function"] = m_context.enqueueFunctionForCodeGeneration(*funDef);
 		else if (VariableDeclaration const* varDecl = dynamic_cast<VariableDeclaration const*>(&type->declaration()))
 			templ["function"] = generateGetter(*varDecl);
 		else
@@ -385,14 +385,14 @@ string IRGenerator::dispatchRoutine(ContractDefinition const& _contract)
 		string fallbackCode;
 		if (!fallback->isPayable())
 			fallbackCode += callValueCheck();
-		fallbackCode += generateFunction(*fallback) + "() stop()";
+		fallbackCode += m_context.enqueueFunctionForCodeGeneration(*fallback) + "() stop()";
 
 		t("fallback", fallbackCode);
 	}
 	else
 		t("fallback", "revert(0, 0)");
 	if (FunctionDefinition const* etherReceiver = _contract.receiveFunction())
-		t("receiveEther", generateFunction(*etherReceiver) + "() stop()");
+		t("receiveEther", m_context.enqueueFunctionForCodeGeneration(*etherReceiver) + "() stop()");
 	else
 		t("receiveEther", "");
 	return t.render();
@@ -412,6 +412,10 @@ string IRGenerator::memoryInit()
 
 void IRGenerator::resetContext(ContractDefinition const& _contract)
 {
+	solAssert(
+		m_context.functionGenerationQueueEmpty(),
+		"Reset function generation queue while it still had functions."
+	);
 	solAssert(
 		m_context.functionCollector().requestedFunctions().empty(),
 		"Reset context while it still had functions."

libsolidity/codegen/ir/IRGenerator.h

@@ -56,6 +56,9 @@ private:
 	std::string generate(ContractDefinition const& _contract);
 	std::string generate(Block const& _block);
 
+	/// Generates code for all the functions from the function generation queue.
+	/// The resulting code is stored in the function collector in IRGenerationContext.
+	void generateQueuedFunctions();
 	/// Generates code for and returns the name of the function.
 	std::string generateFunction(FunctionDefinition const& _function);
 	/// Generates a getter for the given declaration and returns its name

libsolidity/codegen/ir/IRGeneratorForStatements.cpp

@@ -577,7 +577,9 @@ void IRGeneratorForStatements::endVisit(FunctionCall const& _functionCall)
 			if (auto functionDef = dynamic_cast<FunctionDefinition const*>(identifier->annotation().referencedDeclaration))
 			{
 				define(_functionCall) <<
-					m_context.virtualFunctionName(*functionDef) <<
+					m_context.enqueueFunctionForCodeGeneration(
+						functionDef->resolveVirtual(m_context.mostDerivedContract())
+					) <<
 					"(" <<
 					joinHumanReadable(args) <<
 					")\n";
@@ -586,6 +588,7 @@ void IRGeneratorForStatements::endVisit(FunctionCall const& _functionCall)
 		}
 
 		define(_functionCall) <<
+			// NOTE: internalDispatch() takes care of adding the function to function generation queue
 			m_context.internalDispatch(
 				TupleType(functionType->parameterTypes()).sizeOnStack(),
 				TupleType(functionType->returnParameterTypes()).sizeOnStack()

test/cmdlineTests/standard_irOptimized_requested/output.json

@@ -10,8 +10,6 @@ object \"C_6\" {
         mstore(64, 128)
         codecopy(0, dataoffset(\"C_6_deployed\"), datasize(\"C_6_deployed\"))
         return(0, datasize(\"C_6_deployed\"))
-        function fun_f_5()
-        { }
     }
     object \"C_6_deployed\" {
         code {

test/cmdlineTests/standard_ir_requested/output.json

@@ -18,11 +18,6 @@ object \"C_6\" {
         return(0, datasize(\"C_6_deployed\"))
 
 
-        function fun_f_5()  {
-
-
-        }
-
     }
     object \"C_6_deployed\" {
         code {

test/cmdlineTests/yul_string_format_ascii/output.json

@@ -18,35 +18,6 @@ object \"C_10\" {
         return(0, datasize(\"C_10_deployed\"))
 
 
-        function allocateMemory(size) -> memPtr {
-            memPtr := mload(64)
-            let newFreePtr := add(memPtr, size)
-            // protect against overflow
-            if or(gt(newFreePtr, 0xffffffffffffffff), lt(newFreePtr, memPtr)) { revert(0, 0) }
-            mstore(64, newFreePtr)
-        }
-
-        function convert_t_stringliteral_9f0adad0a59b05d2e04a1373342b10b9eb16c57c164c8a3bfcbf46dccee39a21_to_t_string_memory_ptr() -> converted {
-            converted := allocateMemory(64)
-            mstore(converted, 6)
-
-            mstore(add(converted, 32), \"abcabc\")
-
-        }
-
-        function fun_f_9()  -> vloc__4_mpos {
-            let zero_value_for_type_t_string_memory_ptr_1_mpos := zero_value_for_split_t_string_memory_ptr()
-            vloc__4_mpos := zero_value_for_type_t_string_memory_ptr_1_mpos
-
-            vloc__4_mpos := convert_t_stringliteral_9f0adad0a59b05d2e04a1373342b10b9eb16c57c164c8a3bfcbf46dccee39a21_to_t_string_memory_ptr()
-            leave
-
-        }
-
-        function zero_value_for_split_t_string_memory_ptr() -> ret {
-            ret := 96
-        }
-
     }
     object \"C_10_deployed\" {
         code {

test/cmdlineTests/yul_string_format_ascii_bytes32/output.json

@@ -18,23 +18,6 @@ object \"C_10\" {
         return(0, datasize(\"C_10_deployed\"))
 
 
-        function convert_t_stringliteral_9f0adad0a59b05d2e04a1373342b10b9eb16c57c164c8a3bfcbf46dccee39a21_to_t_bytes32() -> converted {
-            converted := 0x6162636162630000000000000000000000000000000000000000000000000000
-        }
-
-        function fun_f_9()  -> vloc__4 {
-            let zero_value_for_type_t_bytes32_1 := zero_value_for_split_t_bytes32()
-            vloc__4 := zero_value_for_type_t_bytes32_1
-
-            vloc__4 := convert_t_stringliteral_9f0adad0a59b05d2e04a1373342b10b9eb16c57c164c8a3bfcbf46dccee39a21_to_t_bytes32()
-            leave
-
-        }
-
-        function zero_value_for_split_t_bytes32() -> ret {
-            ret := 0
-        }
-
     }
     object \"C_10_deployed\" {
         code {

test/cmdlineTests/yul_string_format_ascii_bytes32_from_number/output.json

@@ -18,35 +18,6 @@ object \"C_10\" {
         return(0, datasize(\"C_10_deployed\"))
 
 
-        function cleanup_t_rational_1633837924_by_1(value) -> cleaned {
-            cleaned := value
-        }
-
-        function convert_t_rational_1633837924_by_1_to_t_bytes4(value) -> converted {
-            converted := shift_left_224(cleanup_t_rational_1633837924_by_1(value))
-        }
-
-        function fun_f_9()  -> vloc__4 {
-            let zero_value_for_type_t_bytes4_1 := zero_value_for_split_t_bytes4()
-            vloc__4 := zero_value_for_type_t_bytes4_1
-
-            let expr_6 := 0x61626364
-            vloc__4 := convert_t_rational_1633837924_by_1_to_t_bytes4(expr_6)
-            leave
-
-        }
-
-        function shift_left_224(value) -> newValue {
-            newValue :=
-
-            shl(224, value)
-
-        }
-
-        function zero_value_for_split_t_bytes4() -> ret {
-            ret := 0
-        }
-
     }
     object \"C_10_deployed\" {
         code {

test/cmdlineTests/yul_string_format_ascii_long/output.json

@@ -18,39 +18,6 @@ object \"C_10\" {
         return(0, datasize(\"C_10_deployed\"))
 
 
-        function allocateMemory(size) -> memPtr {
-            memPtr := mload(64)
-            let newFreePtr := add(memPtr, size)
-            // protect against overflow
-            if or(gt(newFreePtr, 0xffffffffffffffff), lt(newFreePtr, memPtr)) { revert(0, 0) }
-            mstore(64, newFreePtr)
-        }
-
-        function convert_t_stringliteral_d6604f85ac07e2b33103a620b3d3d75b0473c7214912beded67b9b624d41c571_to_t_string_memory_ptr() -> converted {
-            converted := allocateMemory(128)
-            mstore(converted, 85)
-
-            mstore(add(converted, 32), \"abcdabcdcafecafeabcdabcdcafecafe\")
-
-            mstore(add(converted, 64), \"ffffzzzzoooo0123456789,.<,>.?:;'\")
-
-            mstore(add(converted, 96), \"[{]}|`~!@#$%^&*()-_=+\")
-
-        }
-
-        function fun_f_9()  -> vloc__4_mpos {
-            let zero_value_for_type_t_string_memory_ptr_1_mpos := zero_value_for_split_t_string_memory_ptr()
-            vloc__4_mpos := zero_value_for_type_t_string_memory_ptr_1_mpos
-
-            vloc__4_mpos := convert_t_stringliteral_d6604f85ac07e2b33103a620b3d3d75b0473c7214912beded67b9b624d41c571_to_t_string_memory_ptr()
-            leave
-
-        }
-
-        function zero_value_for_split_t_string_memory_ptr() -> ret {
-            ret := 96
-        }
-
     }
     object \"C_10_deployed\" {
         code {

test/cmdlineTests/yul_string_format_hex/output.json

@@ -18,35 +18,6 @@ object \"C_10\" {
         return(0, datasize(\"C_10_deployed\"))
 
 
-        function cleanup_t_rational_2864434397_by_1(value) -> cleaned {
-            cleaned := value
-        }
-
-        function convert_t_rational_2864434397_by_1_to_t_bytes4(value) -> converted {
-            converted := shift_left_224(cleanup_t_rational_2864434397_by_1(value))
-        }
-
-        function fun_f_9()  -> vloc__4 {
-            let zero_value_for_type_t_bytes4_1 := zero_value_for_split_t_bytes4()
-            vloc__4 := zero_value_for_type_t_bytes4_1
-
-            let expr_6 := 0xaabbccdd
-            vloc__4 := convert_t_rational_2864434397_by_1_to_t_bytes4(expr_6)
-            leave
-
-        }
-
-        function shift_left_224(value) -> newValue {
-            newValue :=
-
-            shl(224, value)
-
-        }
-
-        function zero_value_for_split_t_bytes4() -> ret {
-            ret := 0
-        }
-
     }
     object \"C_10_deployed\" {
         code {

test/libsolidity/semanticTests/virtualFunctions/internal_virtual_function_calls.sol

@@ -0,0 +1,21 @@
+contract Base {
+    function f() public returns (uint256 i) {
+        return g();
+    }
+
+    function g() internal virtual returns (uint256 i) {
+        return 1;
+    }
+}
+
+
+contract Derived is Base {
+    function g() internal override returns (uint256 i) {
+        return 2;
+    }
+}
+
+// ====
+// compileViaYul: also
+// ----
+// f() -> 2