Re-use knowledge about keccak calls.

Re-use knowledge about keccak calls.

Update gas cost.

More tests.

Fix bug.

Update libyul/optimiser/DataFlowAnalyzer.cpp

Remove util prefixes

fix test

More test cases.

Add Changelog entry

Changelog.md

@@ -11,6 +11,7 @@ Compiler Features:
  * Standard JSON: Add a boolean field `settings.metadata.appendCBOR` that skips CBOR metadata from getting appended at the end of the bytecode.
  * Yul EVM Code Transform: Generate more optimal code for user-defined functions that always terminate a transaction. No return labels will be pushed for calls to functions that always terminate.
  * Yul Optimizer: Allow replacing the previously hard-coded cleanup sequence by specifying custom steps after a colon delimiter (``:``) in the sequence string.
+ * Yul Optimizer: Eliminate ``keccak256`` calls if the value was already calculated by a previous call and can be reused.
  * Language Server: Add basic document hover support.
  * Optimizer: Added optimization rule ``and(shl(X, Y), shl(X, Z)) => shl(X, and(Y, Z))``.
  * SMTChecker: Support Eldarica as a Horn solver for the CHC engine when using the CLI option ``--model-checker-solvers eld``. The binary `eld` must be available in the system.

libyul/optimiser/DataFlowAnalyzer.cpp

@@ -24,6 +24,7 @@
 
 #include <libyul/optimiser/NameCollector.h>
 #include <libyul/optimiser/Semantics.h>
+#include <libyul/optimiser/OptimizerUtilities.h>
 #include <libyul/optimiser/KnowledgeBase.h>
 #include <libyul/AST.h>
 #include <libyul/Dialect.h>
@@ -86,6 +87,8 @@ void DataFlowAnalyzer::operator()(ExpressionStatement& _statement)
 			cxx20::erase_if(m_state.environment.memory, mapTuple([&](auto&& key, auto&& /* value */) {
 				return !m_knowledgeBase.knownToBeDifferentByAtLeast32(vars->first, key);
 			}));
+			// TODO erase keccak knowledge, but in a more clever way
+			m_state.environment.keccak = {};
 			m_state.environment.memory[vars->first] = vars->second;
 			return;
 		}
@@ -127,7 +130,6 @@ void DataFlowAnalyzer::operator()(If& _if)
 	Environment preEnvironment = m_state.environment;
 
 	ASTModifier::operator()(_if);
-
 	joinKnowledge(preEnvironment);
 
 	clearValues(assignedVariableNames(_if.body));
@@ -223,7 +225,7 @@ void DataFlowAnalyzer::operator()(Block& _block)
 
 optional<YulString> DataFlowAnalyzer::storageValue(YulString _key) const
 {
-	if (YulString const* value = util::valueOrNullptr(m_state.environment.storage, _key))
+	if (YulString const* value = valueOrNullptr(m_state.environment.storage, _key))
 		return *value;
 	else
 		return nullopt;
@@ -231,7 +233,15 @@ optional<YulString> DataFlowAnalyzer::storageValue(YulString _key) const
 
 optional<YulString> DataFlowAnalyzer::memoryValue(YulString _key) const
 {
-	if (YulString const* value = util::valueOrNullptr(m_state.environment.memory, _key))
+	if (YulString const* value = valueOrNullptr(m_state.environment.memory, _key))
+		return *value;
+	else
+		return nullopt;
+}
+
+optional<YulString> DataFlowAnalyzer::keccakValue(YulString _start, YulString _length) const
+{
+	if (YulString const* value = valueOrNullptr(m_state.environment.keccak, make_pair(_start, _length)))
 		return *value;
 	else
 		return nullopt;
@@ -271,6 +281,9 @@ void DataFlowAnalyzer::handleAssignment(set<YulString> const& _variables, Expres
 			// assignment to slot denoted by "name"
 			m_state.environment.memory.erase(name);
 			// assignment to slot contents denoted by "name"
+			cxx20::erase_if(m_state.environment.keccak, [&name](auto&& _item) {
+				return _item.first.first == name || _item.first.second == name || _item.second == name;
+			});
 			cxx20::erase_if(m_state.environment.memory, mapTuple([&name](auto&& /* key */, auto&& value) { return value == name; }));
 		}
 	}
@@ -287,6 +300,8 @@ void DataFlowAnalyzer::handleAssignment(set<YulString> const& _variables, Expres
 				m_state.environment.memory[*key] = variable;
 			else if (auto key = isSimpleLoad(StoreLoadLocation::Storage, *_value))
 				m_state.environment.storage[*key] = variable;
+			else if (auto arguments = isKeccak(*_value))
+				m_state.environment.keccak[*arguments] = variable;
 		}
 	}
 }
@@ -314,7 +329,7 @@ void DataFlowAnalyzer::clearValues(set<YulString> _variables)
 	// let a := 1
 	// let b := a
 	// let c := b
-	// let a := 2
+	// a := 2
 	// add(b, c)
 	// In the last line, we can replace c by b, but not b by a.
 	//
@@ -329,6 +344,12 @@ void DataFlowAnalyzer::clearValues(set<YulString> _variables)
 	});
 	cxx20::erase_if(m_state.environment.storage, eraseCondition);
 	cxx20::erase_if(m_state.environment.memory, eraseCondition);
+	cxx20::erase_if(m_state.environment.keccak, [&_variables](auto&& _item) {
+		return
+			_variables.count(_item.first.first) ||
+			_variables.count(_item.first.second) ||
+			_variables.count(_item.second);
+	});
 
 	// Also clear variables that reference variables to be cleared.
 	for (auto const& variableToClear: _variables)
@@ -357,7 +378,10 @@ void DataFlowAnalyzer::clearKnowledgeIfInvalidated(Block const& _block)
 	if (sideEffects.invalidatesStorage())
 		m_state.environment.storage.clear();
 	if (sideEffects.invalidatesMemory())
+	{
 		m_state.environment.memory.clear();
+		m_state.environment.keccak.clear();
+	}
 }
 
 void DataFlowAnalyzer::clearKnowledgeIfInvalidated(Expression const& _expr)
@@ -368,7 +392,10 @@ void DataFlowAnalyzer::clearKnowledgeIfInvalidated(Expression const& _expr)
 	if (sideEffects.invalidatesStorage())
 		m_state.environment.storage.clear();
 	if (sideEffects.invalidatesMemory())
+	{
 		m_state.environment.memory.clear();
+		m_state.environment.keccak.clear();
+	}
 }
 
 bool DataFlowAnalyzer::inScope(YulString _variableName) const
@@ -416,12 +443,26 @@ std::optional<YulString> DataFlowAnalyzer::isSimpleLoad(
 	return {};
 }
 
+optional<pair<YulString, YulString>> DataFlowAnalyzer::isKeccak(Expression const& _expression) const
+{
+	if (FunctionCall const* funCall = get_if<FunctionCall>(&_expression))
+		if (funCall->functionName.name == m_dialect.hashFunction({}))
+			if (Identifier const* start = std::get_if<Identifier>(&funCall->arguments.at(0)))
+				if (Identifier const* length = std::get_if<Identifier>(&funCall->arguments.at(1)))
+					return make_pair(start->name, length->name);
+	return nullopt;
+}
+
 void DataFlowAnalyzer::joinKnowledge(Environment const& _olderEnvironment)
 {
 	if (!m_analyzeStores)
 		return;
 	joinKnowledgeHelper(m_state.environment.storage, _olderEnvironment.storage);
 	joinKnowledgeHelper(m_state.environment.memory, _olderEnvironment.memory);
+	cxx20::erase_if(m_state.environment.keccak, mapTuple([&_olderEnvironment](auto&& key, auto&& currentValue) {
+		YulString const* oldValue = valueOrNullptr(_olderEnvironment.keccak, key);
+		return !oldValue || *oldValue != currentValue;
+	}));
 }
 
 void DataFlowAnalyzer::joinKnowledgeHelper(
@@ -434,7 +475,7 @@ void DataFlowAnalyzer::joinKnowledgeHelper(
 	// of m_state.environment.memory and thus any overlapping write would have cleared the keys
 	// that are not known to be different inside m_state.environment.memory already.
 	cxx20::erase_if(_this, mapTuple([&_older](auto&& key, auto&& currentValue){
-		YulString const* oldValue = util::valueOrNullptr(_older, key);
+		YulString const* oldValue = valueOrNullptr(_older, key);
 		return !oldValue || *oldValue != currentValue;
 	}));
 }

libyul/optimiser/DataFlowAnalyzer.h

@@ -108,6 +108,7 @@ public:
 	std::map<YulString, AssignedValue> const& allValues() const { return m_state.value; }
 	std::optional<YulString> storageValue(YulString _key) const;
 	std::optional<YulString> memoryValue(YulString _key) const;
+	std::optional<YulString> keccakValue(YulString _start, YulString _length) const;
 
 protected:
 	/// Registers the assignment.
@@ -157,6 +158,10 @@ protected:
 		Expression const& _expression
 	) const;
 
+	/// Checks if the expression is keccak256(s, l)
+	/// where s and l are variables and returns these variables in that case.
+	std::optional<std::pair<YulString, YulString>> isKeccak(Expression const& _expression) const;
+
 	Dialect const& m_dialect;
 	/// Side-effects of user-defined functions. Worst-case side-effects are assumed
 	/// if this is not provided or the function is not found.
@@ -167,6 +172,8 @@ private:
 	{
 		std::unordered_map<YulString, YulString> storage;
 		std::unordered_map<YulString, YulString> memory;
+		/// If keccak[s, l] = y then y := keccak256(s, l) occurs in the code.
+		std::map<std::pair<YulString, YulString>, YulString> keccak;
 	};
 	struct State
 	{

libyul/optimiser/LoadResolver.cpp

@@ -26,6 +26,7 @@
 #include <libyul/backends/evm/EVMMetrics.h>
 #include <libyul/optimiser/Semantics.h>
 #include <libyul/optimiser/CallGraphGenerator.h>
+#include <libyul/optimiser/OptimizerUtilities.h>
 #include <libyul/SideEffects.h>
 #include <libyul/AST.h>
 #include <libyul/Utilities.h>
@@ -58,15 +59,23 @@ void LoadResolver::visit(Expression& _e)
 
 	if (FunctionCall const* funCall = std::get_if<FunctionCall>(&_e))
 	{
-		for (auto location: { StoreLoadLocation::Memory, StoreLoadLocation::Storage })
+		if (funCall->functionName.name == m_loadFunctionName[static_cast<unsigned>(StoreLoadLocation::Memory)])
-			if (funCall->functionName.name == m_loadFunctionName[static_cast<unsigned>(location)])
+			tryResolve(_e, StoreLoadLocation::Memory, funCall->arguments);
-			{
+		else if (funCall->functionName.name == m_loadFunctionName[static_cast<unsigned>(StoreLoadLocation::Storage)])
-				tryResolve(_e, location, funCall->arguments);
+			tryResolve(_e, StoreLoadLocation::Storage, funCall->arguments);
-				break;
+		else if (!m_containsMSize && funCall->functionName.name == m_dialect.hashFunction({}))
-			}
+		{
-
+			Identifier const* start = get_if<Identifier>(&funCall->arguments.at(0));
-		if (!m_containsMSize && funCall->functionName.name == m_dialect.hashFunction({}))
+			Identifier const* length = get_if<Identifier>(&funCall->arguments.at(1));
+			if (start && length)
+				if (auto const& value = keccakValue(start->name, length->name))
+					if (inScope(*value))
+					{
+						_e = Identifier{debugDataOf(_e), *value};
+						return;
+					}
 			tryEvaluateKeccak(_e, funCall->arguments);
+		}
 	}
 }
 

test/libsolidity/semanticTests/array/reusing_memory.sol

@@ -24,6 +24,6 @@ contract Main {
 }
 // ----
 // f(uint256): 0x34 -> 0x46bddb1178e94d7f2892ff5f366840eb658911794f2c3a44c450aa2c505186c1
-// gas irOptimized: 112899
+// gas irOptimized: 112757
 // gas legacy: 126596
 // gas legacyOptimized: 113823

test/libsolidity/semanticTests/structs/copy_substructures_to_mapping.sol

@@ -53,14 +53,14 @@ contract C {
 
 // ----
 // from_memory() -> 0x20, 0x60, 0xa0, 0x15, 3, 0x666F6F0000000000000000000000000000000000000000000000000000000000, 2, 13, 14
-// gas irOptimized: 123062
+// gas irOptimized: 123041
 // gas legacy: 130289
 // gas legacyOptimized: 128785
 // from_state() -> 0x20, 0x60, 0xa0, 21, 3, 0x666F6F0000000000000000000000000000000000000000000000000000000000, 2, 13, 14
-// gas irOptimized: 121776
+// gas irOptimized: 121737
 // gas legacy: 123341
 // gas legacyOptimized: 121892
 // from_calldata((bytes,uint16[],uint16)): 0x20, 0x60, 0xa0, 21, 3, 0x666F6F0000000000000000000000000000000000000000000000000000000000, 2, 13, 14 -> 0x20, 0x60, 0xa0, 0x15, 3, 0x666F6F0000000000000000000000000000000000000000000000000000000000, 2, 13, 14
-// gas irOptimized: 115169
+// gas irOptimized: 115127
 // gas legacy: 122579
 // gas legacyOptimized: 120829

test/libsolidity/semanticTests/structs/struct_copy.sol

@@ -36,7 +36,7 @@ contract c {
 
 // ----
 // set(uint256): 7 -> true
-// gas irOptimized: 110032
+// gas irOptimized: 109897
 // gas legacy: 110616
 // gas legacyOptimized: 110006
 // retrieve(uint256): 7 -> 1, 3, 4, 2

test/libyul/yulOptimizerTests/loadResolver/keccak_reuse_basic.yul

@@ -0,0 +1,19 @@
+{
+    let x := calldataload(0)
+    let a := keccak256(0, x)
+    sstore(a, 2)
+    let t := mload(2)
+    let b := keccak256(0, x)
+    sstore(b, 3)
+}
+// ----
+// step: loadResolver
+//
+// {
+//     {
+//         let _1 := 0
+//         let a := keccak256(_1, calldataload(_1))
+//         sstore(a, 2)
+//         sstore(a, 3)
+//     }
+// }

test/libyul/yulOptimizerTests/loadResolver/keccak_reuse_expr_mstore.yul

@@ -0,0 +1,30 @@
+{
+    let a := calldataload(0)
+    sstore(f(keccak256(0, a)), keccak256(0, a))
+    sstore(f(keccak256(0, a)), keccak256(0, a))
+    sstore(keccak256(0, a), f(keccak256(0, a)))
+
+    function f(x) -> y {
+        mstore(x, 2)
+        y := mload(8)
+    }
+}
+// ----
+// step: loadResolver
+//
+// {
+//     {
+//         let _1 := 0
+//         let a := calldataload(_1)
+//         let _3 := keccak256(_1, a)
+//         sstore(f(_3), _3)
+//         let _8 := keccak256(_1, a)
+//         sstore(f(_8), _8)
+//         sstore(keccak256(_1, a), f(keccak256(_1, a)))
+//     }
+//     function f(x) -> y
+//     {
+//         mstore(x, 2)
+//         y := mload(8)
+//     }
+// }

test/libyul/yulOptimizerTests/loadResolver/keccak_reuse_in_expression.yul

@@ -0,0 +1,14 @@
+{
+    let x := calldataload(0)
+    sstore(keccak256(0, x), keccak256(0, x))
+}
+// ----
+// step: loadResolver
+//
+// {
+//     {
+//         let _1 := 0
+//         let _3 := keccak256(_1, calldataload(_1))
+//         sstore(_3, _3)
+//     }
+// }

test/libyul/yulOptimizerTests/loadResolver/keccak_reuse_msize.yul

@@ -0,0 +1,18 @@
+{
+    let a := calldataload(0)
+    let t := msize()
+    let x := keccak256(0, a)
+    let y := keccak256(0, a)
+    sstore(x, y)
+}
+// ----
+// step: loadResolver
+//
+// {
+//     {
+//         let _1 := 0
+//         let a := calldataload(_1)
+//         let x := keccak256(_1, a)
+//         sstore(x, keccak256(_1, a))
+//     }
+// }

test/libyul/yulOptimizerTests/loadResolver/keccak_reuse_mstore.yul

@@ -0,0 +1,22 @@
+{
+    let x := calldataload(0)
+    let a := keccak256(0x20, x)
+    sstore(a, 2)
+    // will disable loading for now, might improve later
+    mstore(0, 1)
+    let b := keccak256(0x20, x)
+    sstore(b, 3)
+}
+// ----
+// step: loadResolver
+//
+// {
+//     {
+//         let _1 := 0
+//         let x := calldataload(_1)
+//         let _2 := 0x20
+//         sstore(keccak256(_2, x), 2)
+//         mstore(_1, 1)
+//         sstore(keccak256(_2, x), 3)
+//     }
+// }

test/libyul/yulOptimizerTests/loadResolver/keccak_reuse_reassigned_branch.yul

@@ -0,0 +1,37 @@
+{
+    let x := calldataload(0)
+    let y := calldataload(1)
+    let a := keccak256(x, y)
+    if calldataload(2) {
+        a := 8
+    }
+    let b := keccak256(x, y)
+    sstore(b, 2)
+    if calldataload(3) {
+        x := 8
+    }
+    let c := keccak256(x, y)
+    sstore(c, 2)
+    if calldataload(4) {
+        y := 8
+    }
+    let d := keccak256(x, y)
+    sstore(d, 2)
+}
+// ----
+// step: loadResolver
+//
+// {
+//     {
+//         let x := calldataload(0)
+//         let y := calldataload(1)
+//         let a := keccak256(x, y)
+//         let _3 := 2
+//         if calldataload(_3) { a := 8 }
+//         sstore(keccak256(x, y), _3)
+//         if calldataload(3) { x := 8 }
+//         sstore(keccak256(x, y), _3)
+//         if calldataload(4) { y := 8 }
+//         sstore(keccak256(x, y), _3)
+//     }
+// }

test/libyul/yulOptimizerTests/loadResolver/keccak_reuse_reassigned_value.yul

@@ -0,0 +1,41 @@
+{
+    let x := calldataload(0)
+    let y := calldataload(1)
+    let a := keccak256(x, y)
+    sstore(a, 2)
+    // reassign value
+    a := calldataload(10)
+    let b := keccak256(x, y)
+    sstore(b, 3)
+    // reassign arg1
+    x := 10
+    let c := keccak256(x, y)
+    sstore(c, 4)
+    // reassign arg2
+    y := 9
+    let d := keccak256(x, y)
+    sstore(d, 5)
+    // no reassign, check that it is still working here.
+    let e := keccak256(x, y)
+    sstore(e, 6)
+}
+// ----
+// step: loadResolver
+//
+// {
+//     {
+//         let x := calldataload(0)
+//         let y := calldataload(1)
+//         let a := keccak256(x, y)
+//         sstore(a, 2)
+//         let _4 := 10
+//         a := calldataload(_4)
+//         sstore(keccak256(x, y), 3)
+//         x := _4
+//         sstore(keccak256(_4, y), 4)
+//         y := 9
+//         let d := keccak256(_4, y)
+//         sstore(d, 5)
+//         sstore(d, 6)
+//     }
+// }