Merge pull request #9966 from ethereum/bytesArrayResizeSol2Yul

[Sol->Yul] Implementing resizing of dynamic byte arrays

libsolidity/codegen/YulUtilFunctions.cpp

@@ -959,9 +959,11 @@ std::string YulUtilFunctions::resizeDynamicArrayFunction(ArrayType const& _type)
 {
 	solAssert(_type.location() == DataLocation::Storage, "");
 	solAssert(_type.isDynamicallySized(), "");
-	solUnimplementedAssert(!_type.isByteArray(), "Byte Arrays not yet implemented!");
 	solUnimplementedAssert(_type.baseType()->storageBytes() <= 32, "...");
 
+	if (_type.isByteArray())
+		return resizeDynamicByteArrayFunction(_type);
+
 	string functionName = "resize_array_" + _type.identifier();
 	return m_functionCollector.createFunction(functionName, [&]() {
 		return Whiskers(R"(
@@ -1006,6 +1008,151 @@ std::string YulUtilFunctions::resizeDynamicArrayFunction(ArrayType const& _type)
 	});
 }
 
+string YulUtilFunctions::resizeDynamicByteArrayFunction(ArrayType const& _type)
+{
+	string functionName = "resize_array_" + _type.identifier();
+	return m_functionCollector.createFunction(functionName, [&]() {
+		return Whiskers(R"(
+			function <functionName>(array, newLen) {
+				if gt(newLen, <maxArrayLength>) {
+					<panic>()
+				}
+
+				let data := sload(array)
+				let oldLen := <extractLength>(data)
+
+				if gt(newLen, oldLen) {
+					<increaseSize>(array, data, oldLen, newLen)
+				}
+
+				if lt(newLen, oldLen) {
+					<decreaseSize>(array, data, oldLen, newLen)
+				}
+			})")
+			("functionName", functionName)
+			("panic", panicFunction())
+			("extractLength", extractByteArrayLengthFunction())
+			("maxArrayLength", (u256(1) << 64).str())
+			("decreaseSize", decreaseByteArraySizeFunction(_type))
+			("increaseSize", increaseByteArraySizeFunction(_type))
+			.render();
+	});
+}
+
+string YulUtilFunctions::decreaseByteArraySizeFunction(ArrayType const& _type)
+{
+	string functionName = "byte_array_decrease_size_" + _type.identifier();
+	return m_functionCollector.createFunction(functionName, [&]() {
+		return Whiskers(R"(
+			function <functionName>(array, data, oldLen, newLen) {
+				switch lt(newLen, 32)
+				case  0 {
+					let arrayDataStart := <dataPosition>(array)
+					let deleteStart := add(arrayDataStart, div(add(newLen, 31), 32))
+
+					// we have to partially clear last slot that is still used
+					let offset := and(newLen, 0x1f)
+					if offset { <partialClearStorageSlot>(sub(deleteStart, 1), offset) }
+
+					<clearStorageRange>(deleteStart, add(arrayDataStart, div(add(oldLen, 31), 32)))
+
+					sstore(array, or(mul(2, newLen), 1))
+				}
+				default {
+					switch gt(oldLen, 31)
+					case 1 {
+						let arrayDataStart := <dataPosition>(array)
+						// clear whole old array, as we are transforming to short bytes array
+						<clearStorageRange>(add(arrayDataStart, 1), add(arrayDataStart, div(add(oldLen, 31), 32)))
+						<transitLongToShort>(array, newLen)
+					}
+					default {
+						sstore(array, <encodeUsedSetLen>(data, newLen))
+					}
+				}
+			})")
+			("functionName", functionName)
+			("dataPosition", arrayDataAreaFunction(_type))
+			("partialClearStorageSlot", partialClearStorageSlotFunction())
+			("clearStorageRange", clearStorageRangeFunction(*_type.baseType()))
+			("transitLongToShort", byteArrayTransitLongToShortFunction(_type))
+			("encodeUsedSetLen", shortByteArrayEncodeUsedAreaSetLengthFunction())
+			.render();
+	});
+}
+
+string YulUtilFunctions::increaseByteArraySizeFunction(ArrayType const& _type)
+{
+	string functionName = "byte_array_increase_size_" + _type.identifier();
+	return m_functionCollector.createFunction(functionName, [&]() {
+		return Whiskers(R"(
+			function <functionName>(array, data, oldLen, newLen) {
+				switch lt(oldLen, 32)
+				case 0 {
+					// in this case array stays unpacked, so we just set new length
+					sstore(array, add(mul(2, newLen), 1))
+				}
+				default {
+					switch lt(newLen, 32)
+					case 0 {
+						// we need to copy elements to data area as we changed array from packed to unpacked
+						data := and(not(0xff), data)
+						sstore(<dataPosition>(array), data)
+						sstore(array, add(mul(2, newLen), 1))
+					}
+					default {
+						// here array stays packed, we just need to increase length
+						sstore(array, <encodeUsedSetLen>(data, newLen))
+					}
+				}
+			})")
+			("functionName", functionName)
+			("dataPosition", arrayDataAreaFunction(_type))
+			("encodeUsedSetLen", shortByteArrayEncodeUsedAreaSetLengthFunction())
+			.render();
+	});
+}
+
+string YulUtilFunctions::byteArrayTransitLongToShortFunction(ArrayType const& _type)
+{
+	string functionName = "transit_byte_array_long_to_short_" + _type.identifier();
+	return m_functionCollector.createFunction(functionName, [&]() {
+		return Whiskers(R"(
+			function <functionName>(array, len) {
+				// we need to copy elements from old array to new
+				// we want to copy only elements that are part of the array after resizing
+				let dataPos := <dataPosition>(array)
+				let data := <extractUsedApplyLen>(sload(dataPos), len)
+				sstore(array, data)
+				sstore(dataPos, 0)
+			})")
+			("functionName", functionName)
+			("dataPosition", arrayDataAreaFunction(_type))
+			("extractUsedApplyLen", shortByteArrayEncodeUsedAreaSetLengthFunction())
+			("shl", shiftLeftFunctionDynamic())
+			("ones", formatNumber((bigint(1) << 256) - 1))
+			.render();
+	});
+}
+
+string YulUtilFunctions::shortByteArrayEncodeUsedAreaSetLengthFunction()
+{
+	string functionName = "extract_used_part_and_set_length_of_short_byte_array";
+	return m_functionCollector.createFunction(functionName, [&]() {
+		return Whiskers(R"(
+			function <functionName>(data, len) -> used {
+				// we want to save only elements that are part of the array after resizing
+				// others should be set to zero
+				let mask := <shl>(mul(8, sub(32, len)), <ones>)
+				used := or(and(data, mask), mul(2, len))
+			})")
+			("functionName", functionName)
+			("shl", shiftLeftFunctionDynamic())
+			("ones", formatNumber((bigint(1) << 256) - 1))
+			.render();
+	});
+}
+
 string YulUtilFunctions::storageArrayPopFunction(ArrayType const& _type)
 {
 	solAssert(_type.location() == DataLocation::Storage, "");
@@ -1048,36 +1195,30 @@ string YulUtilFunctions::storageByteArrayPopFunction(ArrayType const& _type)
 				let oldLen := <extractByteArrayLength>(data)
 				if iszero(oldLen) { <panic>() }
 
-				switch eq(oldLen, 32)
+				switch oldLen
-				case 1 {
+				case 32 {
 					// Here we have a special case where array transitions to shorter than 32
 					// So we need to copy data
-					let copyFromSlot := <dataAreaFunction>(array)
+					<transitLongToShort>(array, 31)
-					data := sload(copyFromSlot)
-					sstore(copyFromSlot, 0)
-					// New length is 31, encoded to 31 * 2 = 62
-					data := or(and(data, not(0xff)), 62)
 				}
 				default {
-					data := sub(data, 2)
 					let newLen := sub(oldLen, 1)
 					switch lt(oldLen, 32)
 					case 1 {
-						// set last element to zero
+						sstore(array, <encodeUsedSetLen>(data, newLen))
-						let mask := not(<shl>(mul(8, sub(31, newLen)), 0xff))
-						data := and(data, mask)
 					}
 					default {
 						let slot, offset := <indexAccess>(array, newLen)
 						<setToZero>(slot, offset)
+						sstore(array, sub(data, 2))
 					}
 				}
-				sstore(array, data)
 			})")
 			("functionName", functionName)
 			("panic", panicFunction())
 			("extractByteArrayLength", extractByteArrayLengthFunction())
-			("dataAreaFunction", arrayDataAreaFunction(_type))
+			("transitLongToShort", byteArrayTransitLongToShortFunction(_type))
+			("encodeUsedSetLen", shortByteArrayEncodeUsedAreaSetLengthFunction())
 			("indexAccess", storageArrayIndexAccessFunction(_type))
 			("setToZero", storageSetToZeroFunction(*_type.baseType()))
 			("shl", shiftLeftFunctionDynamic())
@@ -1982,7 +2123,10 @@ string YulUtilFunctions::updateStorageValueFunction(
 				fromReferenceType->location(),
 				fromReferenceType->isPointer()
 			).get() == *fromReferenceType, "");
-			solUnimplementedAssert(fromReferenceType->location() != DataLocation::Storage, "");
+			solUnimplementedAssert(
+				fromReferenceType->location() != DataLocation::Storage,
+				"Copying from storage to storage is not yet implemented."
+			);
 			solAssert(toReferenceType->category() == fromReferenceType->category(), "");
 
 			if (_toType.category() == Type::Category::Array)

libsolidity/codegen/YulUtilFunctions.h

@@ -446,6 +446,29 @@ private:
 	/// signature: (slot) ->
 	std::string clearStorageStructFunction(StructType const& _type);
 
+	/// @returns the name of a function that resizes a storage byte array
+	/// signature: (array, newLen)
+	std::string resizeDynamicByteArrayFunction(ArrayType const& _type);
+
+	/// @returns the name of a function that increases size of byte array
+	/// when we resize byte array frextractUsedSetLenom < 32 elements to >= 32 elements or we push to byte array of size 31 copying of data will  occur
+	/// signature: (array, data, oldLen, newLen)
+	std::string increaseByteArraySizeFunction(ArrayType const& _type);
+
+	/// @returns the name of a function that decreases size of byte array
+	/// when we resize byte array from >= 32 elements to < 32 elements or we pop from byte array of size 32 copying of data will  occur
+	/// signature: (array, data, oldLen, newLen)
+	std::string decreaseByteArraySizeFunction(ArrayType const& _type);
+
+	/// @returns the name of a function that sets size of short byte array while copying data
+	/// should be called when we resize from long byte array (more than 32 elements) to short byte array
+	/// signature: (array, data, len)
+	std::string byteArrayTransitLongToShortFunction(ArrayType const& _type);
+
+	/// @returns the name of a function that extracts only used part of slot that represents short byte array
+	/// signature: (data, len) -> data
+	std::string shortByteArrayEncodeUsedAreaSetLengthFunction();
+
 	langutil::EVMVersion m_evmVersion;
 	RevertStrings m_revertStrings;
 	MultiUseYulFunctionCollector& m_functionCollector;

test/cmdlineTests/yul_unimplemented/err

@@ -1,5 +1,5 @@
-Error (1834): Unimplemented feature error: Byte Arrays not yet implemented! in FILENAME REMOVED
+Error (1834): Unimplemented feature error: Copying from storage to storage is not yet implemented. in FILENAME REMOVED
- --> yul_unimplemented/input.sol:6:9:
+ --> yul_unimplemented/input.sol:7:9:
   |
-6 |         delete a;
+7 |         a = b;
-  |         ^^^^^^^^
+  |         ^^^^^

test/cmdlineTests/yul_unimplemented/input.sol

@@ -2,7 +2,8 @@
 pragma solidity >=0.0;
 contract test {
     bytes a;
+    bytes b;
     function f() public {
-        delete a;
+        a = b;
     }
 }

test/libsolidity/semanticTests/array/byte_array_storage_layout.sol

@@ -32,6 +32,7 @@ contract c {
             data.push(bytes1(i));
         }
         data.pop();
+        data.pop();
         assembly {
             r := sload(data.slot)
         }
@@ -42,4 +43,4 @@ contract c {
 // ----
 // test_short() -> 1780731860627700044960722568376587075150542249149356309979516913770823710
 // test_long() -> 67
-// test_pop() -> 1780731860627700044960722568376592200742329637303199754547598369979440702
+// test_pop() -> 1780731860627700044960722568376592200742329637303199754547598369979433020

test/libsolidity/semanticTests/array/delete_bytes_array.sol

@@ -0,0 +1,37 @@
+contract C {
+    bytes data;
+
+    function f() public returns (uint ret) {
+        data.push("a");
+        data.push("b");
+        delete data;
+        assembly {
+            ret := sload(data.slot)
+        }
+    }
+
+    function g() public returns (uint ret) {
+        assembly {
+            sstore(data.slot, 67)
+        }
+        data.push("a");
+        data.push("b");
+        assert(data.length == 35);
+        delete data;
+        assert(data.length == 0);
+
+        uint size = 999;
+        assembly {
+            size := sload(data.slot)
+            mstore(0, data.slot)
+            ret := sload(keccak256(0, 32))
+        }
+        assert(size == 0);
+    }
+}
+
+// ====
+// compileViaYul: also
+// ----
+// f() -> 0
+// g() -> 0