Merge pull request #10083 from ethereum/copyByteArray

Copy byte array

libsolidity/codegen/YulUtilFunctions.cpp

@@ -1462,13 +1462,14 @@ string YulUtilFunctions::clearStorageStructFunction(StructType const& _type)
 	});
 }
 
-string YulUtilFunctions::copyArrayToStorage(ArrayType const& _fromType, ArrayType const& _toType)
+string YulUtilFunctions::copyArrayToStorageFunction(ArrayType const& _fromType, ArrayType const& _toType)
 {
 	solAssert(
 		*_fromType.copyForLocation(_toType.location(), _toType.isPointer()) == dynamic_cast<ReferenceType const&>(_toType),
 		""
 	);
-	solUnimplementedAssert(!_fromType.isByteArray(), "");
+	if (_fromType.isByteArray())
+		return copyByteArrayToStorageFunction(_fromType, _toType);
 	solUnimplementedAssert(!_fromType.dataStoredIn(DataLocation::Storage), "");
 
 	string functionName = "copy_array_to_storage_from_" + _fromType.identifier() + "_to_" + _toType.identifier();
@@ -1561,6 +1562,84 @@ string YulUtilFunctions::copyArrayToStorage(ArrayType const& _fromType, ArrayTyp
 	});
 }
 
+
+string YulUtilFunctions::copyByteArrayToStorageFunction(ArrayType const& _fromType, ArrayType const& _toType)
+{
+	solAssert(
+		*_fromType.copyForLocation(_toType.location(), _toType.isPointer()) == dynamic_cast<ReferenceType const&>(_toType),
+		""
+	);
+	solAssert(_fromType.isByteArray(), "");
+	solAssert(_toType.isByteArray(), "");
+	solUnimplementedAssert(!_fromType.dataStoredIn(DataLocation::Storage), "");
+
+	string functionName = "copy_byte_array_to_storage_from_" + _fromType.identifier() + "_to_" + _toType.identifier();
+	return m_functionCollector.createFunction(functionName, [&](){
+		Whiskers templ(R"(
+			function <functionName>(slot, src<?fromCalldata>, len</fromCalldata>) {
+				let newLen := <arrayLength>(src<?fromCalldata>, len</fromCalldata>)
+				// Make sure array length is sane
+				if gt(newLen, 0xffffffffffffffff) { <panic>() }
+
+				let oldLen := <byteArrayLength>(sload(slot))
+
+				<?fromMemory>
+					src := add(src, 0x20)
+				</fromMemory>
+
+				// This is not needed in all branches.
+				let dstDataArea
+				if or(gt(oldLen, 31), gt(newLen, 31)) {
+					dstDataArea := <dstDataLocation>(slot)
+				}
+
+				if gt(oldLen, 31) {
+					// potentially truncate data
+					let deleteStart := add(dstDataArea, div(add(newLen, 31), 32))
+					if lt(newLen, 32) { deleteStart := dstDataArea }
+					<clearStorageRange>(deleteStart, add(dstDataArea, div(add(oldLen, 31), 32)))
+				}
+				switch gt(newLen, 31)
+				case 1 {
+					let loopEnd := and(newLen, not(0x1f))
+					let dstPtr := dstDataArea
+					let i := 0
+					for { } lt(i, loopEnd) { i := add(i, 32) } {
+						sstore(dstPtr, <readFromCalldataOrMemory>(add(src, i)))
+						dstPtr := add(dstPtr, 1)
+					}
+					if lt(loopEnd, newLen) {
+						let lastValue := <readFromCalldataOrMemory>(add(src, i))
+						sstore(dstPtr, <maskBytes>(lastValue, and(newLen, 0x1f)))
+					}
+					sstore(slot, add(mul(newLen, 2), 1))
+				}
+				default {
+					let value := 0
+					if newLen {
+						value := <readFromCalldataOrMemory>(src)
+					}
+					sstore(slot, <byteArrayCombineShort>(value, newLen))
+				}
+			}
+		)");
+		templ("functionName", functionName);
+		bool fromCalldata = _fromType.dataStoredIn(DataLocation::CallData);
+		templ("fromMemory", _fromType.dataStoredIn(DataLocation::Memory));
+		templ("fromCalldata", fromCalldata);
+		templ("arrayLength", arrayLengthFunction(_fromType));
+		templ("panic", panicFunction());
+		templ("byteArrayLength", extractByteArrayLengthFunction());
+		templ("dstDataLocation", arrayDataAreaFunction(_toType));
+		templ("clearStorageRange", clearStorageRangeFunction(*_toType.baseType()));
+		templ("readFromCalldataOrMemory", readFromMemoryOrCalldata(*TypeProvider::uint256(), fromCalldata));
+		templ("maskBytes", maskBytesFunctionDynamic());
+		templ("byteArrayCombineShort", shortByteArrayEncodeUsedAreaSetLengthFunction());
+
+		return templ.render();
+	});
+}
+
 string YulUtilFunctions::arrayConvertLengthToSize(ArrayType const& _type)
 {
 	string functionName = "array_convert_length_to_size_" + _type.identifier();
@@ -2154,7 +2233,7 @@ string YulUtilFunctions::updateStorageValueFunction(
 				)");
 				templ("functionName", functionName);
 				templ("value", suffixedVariableNameList("value_", 0, _fromType.sizeOnStack()));
-				templ("copyArrayToStorage", copyArrayToStorage(
+				templ("copyArrayToStorage", copyArrayToStorageFunction(
 					dynamic_cast<ArrayType const&>(_fromType),
 					dynamic_cast<ArrayType const&>(_toType)
 				));

libsolidity/codegen/YulUtilFunctions.h

@@ -188,7 +188,11 @@ public:
 
 	/// @returns the name of a function that will copy array from calldata or memory to storage
 	/// signature (to_slot, from_ptr) ->
-	std::string copyArrayToStorage(ArrayType const& _fromType, ArrayType const& _toType);
+	std::string copyArrayToStorageFunction(ArrayType const& _fromType, ArrayType const& _toType);
+
+	/// @returns the name of a function that will copy a byte array from calldata or memory to storage
+	/// signature (to_slot, from_ptr) ->
+	std::string copyByteArrayToStorageFunction(ArrayType const& _fromType, ArrayType const& _toType);
 
 	/// Returns the name of a function that will convert a given length to the
 	/// size in memory (number of storage slots or calldata/memory bytes) it

test/libsolidity/SolidityEndToEndTest.cpp

@@ -2857,11 +2857,14 @@ BOOST_AUTO_TEST_CASE(delete_removes_bytes_data)
 			bytes data;
 		}
 	)";
+	ALSO_VIA_YUL(
+		DISABLE_EWASM_TESTRUN()
 		compileAndRun(sourceCode);
 		ABI_CHECK(callContractFunction("---", 7), bytes());
 		BOOST_CHECK(!storageEmpty(m_contractAddress));
 		ABI_CHECK(callContractFunction("del()", 7), encodeArgs(true));
 		BOOST_CHECK(storageEmpty(m_contractAddress));
+	);
 }
 
 BOOST_AUTO_TEST_CASE(copy_from_calldata_removes_bytes_data)
@@ -2873,6 +2876,8 @@ BOOST_AUTO_TEST_CASE(copy_from_calldata_removes_bytes_data)
 			bytes data;
 		}
 	)";
+	ALSO_VIA_YUL(
+		DISABLE_EWASM_TESTRUN()
 		compileAndRun(sourceCode);
 		ABI_CHECK(callContractFunction("set()", 1, 2, 3, 4, 5), encodeArgs(true));
 		BOOST_CHECK(!storageEmpty(m_contractAddress));
@@ -2880,6 +2885,7 @@ BOOST_AUTO_TEST_CASE(copy_from_calldata_removes_bytes_data)
 		BOOST_CHECK(m_transactionSuccessful);
 		BOOST_CHECK(m_output.empty());
 		BOOST_CHECK(storageEmpty(m_contractAddress));
+	);
 }
 
 BOOST_AUTO_TEST_CASE(copy_removes_bytes_data)
@@ -3125,6 +3131,9 @@ BOOST_AUTO_TEST_CASE(bytes_in_arguments)
 			}
 		}
 	)";
+	ALSO_VIA_YUL(
+		DISABLE_EWASM_TESTRUN()
+
 		compileAndRun(sourceCode);
 
 		string innercalldata1 = asString(FixedHash<4>(util::keccak256("f(uint256,uint256)")).asBytes() + encodeArgs(8, 9));
@@ -3137,6 +3146,7 @@ BOOST_AUTO_TEST_CASE(bytes_in_arguments)
 			callContractFunction("test(uint256,bytes,bytes,uint256)", calldata),
 			encodeArgs(12, (8 + 9) * 3, 13, u256(innercalldata1.length()))
 		);
+	);
 }
 
 BOOST_AUTO_TEST_CASE(fixed_array_cleanup)
@@ -3239,12 +3249,16 @@ BOOST_AUTO_TEST_CASE(dynamic_multi_array_cleanup)
 			function clear() public { delete data; }
 		}
 	)";
+	ALSO_VIA_YUL(
+		DISABLE_EWASM_TESTRUN()
+
 		compileAndRun(sourceCode);
 		BOOST_CHECK(storageEmpty(m_contractAddress));
 		ABI_CHECK(callContractFunction("fill()"), encodeArgs(8));
 		BOOST_CHECK(!storageEmpty(m_contractAddress));
 		ABI_CHECK(callContractFunction("clear()"), bytes());
 		BOOST_CHECK(storageEmpty(m_contractAddress));
+	);
 }
 
 BOOST_AUTO_TEST_CASE(array_copy_storage_storage_dyn_dyn)
@@ -3361,6 +3375,9 @@ BOOST_AUTO_TEST_CASE(array_copy_storage_abi)
 			}
 		}
 	)";
+	ALSO_VIA_YUL(
+		DISABLE_EWASM_TESTRUN()
+
 		compileAndRun(sourceCode);
 		bytes valueSequence;
 		for (size_t i = 0; i < 101; ++i)
@@ -3376,6 +3393,7 @@ BOOST_AUTO_TEST_CASE(array_copy_storage_abi)
 			encodeArgs(101) + valueSequence +
 			encodeArgs(101) + valueSequence
 		);
+	);
 }
 
 BOOST_AUTO_TEST_CASE(array_pop_uint16_transition)
@@ -3400,9 +3418,12 @@ BOOST_AUTO_TEST_CASE(array_pop_uint16_transition)
 			}
 		}
 	)";
+	ALSO_VIA_YUL(
+		DISABLE_EWASM_TESTRUN()
 		compileAndRun(sourceCode);
 		ABI_CHECK(callContractFunction("test()"), encodeArgs(38, 28, 18));
 		BOOST_CHECK(storageEmpty(m_contractAddress));
+	);
 }
 
 BOOST_AUTO_TEST_CASE(array_pop_uint24_transition)
@@ -3427,9 +3448,12 @@ BOOST_AUTO_TEST_CASE(array_pop_uint24_transition)
 			}
 		}
 	)";
+	ALSO_VIA_YUL(
+		DISABLE_EWASM_TESTRUN()
 		compileAndRun(sourceCode);
 		ABI_CHECK(callContractFunction("test()"), encodeArgs(20, 10));
 		BOOST_CHECK(storageEmpty(m_contractAddress));
+	);
 }
 
 BOOST_AUTO_TEST_CASE(array_pop_array_transition)
@@ -3475,9 +3499,12 @@ BOOST_AUTO_TEST_CASE(array_pop_storage_empty)
 			}
 		}
 	)";
+	ALSO_VIA_YUL(
+		DISABLE_EWASM_TESTRUN()
 		compileAndRun(sourceCode);
 		ABI_CHECK(callContractFunction("test()"), encodeArgs());
 		BOOST_CHECK(storageEmpty(m_contractAddress));
+	);
 }
 
 BOOST_AUTO_TEST_CASE(byte_array_pop_storage_empty)
@@ -3495,9 +3522,12 @@ BOOST_AUTO_TEST_CASE(byte_array_pop_storage_empty)
 			}
 		}
 	)";
+	ALSO_VIA_YUL(
+		DISABLE_EWASM_TESTRUN()
 		compileAndRun(sourceCode);
 		ABI_CHECK(callContractFunction("test()"), encodeArgs());
 		BOOST_CHECK(storageEmpty(m_contractAddress));
+	);
 }
 
 BOOST_AUTO_TEST_CASE(byte_array_pop_long_storage_empty)
@@ -3520,9 +3550,12 @@ BOOST_AUTO_TEST_CASE(byte_array_pop_long_storage_empty)
 			}
 		}
 	)";
+	ALSO_VIA_YUL(
+		DISABLE_EWASM_TESTRUN()
 		compileAndRun(sourceCode);
 		ABI_CHECK(callContractFunction("test()"), encodeArgs(true));
 		BOOST_CHECK(storageEmpty(m_contractAddress));
+	);
 }
 
 BOOST_AUTO_TEST_CASE(byte_array_pop_long_storage_empty_garbage_ref)
@@ -3598,15 +3631,19 @@ BOOST_AUTO_TEST_CASE(bytes_index_access)
 			}
 		}
 	)";
-	compileAndRun(sourceCode);
 	string array{
 		0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
 		10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
 		20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
 		30, 31, 32, 33};
+	ALSO_VIA_YUL(
+		DISABLE_EWASM_TESTRUN()
+
+		compileAndRun(sourceCode);
 		ABI_CHECK(callContractFunction("direct(bytes,uint256)", 64, 33, u256(array.length()), array), encodeArgs(33));
 		ABI_CHECK(callContractFunction("storageCopyRead(bytes,uint256)", 64, 33, u256(array.length()), array), encodeArgs(33));
 		ABI_CHECK(callContractFunction("storageWrite()"), encodeArgs(0x193));
+	);
 }
 
 BOOST_AUTO_TEST_CASE(array_copy_calldata_storage)

test/libsolidity/semanticTests/abiEncoderV1/abi_decode_v2_storage.sol

@@ -20,5 +20,7 @@ contract C {
     }
 }
 
+// ====
+// compileViaYul: also
 // ----
 // f() -> 0x20, 0x8, 0x40, 0x3, 0x9, 0xa, 0xb

test/libsolidity/semanticTests/abiencodedecode/abi_decode_simple_storage.sol

@@ -7,5 +7,7 @@ contract C {
     }
 }
 
+// ====
+// compileViaYul: also
 // ----
 // f(bytes): 0x20, 0x80, 0x21, 0x40, 0x7, "abcdefg" -> 0x21, 0x40, 0x7, "abcdefg"

test/libsolidity/semanticTests/array/bytes_delete_element.sol

@@ -14,5 +14,7 @@ contract c {
             uint8(data[97]) == 97;
     }
 }
+// ====
+// compileViaYul: also
 // ----
 // test1() -> true

test/libsolidity/semanticTests/array/bytes_length_member.sol

@@ -10,6 +10,8 @@ contract c {
 
     bytes data;
 }
+// ====
+// compileViaYul: also
 // ----
 // getLength() -> 0
 // set(): 1, 2 -> true

test/libsolidity/semanticTests/array/copying/bytes_memory_to_storage.sol

@@ -6,5 +6,7 @@ contract C {
     return s[0];
   }
 }
+// ====
+// compileViaYul: also
 // ----
 // f() -> "a"

test/libsolidity/semanticTests/array/copying/calldata_bytes_to_storage.sol

@@ -5,5 +5,7 @@ contract C {
     return s[0];
   }
 }
+// ====
+// compileViaYul: also
 // ----
 // f(bytes): 0x20, 0x08, "abcdefgh" -> "a"

test/libsolidity/semanticTests/array/copying/copy_byte_array_in_struct_to_storage.sol

@@ -0,0 +1,42 @@
+pragma experimental ABIEncoderV2;
+struct S {
+    uint16 x;
+    bytes a;
+    uint16 y;
+    bytes b;
+}
+contract C {
+    uint padding;
+    S data;
+
+    function f() public returns (bytes memory, bytes memory) {
+        S memory x;
+        x.x = 7;
+        x.b = "1234567890123456789012345678901 1234567890123456789012345678901 123456789";
+        x.a = "abcdef";
+        x.y = 9;
+        data = x;
+        return (data.a, data.b);
+    }
+    function g() public returns (bytes memory, bytes memory) {
+        S memory x;
+        x.x = 7;
+        x.b = "12345678923456789";
+        x.a = "1234567890123456789012345678901 1234567890123456789012345678901 123456789";
+        x.y = 9;
+        data = x;
+        return (data.a, data.b);
+    }
+    function h() public returns (bytes memory, bytes memory) {
+        S memory x;
+        data = x;
+        return (data.a, data.b);
+    }
+}
+// ====
+// compileViaYul: also
+// ----
+// f() -> 0x40, 0x80, 6, 0x6162636465660000000000000000000000000000000000000000000000000000, 0x49, 0x3132333435363738393031323334353637383930313233343536373839303120, 0x3132333435363738393031323334353637383930313233343536373839303120, 0x3132333435363738390000000000000000000000000000000000000000000000
+// g() -> 0x40, 0xc0, 0x49, 0x3132333435363738393031323334353637383930313233343536373839303120, 0x3132333435363738393031323334353637383930313233343536373839303120, 0x3132333435363738390000000000000000000000000000000000000000000000, 0x11, 0x3132333435363738393233343536373839000000000000000000000000000000
+// h() -> 0x40, 0x60, 0x00, 0x00
+// storage: empty

test/libsolidity/semanticTests/array/copying/copy_byte_array_to_storage.sol

@@ -0,0 +1,50 @@
+function dataslot() pure returns (bytes32) {
+    return keccak256(abi.encode(1));
+}
+
+function readDataSlot(uint offset) view returns (bytes32 r) {
+    bytes32 s = dataslot();
+    assembly { r := sload(add(s, offset)) }
+}
+
+function readDataSlot() view returns (bytes32) {
+    return readDataSlot(0);
+}
+
+function readHead() view returns (bytes32 r) {
+    assembly { r := sload(1) }
+}
+
+contract C {
+    uint padding;
+    bytes data;
+
+    function f() public returns (uint) {
+        bytes32 zero;
+        if (!(readDataSlot() == zero)) return 1;
+        data = "abc";
+        if (!(readDataSlot() == zero)) return 2;
+        data = "1234567890123456789012345678901234567890123456789012345678901234567890";
+        if (!(readDataSlot() != zero)) return 3;
+        if (!(readDataSlot(1) != zero)) return 4;
+        if (!(readDataSlot(2) != zero)) return 5;
+        if (!(readDataSlot(3) == zero)) return 6;
+        if (!(readDataSlot(4) == zero)) return 7;
+        data = "abc";
+        if (!(readDataSlot() == zero)) return 8;
+        if (!(readDataSlot(1) == zero)) return 9;
+        if (!(readDataSlot(2) == zero)) return 10;
+        if (!(readDataSlot(3) == zero)) return 11;
+        data = "1234567890123456789012345678901234567890123456789012345678901234567890";
+        data = "123456789012345678901234567890123456";
+        if (!(readDataSlot() != zero)) return 12;
+        if (!(readDataSlot(1) != zero)) return 13;
+        if (!(readDataSlot(2) == zero)) return 14;
+        if (!(readDataSlot(3) == zero)) return 15;
+        return 0xff;
+    }
+}
+// ====
+// compileViaYul: also
+// ----
+// f() -> 0xff

test/libsolidity/semanticTests/storage/empty_nonempty_empty.sol

@@ -2,6 +2,8 @@ contract Test {
     bytes x;
     function set(bytes memory _a) public { x = _a; }
 }
+// ====
+// compileViaYul: also
 // ----
 // set(bytes): 0x20, 3, "abc"
 // storage: nonempty

test/libsolidity/semanticTests/various/destructuring_assignment.sol

@@ -32,5 +32,7 @@ contract C {
     }
 }
 
+// ====
+// compileViaYul: also
 // ----
 // f(bytes): 0x20, 0x5, "abcde" -> 0

test/libsolidity/semanticTests/various/store_bytes.sol

@@ -8,6 +8,8 @@ contract C {
 
     bytes savedData;
 }
+// ====
+// compileViaYul: also
 // ----
 // save() -> 24 # empty copy loop #
 // save(): "abcdefg" -> 24