From d235d0c1660a89e302d66b96f7e5dc8073dedae0 Mon Sep 17 00:00:00 2001 From: Djordje Mijovic Date: Tue, 5 May 2020 19:53:17 +0200 Subject: [PATCH] [Sol->Yul] Implementing Byte array push() and pop() --- libsolidity/codegen/YulUtilFunctions.cpp | 168 +++++++++++++++--- libsolidity/codegen/YulUtilFunctions.h | 8 + .../codegen/ir/IRGeneratorForStatements.cpp | 2 + .../semanticTests/array/byte_array_pop.sol | 3 +- .../array/byte_array_pop_empty_exception.sol | 2 + .../semanticTests/array/byte_array_push.sol | 3 +- .../array/byte_array_push_transition.sol | 3 +- .../array/byte_array_storage_layout.sol | 45 +++++ .../array/byte_array_transitional_2.sol | 21 +++ 9 files changed, 227 insertions(+), 28 deletions(-) create mode 100644 test/libsolidity/semanticTests/array/byte_array_storage_layout.sol create mode 100644 test/libsolidity/semanticTests/array/byte_array_transitional_2.sol diff --git a/libsolidity/codegen/YulUtilFunctions.cpp b/libsolidity/codegen/YulUtilFunctions.cpp index 34b2e1279..64639efb1 100644 --- a/libsolidity/codegen/YulUtilFunctions.cpp +++ b/libsolidity/codegen/YulUtilFunctions.cpp @@ -602,6 +602,25 @@ string YulUtilFunctions::overflowCheckedIntSubFunction(IntegerType const& _type) }); } +string YulUtilFunctions::extractByteArrayLengthFunction() +{ + string functionName = "extract_byte_array_length"; + return m_functionCollector.createFunction(functionName, [&]() { + Whiskers w(R"( + function (data) -> length { + // Retrieve length both for in-place strings and off-place strings: + // Computes (x & (0x100 * (ISZERO (x & 1)) - 1)) / 2 + // i.e. for short strings (x & 1 == 0) it does (x & 0xff) / 2 and for long strings it + // computes (x & (-1)) / 2, which is equivalent to just x / 2. + let mask := sub(mul(0x100, iszero(and(data, 1))), 1) + length := div(and(data, mask), 2) + } + )"); + w("functionName", functionName); + return w.render(); + }); +} + string YulUtilFunctions::arrayLengthFunction(ArrayType const& _type) { string functionName = "array_length_" + _type.identifier(); @@ -615,12 +634,7 @@ string YulUtilFunctions::arrayLengthFunction(ArrayType const& _type) length := sload(value) - // Retrieve length both for in-place strings and off-place strings: - // Computes (x & (0x100 * (ISZERO (x & 1)) - 1)) / 2 - // i.e. for short strings (x & 1 == 0) it does (x & 0xff) / 2 and for long strings it - // computes (x & (-1)) / 2, which is equivalent to just x / 2. - let mask := sub(mul(0x100, iszero(and(length, 1))), 1) - length := div(and(length, mask), 2) + length := (length) @@ -634,7 +648,12 @@ string YulUtilFunctions::arrayLengthFunction(ArrayType const& _type) w("length", toCompactHexWithPrefix(_type.length())); w("memory", _type.location() == DataLocation::Memory); w("storage", _type.location() == DataLocation::Storage); - w("byteArray", _type.isByteArray()); + if (_type.location() == DataLocation::Storage) + { + w("byteArray", _type.isByteArray()); + if (_type.isByteArray()) + w("extractByteArrayLength", extractByteArrayLengthFunction()); + } if (_type.isDynamicallySized()) solAssert( _type.location() != DataLocation::CallData, @@ -689,8 +708,9 @@ string YulUtilFunctions::storageArrayPopFunction(ArrayType const& _type) { solAssert(_type.location() == DataLocation::Storage, ""); solAssert(_type.isDynamicallySized(), ""); - solUnimplementedAssert(!_type.isByteArray(), "Byte Arrays not yet implemented!"); solUnimplementedAssert(_type.baseType()->storageBytes() <= 32, "Base type is not yet implemented."); + if (_type.isByteArray()) + return storageByteArrayPopFunction(_type); string functionName = "array_pop_" + _type.identifier(); return m_functionCollector.createFunction(functionName, [&]() { @@ -699,10 +719,8 @@ string YulUtilFunctions::storageArrayPopFunction(ArrayType const& _type) let oldLen := (array) if iszero(oldLen) { invalid() } let newLen := sub(oldLen, 1) - let slot, offset := (array, newLen) (slot, offset) - sstore(array, newLen) })") ("functionName", functionName) @@ -713,29 +731,115 @@ string YulUtilFunctions::storageArrayPopFunction(ArrayType const& _type) }); } +string YulUtilFunctions::storageByteArrayPopFunction(ArrayType const& _type) +{ + solAssert(_type.location() == DataLocation::Storage, ""); + solAssert(_type.isDynamicallySized(), ""); + solAssert(_type.isByteArray(), ""); + + string functionName = "byte_array_pop_" + _type.identifier(); + return m_functionCollector.createFunction(functionName, [&]() { + return Whiskers(R"( + function (array) { + let data := sload(array) + let oldLen := (data) + if iszero(oldLen) { invalid() } + + switch eq(oldLen, 32) + case 1 { + // Here we have a special case where array transitions to shorter than 32 + // So we need to copy data + let copyFromSlot := (array) + data := sload(copyFromSlot) + sstore(copyFromSlot, 0) + // New length is 31, encoded to 31 * 2 = 62 + data := or(and(data, not(0xff)), 62) + } + default { + data := sub(data, 2) + let newLen := sub(oldLen, 1) + switch lt(oldLen, 32) + case 1 { + // set last element to zero + let mask := not((mul(8, sub(31, newLen)), 0xff)) + data := and(data, mask) + } + default { + let slot, offset := (array, newLen) + (slot, offset) + } + } + sstore(array, data) + })") + ("functionName", functionName) + ("extractByteArrayLength", extractByteArrayLengthFunction()) + ("dataAreaFunction", arrayDataAreaFunction(_type)) + ("indexAccess", storageArrayIndexAccessFunction(_type)) + ("setToZero", storageSetToZeroFunction(*_type.baseType())) + ("shl", shiftLeftFunctionDynamic()) + .render(); + }); +} + string YulUtilFunctions::storageArrayPushFunction(ArrayType const& _type) { solAssert(_type.location() == DataLocation::Storage, ""); solAssert(_type.isDynamicallySized(), ""); - solUnimplementedAssert(!_type.isByteArray(), "Byte Arrays not yet implemented!"); solUnimplementedAssert(_type.baseType()->storageBytes() <= 32, "Base type is not yet implemented."); string functionName = "array_push_" + _type.identifier(); return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (array, value) { - let oldLen := (array) - if iszero(lt(oldLen, )) { invalid() } - sstore(array, add(oldLen, 1)) + + let data := sload(array) + let oldLen := (data) + if iszero(lt(oldLen, )) { invalid() } - let slot, offset := (array, oldLen) - (slot, offset, value) + switch gt(oldLen, 31) + case 0 { + value := byte(0, value) + switch oldLen + case 31 { + // Here we have special case when array switches from short array to long array + // We need to copy data + let dataArea := (array) + data := and(data, not(0xff)) + sstore(dataArea, or(and(0xff, value), data)) + // New length is 32, encoded as (32 * 2 + 1) + sstore(array, 65) + } + default { + data := add(data, 2) + let shiftBits := mul(8, sub(31, oldLen)) + let valueShifted := (shiftBits, and(0xff, value)) + let mask := (shiftBits, 0xff) + data := or(and(data, not(mask)), valueShifted) + sstore(array, data) + } + } + default { + sstore(array, add(data, 2)) + let slot, offset := (array, oldLen) + (slot, offset, value) + } + + let oldLen := sload(array) + if iszero(lt(oldLen, )) { invalid() } + sstore(array, add(oldLen, 1)) + let slot, offset := (array, oldLen) + (slot, offset, value) + })") ("functionName", functionName) - ("fetchLength", arrayLengthFunction(_type)) + ("extractByteArrayLength", _type.isByteArray() ? extractByteArrayLengthFunction() : "") + ("dataAreaFunction", arrayDataAreaFunction(_type)) + ("isByteArray", _type.isByteArray()) ("indexAccess", storageArrayIndexAccessFunction(_type)) ("storeValue", updateStorageValueFunction(*_type.baseType())) ("maxArrayLength", (u256(1) << 64).str()) + ("shl", shiftLeftFunctionDynamic()) + ("shr", shiftRightFunction(248)) .render(); }); } @@ -947,21 +1051,33 @@ string YulUtilFunctions::arrayDataAreaFunction(ArrayType const& _type) string YulUtilFunctions::storageArrayIndexAccessFunction(ArrayType const& _type) { - solUnimplementedAssert(_type.baseType()->storageBytes() > 16, ""); - string functionName = "storage_array_index_access_" + _type.identifier(); return m_functionCollector.createFunction(functionName, [&]() { return Whiskers(R"( function (array, index) -> slot, offset { - if iszero(lt(index, (array))) { - invalid() - } + let arrayLength := (array) + if iszero(lt(index, arrayLength)) { invalid() } - let data := (array) - + + offset := sub(31, mod(index, 0x20)) + switch lt(arrayLength, 0x20) + case 0 { + let dataArea := (array) + slot := add(dataArea, div(index, 0x20)) + } + default { + slot := array + } + + let itemsPerSlot := div(0x20, ) + let dataArea := (array) + slot := add(dataArea, div(index, itemsPerSlot)) + offset := mod(index, itemsPerSlot) + - slot := add(data, mul(index, )) + let dataArea := (array) + slot := add(dataArea, mul(index, )) offset := 0 } @@ -970,7 +1086,9 @@ string YulUtilFunctions::storageArrayIndexAccessFunction(ArrayType const& _type) ("arrayLen", arrayLengthFunction(_type)) ("dataAreaFunc", arrayDataAreaFunction(_type)) ("multipleItemsPerSlot", _type.baseType()->storageBytes() <= 16) + ("isBytesArray", _type.isByteArray()) ("storageSize", _type.baseType()->storageSize().str()) + ("storageBytes", toString(_type.baseType()->storageBytes())) .render(); }); } diff --git a/libsolidity/codegen/YulUtilFunctions.h b/libsolidity/codegen/YulUtilFunctions.h index e72d6d293..c23a7d735 100644 --- a/libsolidity/codegen/YulUtilFunctions.h +++ b/libsolidity/codegen/YulUtilFunctions.h @@ -364,6 +364,14 @@ private: /// use exactly one variable to hold the value. std::string conversionFunctionSpecial(Type const& _from, Type const& _to); + /// @returns function name that extracts and returns byte array length + /// signature: (data) -> length + std::string extractByteArrayLengthFunction(); + + /// @returns the name of a function that reduces the size of a storage byte array by one element + /// signature: (byteArray) + std::string storageByteArrayPopFunction(ArrayType const& _type); + std::string readFromMemoryOrCalldata(Type const& _type, bool _fromCalldata); langutil::EVMVersion m_evmVersion; diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp index c9f5d403e..58cd606ba 100644 --- a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp +++ b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp @@ -963,10 +963,12 @@ void IRGeneratorForStatements::endVisit(FunctionCall const& _functionCall) ")\n"; break; } + case FunctionType::Kind::ByteArrayPush: case FunctionType::Kind::ArrayPush: { auto const& memberAccessExpression = dynamic_cast(_functionCall.expression()).expression(); ArrayType const& arrayType = dynamic_cast(*memberAccessExpression.annotation().type); + if (arguments.empty()) { auto slotName = m_context.newYulVariable(); diff --git a/test/libsolidity/semanticTests/array/byte_array_pop.sol b/test/libsolidity/semanticTests/array/byte_array_pop.sol index 5ed849702..3f0e05b34 100644 --- a/test/libsolidity/semanticTests/array/byte_array_pop.sol +++ b/test/libsolidity/semanticTests/array/byte_array_pop.sol @@ -12,6 +12,7 @@ contract c { l = data.length; } } - +// ==== +// compileViaYul: also // ---- // test() -> 2, 1, 1 diff --git a/test/libsolidity/semanticTests/array/byte_array_pop_empty_exception.sol b/test/libsolidity/semanticTests/array/byte_array_pop_empty_exception.sol index 08aea4254..abe8c737b 100644 --- a/test/libsolidity/semanticTests/array/byte_array_pop_empty_exception.sol +++ b/test/libsolidity/semanticTests/array/byte_array_pop_empty_exception.sol @@ -9,5 +9,7 @@ contract c { return true; } } +// ==== +// compileViaYul: also // ---- // test() -> FAILURE diff --git a/test/libsolidity/semanticTests/array/byte_array_push.sol b/test/libsolidity/semanticTests/array/byte_array_push.sol index 67ed87b69..1a9aa3cda 100644 --- a/test/libsolidity/semanticTests/array/byte_array_push.sol +++ b/test/libsolidity/semanticTests/array/byte_array_push.sol @@ -13,6 +13,7 @@ contract c { if (l != 0x03) return true; } } - +// ==== +// compileViaYul: also // ---- // test() -> false diff --git a/test/libsolidity/semanticTests/array/byte_array_push_transition.sol b/test/libsolidity/semanticTests/array/byte_array_push_transition.sol index ceecf6726..f0733a7f5 100644 --- a/test/libsolidity/semanticTests/array/byte_array_push_transition.sol +++ b/test/libsolidity/semanticTests/array/byte_array_push_transition.sol @@ -13,6 +13,7 @@ contract c { return 0; } } - +// ==== +// compileViaYul: also // ---- // test() -> 0 diff --git a/test/libsolidity/semanticTests/array/byte_array_storage_layout.sol b/test/libsolidity/semanticTests/array/byte_array_storage_layout.sol new file mode 100644 index 000000000..b04ad9e46 --- /dev/null +++ b/test/libsolidity/semanticTests/array/byte_array_storage_layout.sol @@ -0,0 +1,45 @@ +contract c { + bytes data; + function test_short() public returns (uint256 r) { + assembly { + sstore(data_slot, 0) + } + for (uint8 i = 0; i < 15; i++) { + data.push(bytes1(i)); + } + assembly { + r := sload(data_slot) + } + } + + function test_long() public returns (uint256 r) { + assembly { + sstore(data_slot, 0) + } + for (uint8 i = 0; i < 33; i++) { + data.push(bytes1(i)); + } + assembly { + r := sload(data_slot) + } + } + + function test_pop() public returns (uint256 r) { + assembly { + sstore(data_slot, 0) + } + for (uint8 i = 0; i < 32; i++) { + data.push(bytes1(i)); + } + data.pop(); + assembly { + r := sload(data_slot) + } + } +} +// ==== +// compileViaYul: also +// ---- +// test_short() -> 1780731860627700044960722568376587075150542249149356309979516913770823710 +// test_long() -> 67 +// test_pop() -> 1780731860627700044960722568376592200742329637303199754547598369979440702 diff --git a/test/libsolidity/semanticTests/array/byte_array_transitional_2.sol b/test/libsolidity/semanticTests/array/byte_array_transitional_2.sol new file mode 100644 index 000000000..46ed1f728 --- /dev/null +++ b/test/libsolidity/semanticTests/array/byte_array_transitional_2.sol @@ -0,0 +1,21 @@ +// Tests transition between short and long encoding both ways +contract c { + bytes data; + + function test() public returns (uint256) { + for (uint8 i = 0; i < 33; i++) { + data.push(bytes1(i)); + } + for (uint8 i = 0; i < data.length; i++) + if (data[i] != bytes1(i)) return i; + data.pop(); + data.pop(); + for (uint8 i = 0; i < data.length; i++) + if (data[i] != bytes1(i)) return i; + return 0; + } +} +// ==== +// compileViaYul: also +// ---- +// test() -> 0