Merge pull request #11011 from ethereum/bytesConcat

[Sol->Yul] Implementing bytes concat
This commit is contained in:
chriseth 2021-03-24 12:10:58 +01:00 committed by GitHub
commit a99eb17608
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 355 additions and 3 deletions

View File

@ -1,7 +1,7 @@
### 0.8.4 (unreleased)
Language Features:
* Possibility to use ``bytes.concat`` with variable number of ``bytes`` and ``bytesNN`` arguments which behaves as a restricted version of `abi.encodePacked` with a more descriptive name.
Compiler Features:

View File

@ -82,6 +82,8 @@ Global Variables
the given arguments starting from the second and prepends the given four-byte selector
- ``abi.encodeWithSignature(string memory signature, ...) returns (bytes memory)``: Equivalent
to ``abi.encodeWithSelector(bytes4(keccak256(bytes(signature)), ...)```
- ``bytes.concat(...) returns (bytes memory)``: :ref:`Concatenates variable number of
arguments to one byte array<bytes-concat>`
- ``block.chainid`` (``uint``): current chain id
- ``block.coinbase`` (``address payable``): current block miner's address
- ``block.difficulty`` (``uint``): current block difficulty

View File

@ -146,7 +146,7 @@ length or index access.
Solidity does not have string manipulation functions, but there are
third-party string libraries. You can also compare two strings by their keccak256-hash using
``keccak256(abi.encodePacked(s1)) == keccak256(abi.encodePacked(s2))`` and
concatenate two strings using ``abi.encodePacked(s1, s2)``.
concatenate two strings using ``bytes.concat(bytes(s1), bytes(s2))``.
You should use ``bytes`` over ``byte[]`` because it is cheaper,
since ``byte[]`` adds 31 padding bytes between the elements. As a general rule,
@ -160,6 +160,32 @@ always use one of the value types ``bytes1`` to ``bytes32`` because they are muc
that you are accessing the low-level bytes of the UTF-8 representation,
and not the individual characters.
.. index:: ! bytes-concat
.. _bytes-concat:
``bytes.concat`` function
^^^^^^^^^^^^^^^^^^^^^^^^^
You can concatenate a variable number of ``bytes`` or ``bytes1 ... bytes32`` using ``bytes.concat``.
The function returns a single ``bytes memory`` array that contains the contents of the arguments without padding.
If you want to use string parameters or other types, you need to convert them to ``bytes`` or ``bytes1``/.../``bytes32`` first.
::
// SPDX-License-Identifier: GPL-3.0
pragma solidity >0.8.3;
contract C {
bytes s = "Storage";
function f(bytes calldata c, string memory m, bytes16 b) public view {
bytes memory a = bytes.concat(s, c, c[:2], "Literal", bytes(m), b);
assert((s.length + c.length + 2 + 7 + bytes(m).length + 16) == a.length);
}
}
If you call ``bytes.concat`` without arguments it will return an empty ``bytes`` array.
.. index:: ! array;allocating, new
Allocating Memory Arrays
@ -582,4 +608,4 @@ assigning it to a local variable, as in
.. note::
Until Solidity 0.7.0, memory-structs containing members of storage-only types (e.g. mappings)
were allowed and assignments like ``campaigns[campaignID] = Campaign(beneficiary, goal, 0, 0)``
in the example above would work and just silently skip those members.
in the example above would work and just silently skip those members.

View File

@ -136,6 +136,13 @@ ABI Encoding and Decoding Functions
See the documentation about the :ref:`ABI <ABI>` and the
:ref:`tightly packed encoding <abi_packed_mode>` for details about the encoding.
.. index:: bytes members
Members of bytes
----------------
- ``bytes.concat(...) returns (bytes memory)``: :ref:`Concatenates variable number of bytes and bytes1, ..., bytes32 arguments to one byte array<bytes-concat>`
.. index:: assert, revert, require
Error Handling

View File

@ -2027,6 +2027,32 @@ void TypeChecker::typeCheckABIEncodeFunctions(
}
}
void TypeChecker::typeCheckBytesConcatFunction(
FunctionCall const& _functionCall,
FunctionType const* _functionType
)
{
solAssert(_functionType, "");
solAssert(_functionType->kind() == FunctionType::Kind::BytesConcat, "");
solAssert(_functionCall.names().empty(), "");
typeCheckFunctionGeneralChecks(_functionCall, _functionType);
for (shared_ptr<Expression const> const& argument: _functionCall.arguments())
if (
Type const* argumentType = type(*argument);
!argumentType->isImplicitlyConvertibleTo(*TypeProvider::fixedBytes(32)) &&
!argumentType->isImplicitlyConvertibleTo(*TypeProvider::bytesMemory())
)
m_errorReporter.typeError(
8015_error,
argument->location(),
"Invalid type for argument in the bytes.concat function call. "
"bytes or fixed bytes type is required, but " +
argumentType->toString(true) + " provided."
);
}
void TypeChecker::typeCheckFunctionGeneralChecks(
FunctionCall const& _functionCall,
FunctionTypePointer _functionType
@ -2433,6 +2459,12 @@ bool TypeChecker::visit(FunctionCall const& _functionCall)
case FunctionType::Kind::MetaType:
returnTypes = typeCheckMetaTypeFunctionAndRetrieveReturnType(_functionCall);
break;
case FunctionType::Kind::BytesConcat:
{
typeCheckBytesConcatFunction(_functionCall, functionType);
returnTypes = functionType->returnParameterTypes();
break;
}
default:
{
typeCheckFunctionCall(_functionCall, functionType);

View File

@ -111,6 +111,12 @@ private:
FunctionTypePointer _functionType
);
/// Performs general checks and checks specific to bytes concat function call
void typeCheckBytesConcatFunction(
FunctionCall const& _functionCall,
FunctionType const* _functionType
);
void endVisit(InheritanceSpecifier const& _inheritance) override;
void endVisit(ModifierDefinition const& _modifier) override;
bool visit(FunctionDefinition const& _function) override;

View File

@ -2877,6 +2877,7 @@ string FunctionType::richIdentifier() const
case Kind::MulMod: id += "mulmod"; break;
case Kind::ArrayPush: id += "arraypush"; break;
case Kind::ArrayPop: id += "arraypop"; break;
case Kind::BytesConcat: id += "bytesconcat"; break;
case Kind::ObjectCreation: id += "objectcreation"; break;
case Kind::Assert: id += "assert"; break;
case Kind::Require: id += "require"; break;
@ -3736,6 +3737,20 @@ MemberList::MemberMap TypeType::nativeMembers(ASTNode const* _currentScope) cons
for (ASTPointer<EnumValue> const& enumValue: enumDef.members())
members.emplace_back(enumValue.get(), enumType);
}
else if (
auto const* arrayType = dynamic_cast<ArrayType const*>(m_actualType);
arrayType && arrayType->isByteArray()
)
members.emplace_back("concat", TypeProvider::function(
TypePointers{},
TypePointers{TypeProvider::bytesMemory()},
strings{},
strings{string()},
FunctionType::Kind::BytesConcat,
/* _arbitraryParameters */ true,
StateMutability::Pure
));
return members;
}

View File

@ -1158,6 +1158,7 @@ public:
MulMod, ///< MULMOD
ArrayPush, ///< .push() to a dynamically sized array in storage
ArrayPop, ///< .pop() from a dynamically sized array in storage
BytesConcat, ///< .concat() on bytes (type type)
ObjectCreation, ///< array creation using new
Assert, ///< assert()
Require, ///< require()

View File

@ -1021,6 +1021,42 @@ bool ExpressionCompiler::visit(FunctionCall const& _functionCall)
ArrayUtils(m_context).popStorageArrayElement(*arrayType);
break;
}
case FunctionType::Kind::BytesConcat:
{
_functionCall.expression().accept(*this);
vector<Type const*> argumentTypes;
vector<Type const*> targetTypes;
for (auto const& argument: arguments)
{
argument->accept(*this);
solAssert(argument->annotation().type, "");
argumentTypes.emplace_back(argument->annotation().type);
if (argument->annotation().type->category() == Type::Category::FixedBytes)
targetTypes.emplace_back(argument->annotation().type);
else if (
auto const* literalType = dynamic_cast<StringLiteralType const*>(argument->annotation().type);
literalType && literalType->value().size() <= 32
)
targetTypes.emplace_back(TypeProvider::fixedBytes(static_cast<unsigned>(literalType->value().size())));
else
{
solAssert(argument->annotation().type->isImplicitlyConvertibleTo(*TypeProvider::bytesMemory()), "");
targetTypes.emplace_back(TypeProvider::bytesMemory());
}
}
utils().fetchFreeMemoryPointer();
// stack: <arg1> <arg2> ... <argn> <free mem>
m_context << u256(32) << Instruction::ADD;
utils().packedEncode(argumentTypes, targetTypes);
utils().fetchFreeMemoryPointer();
m_context.appendInlineAssembly(R"({
mstore(mem_ptr, sub(sub(mem_end, mem_ptr), 0x20))
})", {"mem_end", "mem_ptr"});
m_context << Instruction::SWAP1;
utils().storeFreeMemoryPointer();
break;
}
case FunctionType::Kind::ObjectCreation:
{
ArrayType const& arrayType = dynamic_cast<ArrayType const&>(*_functionCall.annotation().type);

View File

@ -2351,6 +2351,60 @@ string YulUtilFunctions::copyArrayFromStorageToMemoryFunction(ArrayType const& _
});
}
string YulUtilFunctions::bytesConcatFunction(vector<Type const*> const& _argumentTypes)
{
string functionName = "bytes_concat";
size_t totalParams = 0;
vector<Type const*> targetTypes;
for (Type const* argumentType: _argumentTypes)
{
solAssert(
argumentType->isImplicitlyConvertibleTo(*TypeProvider::bytesMemory()) ||
argumentType->isImplicitlyConvertibleTo(*TypeProvider::fixedBytes(32)),
""
);
if (argumentType->category() == Type::Category::FixedBytes)
targetTypes.emplace_back(argumentType);
else if (
auto const* literalType = dynamic_cast<StringLiteralType const*>(argumentType);
literalType && literalType->value().size() <= 32
)
targetTypes.emplace_back(TypeProvider::fixedBytes(static_cast<unsigned>(literalType->value().size())));
else
{
solAssert(argumentType->isImplicitlyConvertibleTo(*TypeProvider::bytesMemory()), "");
targetTypes.emplace_back(TypeProvider::bytesMemory());
}
totalParams += argumentType->sizeOnStack();
functionName += "_" + argumentType->identifier();
}
return m_functionCollector.createFunction(functionName, [&]() {
Whiskers templ(R"(
function <functionName>(<parameters>) -> outPtr {
outPtr := <allocateUnbounded>()
let dataStart := add(outPtr, 0x20)
let dataEnd := <encodePacked>(dataStart<?+parameters>, <parameters></+parameters>)
mstore(outPtr, sub(dataEnd, dataStart))
<finalizeAllocation>(outPtr, sub(dataEnd, outPtr))
}
)");
templ("functionName", functionName);
templ("parameters", suffixedVariableNameList("param_", 0, totalParams));
templ("allocateUnbounded", allocateUnboundedFunction());
templ("finalizeAllocation", finalizeAllocationFunction());
templ(
"encodePacked",
ABIFunctions{m_evmVersion, m_revertStrings, m_functionCollector}.tupleEncoderPacked(
_argumentTypes,
targetTypes
)
);
return templ.render();
});
}
string YulUtilFunctions::mappingIndexAccessFunction(MappingType const& _mappingType, Type const& _keyType)
{
string functionName = "mapping_index_access_" + _mappingType.identifier() + "_of_" + _keyType.identifier();

View File

@ -292,6 +292,10 @@ public:
/// of the storage array into it.
std::string copyArrayFromStorageToMemoryFunction(ArrayType const& _from, ArrayType const& _to);
/// @returns the name of a function that does concatenation of variadic number of bytes
/// or fixed bytes
std::string bytesConcatFunction(std::vector<Type const*> const& _argumentTypes);
/// @returns the name of a function that performs index access for mappings.
/// @param _mappingType the type of the mapping
/// @param _keyType the type of the value provided

View File

@ -1323,6 +1323,23 @@ void IRGeneratorForStatements::endVisit(FunctionCall const& _functionCall)
}
break;
}
case FunctionType::Kind::BytesConcat:
{
TypePointers argumentTypes;
vector<string> argumentVars;
for (ASTPointer<Expression const> const& argument: arguments)
{
argumentTypes.emplace_back(&type(*argument));
argumentVars += IRVariable(*argument).stackSlots();
}
define(IRVariable(_functionCall)) <<
m_utils.bytesConcatFunction(argumentTypes) <<
"(" <<
joinHumanReadable(argumentVars) <<
")\n";
break;
}
case FunctionType::Kind::MetaType:
{
break;
@ -1993,6 +2010,8 @@ void IRGeneratorForStatements::endVisit(MemberAccess const& _memberAccess)
}
else if (EnumType const* enumType = dynamic_cast<EnumType const*>(&actualType))
define(_memberAccess) << to_string(enumType->memberValue(_memberAccess.memberName())) << "\n";
else if (auto const* arrayType = dynamic_cast<ArrayType const*>(&actualType))
solAssert(arrayType->isByteArray() && member == "concat", "");
else
// The old code generator had a generic "else" case here
// without any specific code being generated,

View File

@ -0,0 +1,30 @@
contract C {
function f(bytes memory a, bytes memory b) public returns (bytes memory) {
return bytes.concat(a, b);
}
}
// ====
// compileViaYul: also
// ----
// f(bytes, bytes): 0x40, 0x80, 32, "abcdabcdabcdabcdabcdabcdabcdabcd", 5, "bcdef" -> 0x20, 37, "abcdabcdabcdabcdabcdabcdabcdabcd", "bcdef"
//
// f(bytes, bytes):
// 0x40, 0xa0, 64, "abcdabcdabcdabcdabcdabcdabcdabcd", "abcdabcdabcdabcdabcdabcdabcdabcd", 5, "bcdef"
// ->
// 0x20, 69, "abcdabcdabcdabcdabcdabcdabcdabcd", "abcdabcdabcdabcdabcdabcdabcdabcd", "bcdef"
// f(bytes, bytes): 0x40, 0x80, 3, "abc", 3, "def" -> 0x20, 6, "abcdef"
//
// f(bytes, bytes):
// 0x40, 0xa0, 34, "abcdabcdabcdabcdabcdabcdabcdabcd", "ab", 30, "cdabcdabcdabcdabcdabcdabcdabcd"
// ->
// 0x20, 64, "abcdabcdabcdabcdabcdabcdabcdabcd", "abcdabcdabcdabcdabcdabcdabcdabcd"
//
// f(bytes, bytes):
// 0x40, 0xa0, 34, "abcdabcdabcdabcdabcdabcdabcdabcd", "ab", 34, "cdabcdabcdabcdabcdabcdabcdabcdab", "cd"
// ->
// 0x20, 68, "abcdabcdabcdabcdabcdabcdabcdabcd", "abcdabcdabcdabcdabcdabcdabcdabcd", "abcd"
//
// f(bytes, bytes):
// 0x40, 0x80, 3, "abc", 30, "dabcdabcdabcdabcdabcdabcdabcda"
// ->
// 0x20, 33, "abcdabcdabcdabcdabcdabcdabcdabcd", "a"

View File

@ -0,0 +1,10 @@
contract C {
function f(bytes memory a, bytes memory b, bytes memory c) public returns (bytes memory) {
return bytes.concat(a, b, c);
}
}
// ====
// compileViaYul: also
// ----
// f(bytes, bytes, bytes): 0x60, 0xa0, 0xe0, 32, "abcdabcdabcdabcdabcdabcdabcdabcd", 5, "bcdef", 3, "abc" -> 0x20, 40, "abcdabcdabcdabcdabcdabcdabcdabcd", "bcdefabc"
// f(bytes, bytes, bytes): 0x60, 0xa0, 0xe0, 3, "abc", 2, "de", 3, "fgh" -> 0x20, 8, "abcdefgh"

View File

@ -0,0 +1,18 @@
contract C {
function f(bytes memory a, bytes memory b) public returns (bytes32) {
return keccak256(bytes.concat(a, b));
}
function h(bytes memory a) internal returns (uint256) {
return a.length;
}
function g(bytes memory a, bytes memory b) public returns (uint256) {
return h(bytes.concat(a, b));
}
}
// ====
// compileViaYul: also
// ----
// f(bytes,bytes): 0x40, 0x80, 32, "abcdabcdabcdabcdabcdabcdabcdabcd", 5, "bcdef" -> 0x1106e19b6f06d1cce71c2d816754f83dfa5b95df958c5cbf12b7c472320c427c
// g(bytes,bytes): 0x40, 0x80, 32, "abcdabcdabcdabcdabcdabcdabcdabcd", 5, "bcdef" -> 37

View File

@ -0,0 +1,37 @@
contract C {
bytes s = "bcdef";
function f(bytes memory a) public returns (bytes memory) {
return bytes.concat(a, "bcdef");
}
function g(bytes calldata a) public returns (bytes memory) {
return bytes.concat(a, "abcdefghabcdefghabcdefghabcdefghab");
}
function h(bytes calldata a) public returns (bytes memory) {
return bytes.concat(a, s);
}
function j(bytes calldata a) public returns (bytes memory) {
bytes storage ref = s;
return bytes.concat(a, ref, s);
}
function k(bytes calldata a, string memory b) public returns (bytes memory) {
return bytes.concat(a, bytes(b));
}
function slice(bytes calldata a) public returns (bytes memory) {
require(a.length > 2, "");
return bytes.concat(a[:2], a[2:]);
}
function strParam(string calldata a) public returns (bytes memory) {
return bytes.concat(bytes(a), "bcdef");
}
}
// ====
// compileViaYul: also
// ----
// f(bytes): 0x20, 32, "abcdabcdabcdabcdabcdabcdabcdabcd" -> 0x20, 37, "abcdabcdabcdabcdabcdabcdabcdabcd", "bcdef"
// g(bytes): 0x20, 32, "abcdabcdabcdabcdabcdabcdabcdabcd" -> 0x20, 66, "abcdabcdabcdabcdabcdabcdabcdabcd", "abcdefghabcdefghabcdefghabcdefgh", "ab"
// h(bytes): 0x20, 32, "abcdabcdabcdabcdabcdabcdabcdabcd" -> 0x20, 37, "abcdabcdabcdabcdabcdabcdabcdabcd", "bcdef"
// j(bytes): 0x20, 32, "abcdabcdabcdabcdabcdabcdabcdabcd" -> 0x20, 42, "abcdabcdabcdabcdabcdabcdabcdabcd", "bcdefbcdef"
// k(bytes, string): 0x40, 0x80, 32, "abcdabcdabcdabcdabcdabcdabcdabcd", 5, "bcdef" -> 0x20, 37, "abcdabcdabcdabcdabcdabcdabcdabcd", "bcdef"
// slice(bytes): 0x20, 4, "abcd" -> 0x20, 4, "abcd"
// strParam(string): 0x20, 32, "abcdabcdabcdabcdabcdabcdabcdabcd" -> 0x20, 37, "abcdabcdabcdabcdabcdabcdabcdabcd", "bcdef"

View File

@ -0,0 +1,9 @@
contract C {
function f() public returns (bytes memory) {
return bytes.concat();
}
}
// ====
// compileViaYul: also
// ----
// f() -> 0x20, 0

View File

@ -0,0 +1,9 @@
contract C {
function f(bytes memory a, bytes memory b, bytes memory c) public returns (bytes memory) {
return bytes.concat(bytes.concat(a, b), c);
}
}
// ====
// compileViaYul: also
// ----
// f(bytes, bytes, bytes): 0x60, 0x60, 0x60, 2, "ab" -> 0x20, 6, "ababab"

View File

@ -0,0 +1,7 @@
contract C {
function g() public pure returns (bytes memory) {
return bytes.concat;
}
}
// ----
// TypeError 6359: (82-94): Return argument type function () pure returns (bytes memory) is not implicitly convertible to expected type (type of first return variable) bytes memory.

View File

@ -0,0 +1,8 @@
contract C {
function f() public {
bytes memory a;
bytes memory b = type(bytes).concat(a);
}
}
// ----
// TypeError 4259: (93-98): Invalid type for argument in the function call. A contract type or an integer type is required, but type(bytes) provided.

View File

@ -0,0 +1,8 @@
contract C {
function f() public {
bytes memory a;
bytes memory b = a.concat();
}
}
// ----
// TypeError 9582: (88-96): Member "concat" not found or not visible after argument-dependent lookup in bytes memory.

View File

@ -0,0 +1,14 @@
contract C {
bytes s;
function f(bytes calldata c, string calldata c1) public {
bytes memory a;
bytes16 b;
uint8[] memory num;
bytes1[] memory m;
bytes memory d = bytes.concat(a, b, c, num, s, "abc", m, c1, bytes(c1));
}
}
// ----
// TypeError 8015: (233-236): Invalid type for argument in the bytes.concat function call. bytes or fixed bytes type is required, but uint8[] provided.
// TypeError 8015: (248-249): Invalid type for argument in the bytes.concat function call. bytes or fixed bytes type is required, but bytes1[] provided.
// TypeError 8015: (251-253): Invalid type for argument in the bytes.concat function call. bytes or fixed bytes type is required, but string provided.