Merge pull request #11011 from ethereum/bytesConcat

[Sol->Yul] Implementing bytes concat

Changelog.md

@@ -1,7 +1,7 @@
 ### 0.8.4 (unreleased)
 
 Language Features:
-
+ * Possibility to use ``bytes.concat`` with variable number of ``bytes`` and ``bytesNN`` arguments which behaves as a restricted version of `abi.encodePacked` with a more descriptive name.
 
 Compiler Features:
 

docs/cheatsheet.rst

@@ -82,6 +82,8 @@ Global Variables
   the given arguments starting from the second and prepends the given four-byte selector
 - ``abi.encodeWithSignature(string memory signature, ...) returns (bytes memory)``: Equivalent
   to ``abi.encodeWithSelector(bytes4(keccak256(bytes(signature)), ...)```
+- ``bytes.concat(...) returns (bytes memory)``: :ref:`Concatenates variable number of
+  arguments to one byte array<bytes-concat>`
 - ``block.chainid`` (``uint``): current chain id
 - ``block.coinbase`` (``address payable``): current block miner's address
 - ``block.difficulty`` (``uint``): current block difficulty

docs/types/reference-types.rst

@@ -146,7 +146,7 @@ length or index access.
 Solidity does not have string manipulation functions, but there are
 third-party string libraries. You can also compare two strings by their keccak256-hash using
 ``keccak256(abi.encodePacked(s1)) == keccak256(abi.encodePacked(s2))`` and
-concatenate two strings using ``abi.encodePacked(s1, s2)``.
+concatenate two strings using ``bytes.concat(bytes(s1), bytes(s2))``.
 
 You should use ``bytes`` over ``byte[]`` because it is cheaper,
 since ``byte[]`` adds 31 padding bytes between the elements. As a general rule,
@@ -160,6 +160,32 @@ always use one of the value types ``bytes1`` to ``bytes32`` because they are muc
     that you are accessing the low-level bytes of the UTF-8 representation,
     and not the individual characters.
 
+.. index:: ! bytes-concat
+
+.. _bytes-concat:
+
+``bytes.concat`` function
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+You can concatenate a variable number of ``bytes`` or ``bytes1 ... bytes32`` using ``bytes.concat``.
+The function returns a single ``bytes memory`` array that contains the contents of the arguments without padding.
+If you want to use string parameters or other types, you need to convert them to ``bytes`` or ``bytes1``/.../``bytes32`` first.
+
+::
+
+    // SPDX-License-Identifier: GPL-3.0
+    pragma solidity >0.8.3;
+
+    contract C {
+        bytes s = "Storage";
+        function f(bytes calldata c, string memory m, bytes16 b) public view {
+            bytes memory a = bytes.concat(s, c, c[:2], "Literal", bytes(m), b);
+            assert((s.length + c.length + 2 + 7 + bytes(m).length + 16) == a.length);
+        }
+    }
+
+If you call ``bytes.concat`` without arguments it will return an empty ``bytes`` array.
+
 .. index:: ! array;allocating, new
 
 Allocating Memory Arrays
@@ -582,4 +608,4 @@ assigning it to a local variable, as in
 .. note::
     Until Solidity 0.7.0, memory-structs containing members of storage-only types (e.g. mappings)
     were allowed and assignments like ``campaigns[campaignID] = Campaign(beneficiary, goal, 0, 0)``
-    in the example above would work and just silently skip those members.
+    in the example above would work and just silently skip those members.

docs/units-and-global-variables.rst

@@ -136,6 +136,13 @@ ABI Encoding and Decoding Functions
 See the documentation about the :ref:`ABI <ABI>` and the
 :ref:`tightly packed encoding <abi_packed_mode>` for details about the encoding.
 
+.. index:: bytes members
+
+Members of bytes
+----------------
+
+- ``bytes.concat(...) returns (bytes memory)``: :ref:`Concatenates variable number of bytes and bytes1, ..., bytes32 arguments to one byte array<bytes-concat>`
+
 .. index:: assert, revert, require
 
 Error Handling

libsolidity/analysis/TypeChecker.cpp

@@ -2027,6 +2027,32 @@ void TypeChecker::typeCheckABIEncodeFunctions(
 	}
 }
 
+void TypeChecker::typeCheckBytesConcatFunction(
+	FunctionCall const& _functionCall,
+	FunctionType const* _functionType
+)
+{
+	solAssert(_functionType, "");
+	solAssert(_functionType->kind() == FunctionType::Kind::BytesConcat, "");
+	solAssert(_functionCall.names().empty(), "");
+
+	typeCheckFunctionGeneralChecks(_functionCall, _functionType);
+
+	for (shared_ptr<Expression const> const& argument: _functionCall.arguments())
+		if (
+			Type const* argumentType = type(*argument);
+			!argumentType->isImplicitlyConvertibleTo(*TypeProvider::fixedBytes(32)) &&
+			!argumentType->isImplicitlyConvertibleTo(*TypeProvider::bytesMemory())
+		)
+			m_errorReporter.typeError(
+				8015_error,
+				argument->location(),
+				"Invalid type for argument in the bytes.concat function call. "
+				"bytes or fixed bytes type is required, but " +
+				argumentType->toString(true) + " provided."
+			);
+}
+
 void TypeChecker::typeCheckFunctionGeneralChecks(
 	FunctionCall const& _functionCall,
 	FunctionTypePointer _functionType
@@ -2433,6 +2459,12 @@ bool TypeChecker::visit(FunctionCall const& _functionCall)
 		case FunctionType::Kind::MetaType:
 			returnTypes = typeCheckMetaTypeFunctionAndRetrieveReturnType(_functionCall);
 			break;
+		case FunctionType::Kind::BytesConcat:
+		{
+			typeCheckBytesConcatFunction(_functionCall, functionType);
+			returnTypes = functionType->returnParameterTypes();
+			break;
+		}
 		default:
 		{
 			typeCheckFunctionCall(_functionCall, functionType);

libsolidity/analysis/TypeChecker.h

@@ -111,6 +111,12 @@ private:
 		FunctionTypePointer _functionType
 	);
 
+	/// Performs general checks and checks specific to bytes concat function call
+	void typeCheckBytesConcatFunction(
+		FunctionCall const& _functionCall,
+		FunctionType const* _functionType
+	);
+
 	void endVisit(InheritanceSpecifier const& _inheritance) override;
 	void endVisit(ModifierDefinition const& _modifier) override;
 	bool visit(FunctionDefinition const& _function) override;

libsolidity/ast/Types.cpp

@@ -2877,6 +2877,7 @@ string FunctionType::richIdentifier() const
 	case Kind::MulMod: id += "mulmod"; break;
 	case Kind::ArrayPush: id += "arraypush"; break;
 	case Kind::ArrayPop: id += "arraypop"; break;
+	case Kind::BytesConcat: id += "bytesconcat"; break;
 	case Kind::ObjectCreation: id += "objectcreation"; break;
 	case Kind::Assert: id += "assert"; break;
 	case Kind::Require: id += "require"; break;
@@ -3736,6 +3737,20 @@ MemberList::MemberMap TypeType::nativeMembers(ASTNode const* _currentScope) cons
 		for (ASTPointer<EnumValue> const& enumValue: enumDef.members())
 			members.emplace_back(enumValue.get(), enumType);
 	}
+	else if (
+		auto const* arrayType = dynamic_cast<ArrayType const*>(m_actualType);
+		arrayType && arrayType->isByteArray()
+	)
+		members.emplace_back("concat", TypeProvider::function(
+			TypePointers{},
+			TypePointers{TypeProvider::bytesMemory()},
+			strings{},
+			strings{string()},
+			FunctionType::Kind::BytesConcat,
+			/* _arbitraryParameters */ true,
+			StateMutability::Pure
+		));
+
 	return members;
 }
 

libsolidity/ast/Types.h

@@ -1158,6 +1158,7 @@ public:
 		MulMod, ///< MULMOD
 		ArrayPush, ///< .push() to a dynamically sized array in storage
 		ArrayPop, ///< .pop() from a dynamically sized array in storage
+		BytesConcat, ///< .concat() on bytes (type type)
 		ObjectCreation, ///< array creation using new
 		Assert, ///< assert()
 		Require, ///< require()

libsolidity/codegen/ExpressionCompiler.cpp

@@ -1021,6 +1021,42 @@ bool ExpressionCompiler::visit(FunctionCall const& _functionCall)
 			ArrayUtils(m_context).popStorageArrayElement(*arrayType);
 			break;
 		}
+		case FunctionType::Kind::BytesConcat:
+		{
+			_functionCall.expression().accept(*this);
+			vector<Type const*> argumentTypes;
+			vector<Type const*> targetTypes;
+			for (auto const& argument: arguments)
+			{
+				argument->accept(*this);
+				solAssert(argument->annotation().type, "");
+				argumentTypes.emplace_back(argument->annotation().type);
+				if (argument->annotation().type->category() == Type::Category::FixedBytes)
+					targetTypes.emplace_back(argument->annotation().type);
+				else if (
+					auto const* literalType = dynamic_cast<StringLiteralType const*>(argument->annotation().type);
+					literalType && literalType->value().size() <= 32
+				)
+					targetTypes.emplace_back(TypeProvider::fixedBytes(static_cast<unsigned>(literalType->value().size())));
+				else
+				{
+					solAssert(argument->annotation().type->isImplicitlyConvertibleTo(*TypeProvider::bytesMemory()), "");
+					targetTypes.emplace_back(TypeProvider::bytesMemory());
+				}
+			}
+			utils().fetchFreeMemoryPointer();
+			// stack: <arg1> <arg2> ... <argn> <free mem>
+			m_context << u256(32) << Instruction::ADD;
+			utils().packedEncode(argumentTypes, targetTypes);
+			utils().fetchFreeMemoryPointer();
+			m_context.appendInlineAssembly(R"({
+				mstore(mem_ptr, sub(sub(mem_end, mem_ptr), 0x20))
+			})", {"mem_end", "mem_ptr"});
+			m_context << Instruction::SWAP1;
+			utils().storeFreeMemoryPointer();
+
+			break;
+		}
 		case FunctionType::Kind::ObjectCreation:
 		{
 			ArrayType const& arrayType = dynamic_cast<ArrayType const&>(*_functionCall.annotation().type);

libsolidity/codegen/YulUtilFunctions.cpp

@@ -2351,6 +2351,60 @@ string YulUtilFunctions::copyArrayFromStorageToMemoryFunction(ArrayType const& _
 	});
 }
 
+string YulUtilFunctions::bytesConcatFunction(vector<Type const*> const& _argumentTypes)
+{
+	string functionName = "bytes_concat";
+	size_t totalParams = 0;
+	vector<Type const*> targetTypes;
+	for (Type const* argumentType: _argumentTypes)
+	{
+		solAssert(
+			argumentType->isImplicitlyConvertibleTo(*TypeProvider::bytesMemory()) ||
+			argumentType->isImplicitlyConvertibleTo(*TypeProvider::fixedBytes(32)),
+			""
+		);
+		if (argumentType->category() == Type::Category::FixedBytes)
+			targetTypes.emplace_back(argumentType);
+		else if (
+			auto const* literalType = dynamic_cast<StringLiteralType const*>(argumentType);
+			literalType && literalType->value().size() <= 32
+		)
+			targetTypes.emplace_back(TypeProvider::fixedBytes(static_cast<unsigned>(literalType->value().size())));
+		else
+		{
+			solAssert(argumentType->isImplicitlyConvertibleTo(*TypeProvider::bytesMemory()), "");
+			targetTypes.emplace_back(TypeProvider::bytesMemory());
+		}
+
+		totalParams += argumentType->sizeOnStack();
+		functionName += "_" + argumentType->identifier();
+	}
+
+	return m_functionCollector.createFunction(functionName, [&]() {
+		Whiskers templ(R"(
+			function <functionName>(<parameters>) -> outPtr {
+				outPtr := <allocateUnbounded>()
+				let dataStart := add(outPtr, 0x20)
+				let dataEnd := <encodePacked>(dataStart<?+parameters>, <parameters></+parameters>)
+				mstore(outPtr, sub(dataEnd, dataStart))
+				<finalizeAllocation>(outPtr, sub(dataEnd, outPtr))
+			}
+		)");
+		templ("functionName", functionName);
+		templ("parameters", suffixedVariableNameList("param_", 0, totalParams));
+		templ("allocateUnbounded", allocateUnboundedFunction());
+		templ("finalizeAllocation", finalizeAllocationFunction());
+		templ(
+			"encodePacked",
+			ABIFunctions{m_evmVersion, m_revertStrings, m_functionCollector}.tupleEncoderPacked(
+				_argumentTypes,
+				targetTypes
+			)
+		);
+		return templ.render();
+	});
+}
+
 string YulUtilFunctions::mappingIndexAccessFunction(MappingType const& _mappingType, Type const& _keyType)
 {
 	string functionName = "mapping_index_access_" + _mappingType.identifier() + "_of_" + _keyType.identifier();

libsolidity/codegen/YulUtilFunctions.h

@@ -292,6 +292,10 @@ public:
 	/// of the storage array into it.
 	std::string copyArrayFromStorageToMemoryFunction(ArrayType const& _from, ArrayType const& _to);
 
+	/// @returns the name of a function that does concatenation of variadic number of bytes
+	/// or fixed bytes
+	std::string bytesConcatFunction(std::vector<Type const*> const& _argumentTypes);
+
 	/// @returns the name of a function that performs index access for mappings.
 	/// @param _mappingType the type of the mapping
 	/// @param _keyType the type of the value provided

libsolidity/codegen/ir/IRGeneratorForStatements.cpp

@@ -1323,6 +1323,23 @@ void IRGeneratorForStatements::endVisit(FunctionCall const& _functionCall)
 		}
 		break;
 	}
+	case FunctionType::Kind::BytesConcat:
+	{
+		TypePointers argumentTypes;
+		vector<string> argumentVars;
+		for (ASTPointer<Expression const> const& argument: arguments)
+		{
+			argumentTypes.emplace_back(&type(*argument));
+			argumentVars += IRVariable(*argument).stackSlots();
+		}
+		define(IRVariable(_functionCall)) <<
+			m_utils.bytesConcatFunction(argumentTypes) <<
+			"(" <<
+			joinHumanReadable(argumentVars) <<
+			")\n";
+
+		break;
+	}
 	case FunctionType::Kind::MetaType:
 	{
 		break;
@@ -1993,6 +2010,8 @@ void IRGeneratorForStatements::endVisit(MemberAccess const& _memberAccess)
 		}
 		else if (EnumType const* enumType = dynamic_cast<EnumType const*>(&actualType))
 			define(_memberAccess) << to_string(enumType->memberValue(_memberAccess.memberName())) << "\n";
+		else if (auto const* arrayType = dynamic_cast<ArrayType const*>(&actualType))
+			solAssert(arrayType->isByteArray() && member == "concat", "");
 		else
 			// The old code generator had a generic "else" case here
 			// without any specific code being generated,

test/libsolidity/semanticTests/array/concat/bytes_concat_2_args.sol

@@ -0,0 +1,30 @@
+contract C {
+    function f(bytes memory a, bytes memory b) public returns (bytes memory) {
+        return bytes.concat(a, b);
+    }
+}
+// ====
+// compileViaYul: also
+// ----
+// f(bytes, bytes): 0x40, 0x80, 32, "abcdabcdabcdabcdabcdabcdabcdabcd", 5, "bcdef" -> 0x20, 37, "abcdabcdabcdabcdabcdabcdabcdabcd", "bcdef"
+//
+// f(bytes, bytes):
+//   0x40, 0xa0, 64, "abcdabcdabcdabcdabcdabcdabcdabcd", "abcdabcdabcdabcdabcdabcdabcdabcd", 5, "bcdef"
+//   ->
+//   0x20, 69, "abcdabcdabcdabcdabcdabcdabcdabcd", "abcdabcdabcdabcdabcdabcdabcdabcd", "bcdef"
+// f(bytes, bytes): 0x40, 0x80, 3, "abc", 3, "def" -> 0x20, 6, "abcdef"
+//
+// f(bytes, bytes):
+//  0x40, 0xa0, 34, "abcdabcdabcdabcdabcdabcdabcdabcd", "ab", 30, "cdabcdabcdabcdabcdabcdabcdabcd"
+//  ->
+//  0x20, 64, "abcdabcdabcdabcdabcdabcdabcdabcd", "abcdabcdabcdabcdabcdabcdabcdabcd"
+//
+// f(bytes, bytes):
+//  0x40, 0xa0, 34, "abcdabcdabcdabcdabcdabcdabcdabcd", "ab", 34, "cdabcdabcdabcdabcdabcdabcdabcdab", "cd"
+//  ->
+//  0x20, 68, "abcdabcdabcdabcdabcdabcdabcdabcd", "abcdabcdabcdabcdabcdabcdabcdabcd", "abcd"
+//
+// f(bytes, bytes):
+//   0x40, 0x80, 3, "abc", 30, "dabcdabcdabcdabcdabcdabcdabcda"
+//   ->
+//   0x20, 33, "abcdabcdabcdabcdabcdabcdabcdabcd", "a"

test/libsolidity/semanticTests/array/concat/bytes_concat_3_args.sol

@@ -0,0 +1,10 @@
+contract C {
+    function f(bytes memory a, bytes memory b, bytes memory c) public returns (bytes memory) {
+        return bytes.concat(a, b, c);
+    }
+}
+// ====
+// compileViaYul: also
+// ----
+// f(bytes, bytes, bytes): 0x60, 0xa0, 0xe0, 32, "abcdabcdabcdabcdabcdabcdabcdabcd", 5, "bcdef", 3, "abc" -> 0x20, 40, "abcdabcdabcdabcdabcdabcdabcdabcd", "bcdefabc"
+// f(bytes, bytes, bytes): 0x60, 0xa0, 0xe0, 3, "abc", 2, "de", 3, "fgh" -> 0x20, 8, "abcdefgh"

test/libsolidity/semanticTests/array/concat/bytes_concat_as_argument.sol

@@ -0,0 +1,18 @@
+contract C {
+    function f(bytes memory a, bytes memory b) public returns (bytes32) {
+        return keccak256(bytes.concat(a, b));
+    }
+
+    function h(bytes memory a) internal returns (uint256) {
+        return a.length;
+    }
+
+    function g(bytes memory a, bytes memory b) public returns (uint256) {
+        return h(bytes.concat(a, b));
+    }
+}
+// ====
+// compileViaYul: also
+// ----
+// f(bytes,bytes): 0x40, 0x80, 32, "abcdabcdabcdabcdabcdabcdabcdabcd", 5, "bcdef" -> 0x1106e19b6f06d1cce71c2d816754f83dfa5b95df958c5cbf12b7c472320c427c
+// g(bytes,bytes): 0x40, 0x80, 32, "abcdabcdabcdabcdabcdabcdabcdabcd", 5, "bcdef" -> 37

test/libsolidity/semanticTests/array/concat/bytes_concat_different_types.sol

@@ -0,0 +1,37 @@
+contract C {
+    bytes s = "bcdef";
+
+    function f(bytes memory a) public returns (bytes memory) {
+        return bytes.concat(a, "bcdef");
+    }
+    function g(bytes calldata a) public returns (bytes memory) {
+        return bytes.concat(a, "abcdefghabcdefghabcdefghabcdefghab");
+    }
+    function h(bytes calldata a) public returns (bytes memory) {
+        return bytes.concat(a, s);
+    }
+    function j(bytes calldata a) public returns (bytes memory) {
+        bytes storage ref = s;
+        return bytes.concat(a, ref, s);
+    }
+    function k(bytes calldata a, string memory b) public returns (bytes memory) {
+        return bytes.concat(a, bytes(b));
+    }
+    function slice(bytes calldata a) public returns (bytes memory) {
+        require(a.length > 2, "");
+        return bytes.concat(a[:2], a[2:]);
+    }
+    function strParam(string calldata a) public returns (bytes memory) {
+        return bytes.concat(bytes(a), "bcdef");
+    }
+}
+// ====
+// compileViaYul: also
+// ----
+// f(bytes): 0x20, 32, "abcdabcdabcdabcdabcdabcdabcdabcd" -> 0x20, 37, "abcdabcdabcdabcdabcdabcdabcdabcd", "bcdef"
+// g(bytes): 0x20, 32, "abcdabcdabcdabcdabcdabcdabcdabcd" -> 0x20, 66, "abcdabcdabcdabcdabcdabcdabcdabcd", "abcdefghabcdefghabcdefghabcdefgh", "ab"
+// h(bytes): 0x20, 32, "abcdabcdabcdabcdabcdabcdabcdabcd" -> 0x20, 37, "abcdabcdabcdabcdabcdabcdabcdabcd", "bcdef"
+// j(bytes): 0x20, 32, "abcdabcdabcdabcdabcdabcdabcdabcd" -> 0x20, 42, "abcdabcdabcdabcdabcdabcdabcdabcd", "bcdefbcdef"
+// k(bytes, string): 0x40, 0x80, 32, "abcdabcdabcdabcdabcdabcdabcdabcd", 5, "bcdef" -> 0x20, 37, "abcdabcdabcdabcdabcdabcdabcdabcd", "bcdef"
+// slice(bytes): 0x20, 4, "abcd" -> 0x20, 4, "abcd"
+// strParam(string): 0x20, 32, "abcdabcdabcdabcdabcdabcdabcdabcd" -> 0x20, 37, "abcdabcdabcdabcdabcdabcdabcdabcd", "bcdef"

test/libsolidity/semanticTests/array/concat/bytes_concat_empty.sol

@@ -0,0 +1,9 @@
+contract C {
+    function f() public returns (bytes memory) {
+        return bytes.concat();
+    }
+}
+// ====
+// compileViaYul: also
+// ----
+// f() -> 0x20, 0

test/libsolidity/semanticTests/array/concat/bytes_concat_nested.sol

@@ -0,0 +1,9 @@
+contract C {
+    function f(bytes memory a, bytes memory b, bytes memory c) public returns (bytes memory) {
+        return bytes.concat(bytes.concat(a, b), c);
+    }
+}
+// ====
+// compileViaYul: also
+// ----
+// f(bytes, bytes, bytes): 0x60, 0x60, 0x60, 2, "ab" -> 0x20, 6, "ababab"

test/libsolidity/syntaxTests/array/concat/bytes_concat_empty_invalid.sol

@@ -0,0 +1,7 @@
+contract C {
+    function g() public pure returns (bytes memory) {
+        return bytes.concat;
+    }
+}
+// ----
+// TypeError 6359: (82-94): Return argument type function () pure returns (bytes memory) is not implicitly convertible to expected type (type of first return variable) bytes memory.

test/libsolidity/syntaxTests/array/concat/bytes_concat_on_type_info.sol

@@ -0,0 +1,8 @@
+contract C {
+    function f() public {
+        bytes memory a;
+        bytes memory b = type(bytes).concat(a);
+    }
+}
+// ----
+// TypeError 4259: (93-98): Invalid type for argument in the function call. A contract type or an integer type is required, but type(bytes) provided.

test/libsolidity/syntaxTests/array/concat/bytes_concat_on_variable.sol

@@ -0,0 +1,8 @@
+contract C {
+    function f() public {
+        bytes memory a;
+        bytes memory b = a.concat();
+    }
+}
+// ----
+// TypeError 9582: (88-96): Member "concat" not found or not visible after argument-dependent lookup in bytes memory.

test/libsolidity/syntaxTests/array/concat/bytes_concat_wrong_type.sol

@@ -0,0 +1,14 @@
+contract C {
+    bytes s;
+    function f(bytes calldata c, string calldata c1) public {
+        bytes memory a;
+        bytes16 b;
+        uint8[] memory num;
+        bytes1[] memory m;
+        bytes memory d = bytes.concat(a, b, c, num, s, "abc", m, c1, bytes(c1));
+    }
+}
+// ----
+// TypeError 8015: (233-236): Invalid type for argument in the bytes.concat function call. bytes or fixed bytes type is required, but uint8[] provided.
+// TypeError 8015: (248-249): Invalid type for argument in the bytes.concat function call. bytes or fixed bytes type is required, but bytes1[] provided.
+// TypeError 8015: (251-253): Invalid type for argument in the bytes.concat function call. bytes or fixed bytes type is required, but string provided.