Merge pull request #3743 from ethereum/popStorageArray

pop() for storage arrays

Changelog.md

@@ -1,5 +1,7 @@
 ### 0.5.0 (unreleased)
 
+Language Features:
+ * General: Support ``pop()`` for storage arrays.
 
 Breaking Changes:
  * Disallow conversions between bytesX and uintY of different size.

docs/types.rst

@@ -682,7 +682,7 @@ possible:
 It is planned to remove this restriction in the future but currently creates
 some complications because of how arrays are passed in the ABI.
 
-.. index:: ! array;length, length, push, !array;push
+.. index:: ! array;length, length, push, pop, !array;push, !array;pop
 
 Members
 ^^^^^^^
@@ -693,6 +693,8 @@ Members
     ``.length`` member. This does not happen automatically when attempting to access elements outside the current length. The size of memory arrays is fixed (but dynamic, i.e. it can depend on runtime parameters) once they are created.
 **push**:
      Dynamic storage arrays and ``bytes`` (not ``string``) have a member function called ``push`` that can be used to append an element at the end of the array. The function returns the new length.
+**pop**:
+     Dynamic storage arrays and ``bytes`` (not ``string``) have a member function called ``pop`` that can be used to remove an element from the end of the array.
 
 .. warning::
     It is not yet possible to use arrays of arrays in external functions.

libsolidity/ast/Types.cpp

@@ -1697,6 +1697,7 @@ MemberList::MemberMap ArrayType::nativeMembers(ContractDefinition const*) const
 	{
 		members.push_back({"length", make_shared<IntegerType>(256)});
 		if (isDynamicallySized() && location() == DataLocation::Storage)
+		{
 			members.push_back({"push", make_shared<FunctionType>(
 				TypePointers{baseType()},
 				TypePointers{make_shared<IntegerType>(256)},
@@ -1704,6 +1705,14 @@ MemberList::MemberMap ArrayType::nativeMembers(ContractDefinition const*) const
 				strings{string()},
 				isByteArray() ? FunctionType::Kind::ByteArrayPush : FunctionType::Kind::ArrayPush
 			)});
+			members.push_back({"pop", make_shared<FunctionType>(
+				TypePointers{},
+				TypePointers{},
+				strings{string()},
+				strings{string()},
+				FunctionType::Kind::ArrayPop
+			)});
+		}
 	}
 	return members;
 }

libsolidity/ast/Types.h

@@ -914,6 +914,7 @@ public:
 		AddMod, ///< ADDMOD
 		MulMod, ///< MULMOD
 		ArrayPush, ///< .push() to a dynamically sized array in storage
+		ArrayPop, ///< .pop() from a dynamically sized array in storage
 		ByteArrayPush, ///< .push() to a dynamically sized byte array in storage
 		ObjectCreation, ///< array creation using new
 		Assert, ///< assert()

libsolidity/codegen/ArrayUtils.cpp

@@ -823,6 +823,85 @@ void ArrayUtils::incrementDynamicArraySize(ArrayType const& _type) const
 		})", {"ref"});
 }
 
+void ArrayUtils::popStorageArrayElement(ArrayType const& _type) const
+{
+	solAssert(_type.location() == DataLocation::Storage, "");
+	solAssert(_type.isDynamicallySized(), "");
+	if (!_type.isByteArray() && _type.baseType()->storageBytes() < 32)
+		solAssert(_type.baseType()->isValueType(), "Invalid storage size for non-value type.");
+
+	if (_type.isByteArray())
+	{
+		m_context.appendInlineAssembly(R"({
+			let slot_value := sload(ref)
+			switch and(slot_value, 1)
+			case 0 {
+				// short byte array
+				let length := and(div(slot_value, 2), 0x1f)
+				if iszero(length) { invalid() }
+
+				// Zero-out the suffix including the least significant byte.
+				let mask := sub(exp(0x100, sub(33, length)), 1)
+				length := sub(length, 1)
+				slot_value := or(and(not(mask), slot_value), mul(length, 2))
+				sstore(ref, slot_value)
+			}
+			case 1 {
+				// long byte array
+				mstore(0, ref)
+				let length := div(slot_value, 2)
+				let slot := keccak256(0, 0x20)
+				switch length
+				case 32
+				{
+					let data := sload(slot)
+					sstore(slot, 0)
+					data := and(data, not(0xff))
+					sstore(ref, or(data, 62))
+				}
+				default
+				{
+					let offset_inside_slot := and(sub(length, 1), 0x1f)
+					slot := add(slot, div(sub(length, 1), 32))
+					let data := sload(slot)
+
+					// Zero-out the suffix of the byte array by masking it.
+					// ((1<<(8 * (32 - offset))) - 1)
+					let mask := sub(exp(0x100, sub(32, offset_inside_slot)), 1)
+					data := and(not(mask), data)
+					sstore(slot, data)
+
+					// Reduce the length by 1
+					slot_value := sub(slot_value, 2)
+					sstore(ref, slot_value)
+				}
+			}
+		})", {"ref"});
+		m_context << Instruction::POP;
+	}
+	else
+	{
+		// stack: ArrayReference
+		retrieveLength(_type);
+		// stack: ArrayReference oldLength
+		m_context << Instruction::DUP1;
+		// stack: ArrayReference oldLength oldLength
+		m_context << Instruction::ISZERO;
+		m_context.appendConditionalInvalid();
+
+		// Stack: ArrayReference oldLength
+		m_context << u256(1) << Instruction::SWAP1 << Instruction::SUB;
+		// Stack ArrayReference newLength
+		m_context << Instruction::DUP2 << Instruction::DUP2;
+		// Stack ArrayReference newLength ArrayReference newLength;
+		accessIndex(_type, false);
+		// Stack: ArrayReference newLength storage_slot byte_offset
+		StorageItem(m_context, *_type.baseType()).setToZero(SourceLocation(), true);
+		// Stack: ArrayReference newLength
+		m_context << Instruction::SWAP1 << Instruction::SSTORE;
+	}
+}
+
 void ArrayUtils::clearStorageLoop(TypePointer const& _type) const
 {
 	m_context.callLowLevelFunction(

libsolidity/codegen/ArrayUtils.h

@@ -73,6 +73,11 @@ public:
 	/// Stack pre: reference (excludes byte offset)
 	/// Stack post: new_length
 	void incrementDynamicArraySize(ArrayType const& _type) const;
+	/// Decrements the size of a dynamic array by one if length is nonzero. Causes an invalid instruction otherwise.
+	/// Clears the removed data element. In case of a byte array, this might move the data.
+	/// Stack pre: reference
+	/// Stack post:
+	void popStorageArrayElement(ArrayType const& _type) const;
 	/// Appends a loop that clears a sequence of storage slots of the given type (excluding end).
 	/// Stack pre: end_ref start_ref
 	/// Stack post: end_ref

libsolidity/codegen/ExpressionCompiler.cpp

@@ -866,6 +866,19 @@ bool ExpressionCompiler::visit(FunctionCall const& _functionCall)
 				StorageByteArrayElement(m_context).storeValue(*type, _functionCall.location(), true);
 			break;
 		}
+		case FunctionType::Kind::ArrayPop:
+		{
+			_functionCall.expression().accept(*this);
+			solAssert(function.parameterTypes().empty(), "");
+
+			ArrayType const& arrayType = dynamic_cast<ArrayType const&>(
+				*dynamic_cast<MemberAccess const&>(_functionCall.expression()).expression().annotation().type
+			);
+			solAssert(arrayType.dataStoredIn(DataLocation::Storage), "");
+
+			ArrayUtils(m_context).popStorageArrayElement(arrayType);
+			break;
+		}
 		case FunctionType::Kind::ObjectCreation:
 		{
 			ArrayType const& arrayType = dynamic_cast<ArrayType const&>(*_functionCall.annotation().type);
@@ -1345,11 +1358,13 @@ bool ExpressionCompiler::visit(MemberAccess const& _memberAccess)
 					break;
 				}
 		}
-		else if (member == "push")
+		else if (member == "push" || member == "pop")
 		{
 			solAssert(
-				type.isDynamicallySized() && type.location() == DataLocation::Storage,
-				"Tried to use .push() on a non-dynamically sized array"
+				type.isDynamicallySized() &&
+				type.location() == DataLocation::Storage &&
+				type.category() == Type::Category::Array,
+				"Tried to use ." + member + "() on a non-dynamically sized array"
 			);
 		}
 		else

test/libsolidity/SolidityEndToEndTest.cpp

@@ -5111,6 +5111,292 @@ BOOST_AUTO_TEST_CASE(byte_array_push_transition)
 	ABI_CHECK(callContractFunction("test()"), encodeArgs(0));
 }
 
+BOOST_AUTO_TEST_CASE(array_pop)
+{
+	char const* sourceCode = R"(
+		contract c {
+			uint[] data;
+			function test() public returns (uint x, uint l) {
+				data.push(7);
+				x = data.push(3);
+				data.pop();
+				x = data.length;
+				data.pop();
+				l = data.length;
+			}
+		}
+	)";
+	compileAndRun(sourceCode);
+	ABI_CHECK(callContractFunction("test()"), encodeArgs(1, 0));
+}
+
+BOOST_AUTO_TEST_CASE(array_pop_uint16_transition)
+{
+	char const* sourceCode = R"(
+		contract c {
+			uint16[] data;
+			function test() public returns (uint16 x, uint16 y, uint16 z) {
+				for (uint i = 1; i <= 48; i++)
+					data.push(uint16(i));
+				for (uint j = 1; j <= 10; j++)
+					data.pop();
+				x = data[data.length - 1];
+				for (uint k = 1; k <= 10; k++)
+					data.pop();
+				y = data[data.length - 1];
+				for (uint l = 1; l <= 10; l++)
+					data.pop();
+				z = data[data.length - 1];
+				for (uint m = 1; m <= 18; m++)
+					data.pop();
+			}
+		}
+	)";
+	compileAndRun(sourceCode);
+	ABI_CHECK(callContractFunction("test()"), encodeArgs(38, 28, 18));
+	BOOST_CHECK(storageEmpty(m_contractAddress));
+}
+
+BOOST_AUTO_TEST_CASE(array_pop_uint24_transition)
+{
+	char const* sourceCode = R"(
+		contract c {
+			uint256 a;
+			uint256 b;
+			uint256 c;
+			uint24[] data;
+			function test() public returns (uint24 x, uint24 y) {
+				for (uint i = 1; i <= 30; i++)
+					data.push(uint24(i));
+				for (uint j = 1; j <= 10; j++)
+					data.pop();
+				x = data[data.length - 1];
+				for (uint k = 1; k <= 10; k++)
+					data.pop();
+				y = data[data.length - 1];
+				for (uint l = 1; l <= 10; l++)
+					data.pop();
+			}
+		}
+	)";
+	compileAndRun(sourceCode);
+	ABI_CHECK(callContractFunction("test()"), encodeArgs(20, 10));
+	BOOST_CHECK(storageEmpty(m_contractAddress));
+}
+
+BOOST_AUTO_TEST_CASE(array_pop_array_transition)
+{
+	char const* sourceCode = R"(
+		contract c {
+			uint256 a;
+			uint256 b;
+			uint256 c;
+			uint16[] inner = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16];
+			uint16[][] data;
+			function test() public returns (uint x, uint y, uint z) {
+				for (uint i = 1; i <= 48; i++)
+					data.push(inner);
+				for (uint j = 1; j <= 10; j++)
+					data.pop();
+				x = data[data.length - 1][0];
+				for (uint k = 1; k <= 10; k++)
+					data.pop();
+				y = data[data.length - 1][1];
+				for (uint l = 1; l <= 10; l++)
+					data.pop();
+				z = data[data.length - 1][2];
+				for (uint m = 1; m <= 18; m++)
+					data.pop();
+				delete inner;
+			}
+		}
+	)";
+	compileAndRun(sourceCode);
+	ABI_CHECK(callContractFunction("test()"), encodeArgs(1, 2, 3));
+	BOOST_CHECK(storageEmpty(m_contractAddress));
+}
+
+BOOST_AUTO_TEST_CASE(array_pop_empty_exception)
+{
+	char const* sourceCode = R"(
+		contract c {
+			uint[] data;
+			function test() public returns (bool) {
+				data.pop();
+				return true;
+			}
+		}
+	)";
+	compileAndRun(sourceCode);
+	ABI_CHECK(callContractFunction("test()"), encodeArgs());
+}
+
+BOOST_AUTO_TEST_CASE(array_pop_storage_empty)
+{
+	char const* sourceCode = R"(
+		contract c {
+			uint[] data;
+			function test() public {
+				data.push(7);
+				data.pop();
+			}
+		}
+	)";
+	compileAndRun(sourceCode);
+	ABI_CHECK(callContractFunction("test()"), encodeArgs());
+	BOOST_CHECK(storageEmpty(m_contractAddress));
+}
+
+BOOST_AUTO_TEST_CASE(byte_array_pop)
+{
+	char const* sourceCode = R"(
+		contract c {
+			bytes data;
+			function test() public returns (uint x, uint y, uint l) {
+				data.push(7);
+				x = data.push(3);
+				data.pop();
+				data.pop();
+				y = data.push(2);
+				l = data.length;
+			}
+		}
+	)";
+	compileAndRun(sourceCode);
+	ABI_CHECK(callContractFunction("test()"), encodeArgs(2, 1, 1));
+}
+
+BOOST_AUTO_TEST_CASE(byte_array_pop_empty_exception)
+{
+	char const* sourceCode = R"(
+		contract c {
+			uint256 a;
+			uint256 b;
+			uint256 c;
+			bytes data;
+			function test() public returns (bool) {
+				data.pop();
+				return true;
+			}
+		}
+	)";
+	compileAndRun(sourceCode);
+	ABI_CHECK(callContractFunction("test()"), encodeArgs());
+}
+
+BOOST_AUTO_TEST_CASE(byte_array_pop_storage_empty)
+{
+	char const* sourceCode = R"(
+		contract c {
+			bytes data;
+			function test() public {
+				data.push(7);
+				data.push(5);
+				data.push(3);
+				data.pop();
+				data.pop();
+				data.pop();
+			}
+		}
+	)";
+	compileAndRun(sourceCode);
+	ABI_CHECK(callContractFunction("test()"), encodeArgs());
+	BOOST_CHECK(storageEmpty(m_contractAddress));
+}
+
+BOOST_AUTO_TEST_CASE(byte_array_pop_long_storage_empty)
+{
+	char const* sourceCode = R"(
+		contract c {
+			uint256 a;
+			uint256 b;
+			uint256 c;
+			bytes data;
+			function test() public returns (bool) {
+				for (uint8 i = 0; i <= 40; i++)
+					data.push(byte(i+1));
+				for (int8 j = 40; j >= 0; j--) {
+					require(data[uint8(j)] == byte(j+1));
+					require(data.length == uint8(j+1));
+					data.pop();
+				}
+				return true;
+			}
+		}
+	)";
+	compileAndRun(sourceCode);
+	ABI_CHECK(callContractFunction("test()"), encodeArgs(true));
+	BOOST_CHECK(storageEmpty(m_contractAddress));
+}
+
+BOOST_AUTO_TEST_CASE(byte_array_pop_long_storage_empty_garbage_ref)
+{
+	char const* sourceCode = R"(
+		contract c {
+			uint256 a;
+			uint256 b;
+			bytes data;
+			function test() public {
+				for (uint8 i = 0; i <= 40; i++)
+					data.push(3);
+				for (uint8 j = 0; j <= 40; j++) {
+					assembly {
+						mstore(0, "garbage")
+					}
+					data.pop();
+				}
+			}
+		}
+	)";
+	compileAndRun(sourceCode);
+	ABI_CHECK(callContractFunction("test()"), encodeArgs());
+	BOOST_CHECK(storageEmpty(m_contractAddress));
+}
+
+BOOST_AUTO_TEST_CASE(byte_array_pop_masking_long)
+{
+	char const* sourceCode = R"(
+		contract c {
+			bytes data;
+			function test() public returns (bytes) {
+				for (uint i = 0; i < 34; i++)
+					data.push(3);
+				data.pop();
+				return data;
+			}
+		}
+	)";
+	compileAndRun(sourceCode);
+	ABI_CHECK(callContractFunction("test()"), encodeArgs(
+		u256(0x20),
+		u256(33),
+		asString(fromHex("0303030303030303030303030303030303030303030303030303030303030303")),
+		asString(fromHex("03"))
+	));
+}
+
+BOOST_AUTO_TEST_CASE(byte_array_pop_copy_long)
+{
+	char const* sourceCode = R"(
+		contract c {
+			bytes data;
+			function test() public returns (bytes) {
+				for (uint i = 0; i < 33; i++)
+					data.push(3);
+				for (uint j = 0; j < 4; j++)
+					data.pop();
+				return data;
+			}
+		}
+	)";
+	compileAndRun(sourceCode);
+	ABI_CHECK(callContractFunction("test()"), encodeArgs(
+		u256(0x20),
+		u256(29),
+		asString(fromHex("0303030303030303030303030303030303030303030303030303030303"))
+	));
+}
+
 BOOST_AUTO_TEST_CASE(external_array_args)
 {
 	char const* sourceCode = R"(

test/libsolidity/syntaxTests/array/array_pop.sol

@@ -0,0 +1,7 @@
+contract C {
+    uint[] data;
+    function test() public {
+      data.pop();
+    }
+}
+// ----

test/libsolidity/syntaxTests/array/array_pop_arg.sol

@@ -0,0 +1,8 @@
+contract C {
+    uint[] data;
+    function test() public {
+      data.pop(5);
+    }
+}
+// ----
+// TypeError: (65-76): Wrong argument count for function call: 1 arguments given but expected 0.

test/libsolidity/syntaxTests/array/bytes_pop.sol

@@ -0,0 +1,7 @@
+contract C {
+    bytes data;
+    function test() public {
+      data.pop();
+    }
+}
+// ----

test/libsolidity/syntaxTests/array/dynamic_memory_array_pop.sol

@@ -0,0 +1,8 @@
+contract C {
+    function test() public {
+      uint[] memory data;
+      data.pop();
+    }
+}
+// ----
+// TypeError: (74-82): Member "pop" is not available in uint256[] memory outside of storage.

test/libsolidity/syntaxTests/array/no_array_pop.sol

@@ -0,0 +1,8 @@
+contract C {
+    uint data;
+    function test() public {
+      data.pop();
+    }
+}
+// ----
+// TypeError: (63-71): Member "pop" not found or not visible after argument-dependent lookup in uint256

test/libsolidity/syntaxTests/array/static_storage_array_pop.sol

@@ -0,0 +1,8 @@
+contract C {
+    uint[3] data;
+    function test() public {
+      data.pop();
+    }
+}
+// ----
+// TypeError: (66-74): Member "pop" not found or not visible after argument-dependent lookup in uint256[3] storage ref

test/libsolidity/syntaxTests/array/string_pop.sol

@@ -0,0 +1,8 @@
+contract C {
+    string data;
+    function test() public {
+      data.pop();
+    }
+}
+// ----
+// TypeError: (65-73): Member "pop" not found or not visible after argument-dependent lookup in string storage ref