Fix type check for nested arrays in abi.encode/decode functions in ABIEncoderV1

- Without this fix, nested arrays are not detected as unsupported and compiler fails on an UnimplementedError.
- Now it's consistent with how structs are handled in ABIEncoderV1.

Changelog.md

@@ -18,6 +18,7 @@ Compiler Features:
 
 Bugfixes:
  * Code generator: Fix internal error on stripping dynamic types from return parameters on EVM versions without ``RETURNDATACOPY``.
+ * Type Checker: Add missing check against nested dynamic arrays in ABI encoding functions when ABIEncoderV2 is disabled.
  * Type Checker: Disallow ``virtual`` for modifiers in libraries.
  * Type Checker: Correct the warning for homonymous, but not shadowing declarations.
  * ViewPureChecker: Prevent visibility check on constructors.

libsolidity/ast/Types.cpp

@@ -404,10 +404,16 @@ TypePointer Type::fullEncodingType(bool _inLibraryCall, bool _encoderV2, bool) c
 		return encodingType;
 	TypePointer baseType = encodingType;
 	while (auto const* arrayType = dynamic_cast<ArrayType const*>(baseType))
+	{
 		baseType = arrayType->baseType();
-	if (dynamic_cast<StructType const*>(baseType))
+
-		if (!_encoderV2)
+		auto const* baseArrayType = dynamic_cast<ArrayType const*>(baseType);
+		if (!_encoderV2 && baseArrayType && baseArrayType->isDynamicallySized())
 			return nullptr;
+	}
+	if (!_encoderV2 && dynamic_cast<StructType const*>(baseType))
+		return nullptr;
+
 	return encodingType;
 }
 

test/libsolidity/syntaxTests/specialFunctions/abi_encodePacked_nested_dynamic_array.sol

@@ -0,0 +1,7 @@
+contract C {
+    function f() public pure {
+        abi.encodePacked([new uint[](5), new uint[](7)]);
+    }
+}
+// ----
+// TypeError 9578: (69-99): Type not supported in packed mode.

test/libsolidity/syntaxTests/specialFunctions/abi_encodePacked_nested_dynamic_array_v2.sol

@@ -0,0 +1,9 @@
+pragma experimental ABIEncoderV2;
+
+contract C {
+    function f() public pure {
+        abi.encodePacked([new uint[](5), new uint[](7)]);
+    }
+}
+// ----
+// TypeError 9578: (104-134): Type not supported in packed mode.

test/libsolidity/syntaxTests/specialFunctions/abi_encode_nested_dynamic_array.sol

@@ -0,0 +1,7 @@
+contract C {
+    function test() public pure {
+        abi.encode([new uint[](5), new uint[](7)]);
+    }
+}
+// ----
+// TypeError 2056: (66-96): This type cannot be encoded.

test/libsolidity/syntaxTests/specialFunctions/abi_encode_nested_dynamic_array_v2.sol

@@ -0,0 +1,9 @@
+pragma experimental ABIEncoderV2;
+
+contract C {
+    function f() public pure {
+        abi.encode([new uint[](5), new uint[](7)]);
+    }
+}
+// ----
+// Warning 6133: (87-129): Statement has no effect.

test/libsolidity/syntaxTests/specialFunctions/abidecode/abi_decode_nested_dynamic_array.sol

@@ -0,0 +1,7 @@
+contract C {
+    function f() public pure {
+        abi.decode("1234", (uint[][3]));
+    }
+}
+// ----
+// TypeError 9611: (72-81): Decoding type uint256[] memory[3] memory not supported.

test/libsolidity/syntaxTests/specialFunctions/abidecode/abi_decode_nested_dynamic_array_v2.sol

@@ -0,0 +1,9 @@
+pragma experimental ABIEncoderV2;
+
+contract C {
+    function f() public pure {
+        abi.decode("1234", (uint[][3]));
+    }
+}
+// ----
+// Warning 6133: (87-118): Statement has no effect.

test/libsolidity/syntaxTests/specialFunctions/abidecode/abi_decode_struct.sol

@@ -0,0 +1,11 @@
+struct S {
+    uint x;
+}
+
+contract C {
+    function f() public pure {
+        abi.decode("1234", (S));
+    }
+}
+// ----
+// TypeError 9611: (98-99): Decoding type struct S memory not supported.

test/libsolidity/syntaxTests/specialFunctions/abidecode/abi_decode_struct_v2.sol

@@ -0,0 +1,13 @@
+pragma experimental ABIEncoderV2;
+
+struct S {
+    uint x;
+}
+
+contract C {
+    function f() public pure {
+        abi.decode("1234", (S));
+    }
+}
+// ----
+// Warning 6133: (113-136): Statement has no effect.