Prevent encoding of weird types and support packed encoding of external function types.

test/libsolidity/SolidityEndToEndTest.cpp

@@ -2077,6 +2077,31 @@ BOOST_AUTO_TEST_CASE(packed_keccak256)
 	testContractAgainstCpp("a(bytes32)", f, u256(-1));
 }
 
+BOOST_AUTO_TEST_CASE(packed_keccak256_complex_types)
+{
+	char const* sourceCode = R"(
+		contract test {
+			uint120[3] x;
+			function f() view returns (bytes32 hash1, bytes32 hash2, bytes32 hash3) {
+				uint120[] memory y = new uint120[](3);
+				x[0] = y[0] = uint120(-2);
+				x[1] = y[1] = uint120(-3);
+				x[2] = y[2] = uint120(-4);
+				hash1 = keccak256(x);
+				hash2 = keccak256(y);
+				hash3 = keccak256(this.f);
+			}
+		}
+	)";
+	compileAndRun(sourceCode);
+	// Strangely, arrays are encoded with intra-element padding.
+	ABI_CHECK(callContractFunction("f()"), encodeArgs(
+		dev::keccak256(encodeArgs(u256("0xfffffffffffffffffffffffffffffe"), u256("0xfffffffffffffffffffffffffffffd"), u256("0xfffffffffffffffffffffffffffffc"))),
+		dev::keccak256(encodeArgs(u256("0xfffffffffffffffffffffffffffffe"), u256("0xfffffffffffffffffffffffffffffd"), u256("0xfffffffffffffffffffffffffffffc"))),
+		dev::keccak256(fromHex(m_contractAddress.hex() + "26121ff0"))
+	));
+}
+
 BOOST_AUTO_TEST_CASE(packed_sha256)
 {
 	char const* sourceCode = R"(

test/libsolidity/syntaxTests/specialFunctions/types_with_unspecified_encoding_internal_functions.sol

@@ -0,0 +1,11 @@
+contract C {
+    function f() public pure {
+        bytes32 h = keccak256(keccak256, f, this.f.gas, block.blockhash);
+        h;
+    }
+}
+// ----
+// TypeError: This type cannot be encoded.
+// TypeError: This type cannot be encoded.
+// TypeError: This type cannot be encoded.
+// TypeError: This type cannot be encoded.

test/libsolidity/syntaxTests/specialFunctions/types_with_unspecified_encoding_special_types.sol

@@ -0,0 +1,13 @@
+contract C {
+    function f() public pure {
+        bool a = address(this).call(address(this).delegatecall, super);
+        bool b = address(this).delegatecall(log0, tx, mulmod);
+        a; b;
+    }
+}
+// ----
+// TypeError: This type cannot be encoded.
+// TypeError: This type cannot be encoded.
+// TypeError: This type cannot be encoded.
+// TypeError: This type cannot be encoded.
+// TypeError: This type cannot be encoded.

test/libsolidity/syntaxTests/specialFunctions/types_with_unspecified_encoding_structs.sol

@@ -0,0 +1,13 @@
+contract C {
+    struct S { uint x; }
+    S s;
+    struct T { }
+    T t;
+    function f() public pure {
+        bytes32 a = sha256(s, t);
+        a;
+    }
+}
+// ----
+// TypeError: This type cannot be encoded.
+// TypeError: This type cannot be encoded.

test/libsolidity/syntaxTests/specialFunctions/types_with_unspecified_encoding_types.sol

@@ -0,0 +1,16 @@
+contract C {
+    struct S { uint x; }
+    S s;
+    struct T { }
+    T t;
+    enum A { X, Y }
+    function f() public pure {
+        bool a = address(this).delegatecall(S, A, A.X, T, uint, uint[]);
+    }
+}
+// ----
+// TypeError: This type cannot be encoded.
+// TypeError: This type cannot be encoded.
+// TypeError: This type cannot be encoded.
+// TypeError: This type cannot be encoded.
+// TypeError: This type cannot be encoded.

test/libsolidity/syntaxTests/specialFunctions/types_without_encoding_problems.sol

@@ -0,0 +1,9 @@
+contract C {
+    uint[3] sarr;
+    function f() view public {
+        uint[3] memory arr;
+        bytes32 h = keccak256(this.f, arr, sarr);
+        h;
+    }
+}
+// ----