trie: reduce allocs in recHash (#27770)

trie/encoding.go

@@ -51,9 +51,8 @@ func hexToCompact(hex []byte) []byte {
 	return buf
 }
 
-// hexToCompactInPlace places the compact key in input buffer, returning the length
-// needed for the representation
-func hexToCompactInPlace(hex []byte) int {
+// hexToCompactInPlace places the compact key in input buffer, returning the compacted key.
+func hexToCompactInPlace(hex []byte) []byte {
 	var (
 		hexLen    = len(hex) // length of the hex input
 		firstByte = byte(0)
@@ -77,7 +76,7 @@ func hexToCompactInPlace(hex []byte) int {
 		hex[bi] = hex[ni]<<4 | hex[ni+1]
 	}
 	hex[0] = firstByte
-	return binLen
+	return hex[:binLen]
 }
 
 func compactToHex(compact []byte) []byte {