Lazy hashing for SignedBeaconBlock in sync (#2916)

## Proposed Changes

Allocate less memory in sync by hashing the `SignedBeaconBlock`s in a batch directly, rather than going via SSZ bytes.

Credit to @paulhauner for finding this source of temporary allocations.

Cargo.lock

@@ -1701,6 +1701,7 @@ name = "eth2_ssz_types"
 version = "0.2.2"
 dependencies = [
  "arbitrary",
+ "derivative",
  "eth2_serde_utils",
  "eth2_ssz",
  "serde",

beacon_node/network/src/sync/range_sync/batch.rs

@@ -1,7 +1,6 @@
 use crate::sync::RequestId;
 use lighthouse_network::rpc::methods::BlocksByRangeRequest;
 use lighthouse_network::PeerId;
-use ssz::Encode;
 use std::collections::HashSet;
 use std::hash::{Hash, Hasher};
 use std::ops::Sub;
@@ -390,7 +389,7 @@ impl Attempt {
     #[allow(clippy::ptr_arg)]
     fn new<T: EthSpec>(peer_id: PeerId, blocks: &Vec<SignedBeaconBlock<T>>) -> Self {
         let mut hasher = std::collections::hash_map::DefaultHasher::new();
-        blocks.as_ssz_bytes().hash(&mut hasher);
+        blocks.hash(&mut hasher);
         let hash = hasher.finish();
         Attempt { peer_id, hash }
     }

consensus/ssz_types/Cargo.toml

@@ -17,6 +17,7 @@ eth2_serde_utils = "0.1.1"
 eth2_ssz = "0.4.1"
 typenum = "1.12.0"
 arbitrary = { version = "1.0", features = ["derive"], optional = true }
+derivative = "2.1.1"
 
 [dev-dependencies]
 serde_json = "1.0.58"

consensus/ssz_types/src/bitfield.rs

@@ -1,6 +1,7 @@
 use crate::tree_hash::bitfield_bytes_tree_hash_root;
 use crate::Error;
 use core::marker::PhantomData;
+use derivative::Derivative;
 use eth2_serde_utils::hex::{encode as hex_encode, PrefixedHexVisitor};
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
@@ -87,7 +88,8 @@ pub type BitVector<N> = Bitfield<Fixed<N>>;
 /// The internal representation of the bitfield is the same as that required by SSZ. The lowest
 /// byte (by `Vec` index) stores the lowest bit-indices and the right-most bit stores the lowest
 /// bit-index. E.g., `vec![0b0000_0001, 0b0000_0010]` has bits `0, 9` set.
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, Derivative)]
+#[derivative(PartialEq, Hash(bound = ""))]
 pub struct Bitfield<T> {
     bytes: Vec<u8>,
     len: usize,

consensus/ssz_types/src/fixed_vector.rs

@@ -1,5 +1,6 @@
 use crate::tree_hash::vec_tree_hash_root;
 use crate::Error;
+use derivative::Derivative;
 use serde_derive::{Deserialize, Serialize};
 use std::marker::PhantomData;
 use std::ops::{Deref, DerefMut, Index, IndexMut};
@@ -44,7 +45,8 @@ pub use typenum;
 /// let long: FixedVector<_, typenum::U5> = FixedVector::from(base);
 /// assert_eq!(&long[..], &[1, 2, 3, 4, 0]);
 /// ```
-#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, Derivative)]
+#[derivative(PartialEq, Hash(bound = "T: std::hash::Hash"))]
 #[serde(transparent)]
 pub struct FixedVector<T, N> {
     vec: Vec<T>,

consensus/ssz_types/src/variable_list.rs

@@ -1,5 +1,6 @@
 use crate::tree_hash::vec_tree_hash_root;
 use crate::Error;
+use derivative::Derivative;
 use serde_derive::{Deserialize, Serialize};
 use std::marker::PhantomData;
 use std::ops::{Deref, DerefMut, Index, IndexMut};
@@ -46,7 +47,8 @@ pub use typenum;
 /// // Push a value to if it _does_ exceed the maximum.
 /// assert!(long.push(6).is_err());
 /// ```
-#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, Derivative)]
+#[derivative(PartialEq, Eq, Hash(bound = "T: std::hash::Hash"))]
 #[serde(transparent)]
 pub struct VariableList<T, N> {
     vec: Vec<T>,

consensus/types/src/attestation.rs

@@ -1,3 +1,4 @@
+use derivative::Derivative;
 use safe_arith::ArithError;
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
@@ -23,7 +24,10 @@ pub enum Error {
 ///
 /// Spec v0.12.1
 #[cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))]
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(
+    Debug, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, Derivative,
+)]
+#[derivative(PartialEq, Hash(bound = "T: EthSpec"))]
 #[serde(bound = "T: EthSpec")]
 pub struct Attestation<T: EthSpec> {
     pub aggregation_bits: BitList<T::MaxValidatorsPerCommittee>,

consensus/types/src/beacon_block.rs

@@ -5,6 +5,7 @@ use crate::beacon_block_body::{
 use crate::test_utils::TestRandom;
 use crate::*;
 use bls::Signature;
+use derivative::Derivative;
 use serde_derive::{Deserialize, Serialize};
 use ssz::{Decode, DecodeError};
 use ssz_derive::{Decode, Encode};
@@ -19,15 +20,16 @@ use tree_hash_derive::TreeHash;
     variant_attributes(
         derive(
             Debug,
-            PartialEq,
             Clone,
             Serialize,
             Deserialize,
             Encode,
             Decode,
             TreeHash,
-            TestRandom
+            TestRandom,
+            Derivative,
         ),
+        derivative(PartialEq, Hash(bound = "T: EthSpec")),
         serde(bound = "T: EthSpec", deny_unknown_fields),
         cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary)),
     ),
@@ -36,7 +38,8 @@ use tree_hash_derive::TreeHash;
         tree_hash(enum_behaviour = "transparent")
     )
 )]
-#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, TreeHash)]
+#[derive(Debug, Clone, Serialize, Deserialize, Encode, TreeHash, Derivative)]
+#[derivative(PartialEq, Hash(bound = "T: EthSpec"))]
 #[serde(untagged)]
 #[serde(bound = "T: EthSpec")]
 #[cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))]

consensus/types/src/beacon_block_body.rs

@@ -1,5 +1,6 @@
 use crate::test_utils::TestRandom;
 use crate::*;
+use derivative::Derivative;
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use ssz_types::VariableList;
@@ -15,22 +16,24 @@ use tree_hash_derive::TreeHash;
     variant_attributes(
         derive(
             Debug,
-            PartialEq,
             Clone,
             Serialize,
             Deserialize,
             Encode,
             Decode,
             TreeHash,
-            TestRandom
+            TestRandom,
+            Derivative,
         ),
+        derivative(PartialEq, Hash(bound = "T: EthSpec")),
         serde(bound = "T: EthSpec", deny_unknown_fields),
         cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))
     ),
     cast_error(ty = "Error", expr = "Error::IncorrectStateVariant"),
     partial_getter_error(ty = "Error", expr = "Error::IncorrectStateVariant")
 )]
-#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, Derivative)]
+#[derivative(PartialEq, Hash(bound = "T: EthSpec"))]
 #[serde(untagged)]
 #[serde(bound = "T: EthSpec")]
 #[cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))]

consensus/types/src/deposit.rs

@@ -12,7 +12,9 @@ pub const DEPOSIT_TREE_DEPTH: usize = 32;
 ///
 /// Spec v0.12.1
 #[cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))]
-#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(
+    Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom,
+)]
 pub struct Deposit {
     pub proof: FixedVector<Hash256, U33>,
     pub data: DepositData,

consensus/types/src/deposit_data.rs

@@ -11,7 +11,9 @@ use tree_hash_derive::TreeHash;
 ///
 /// Spec v0.12.1
 #[cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))]
-#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(
+    Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom,
+)]
 pub struct DepositData {
     pub pubkey: PublicKeyBytes,
     pub withdrawal_credentials: Hash256,

consensus/types/src/execution_payload.rs

@@ -1,4 +1,5 @@
 use crate::{test_utils::TestRandom, *};
+use derivative::Derivative;
 use serde_derive::{Deserialize, Serialize};
 use ssz::Encode;
 use ssz_derive::{Decode, Encode};
@@ -9,8 +10,9 @@ pub type Transaction<T> = VariableList<u8, T>;
 
 #[cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))]
 #[derive(
-    Default, Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom,
+    Default, Debug, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, Derivative,
 )]
+#[derivative(PartialEq, Hash(bound = "T: EthSpec"))]
 #[serde(bound = "T: EthSpec")]
 pub struct ExecutionPayload<T: EthSpec> {
     pub parent_hash: Hash256,

consensus/types/src/graffiti.rs

@@ -12,7 +12,7 @@ use tree_hash::TreeHash;
 pub const GRAFFITI_BYTES_LEN: usize = 32;
 
 /// The 32-byte `graffiti` field on a beacon block.
-#[derive(Default, Debug, PartialEq, Clone, Copy, Serialize, Deserialize)]
+#[derive(Default, Debug, PartialEq, Hash, Clone, Copy, Serialize, Deserialize)]
 #[serde(transparent)]
 #[cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))]
 pub struct Graffiti(#[serde(with = "serde_graffiti")] pub [u8; GRAFFITI_BYTES_LEN]);

consensus/types/src/signed_beacon_block.rs

@@ -1,5 +1,6 @@
 use crate::*;
 use bls::Signature;
+use derivative::Derivative;
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use std::fmt;
@@ -41,19 +42,21 @@ impl From<SignedBeaconBlockHash> for Hash256 {
     variant_attributes(
         derive(
             Debug,
-            PartialEq,
             Clone,
             Serialize,
             Deserialize,
             Encode,
             Decode,
-            TreeHash
+            TreeHash,
+            Derivative,
         ),
+        derivative(PartialEq, Hash(bound = "E: EthSpec")),
         cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary)),
         serde(bound = "E: EthSpec")
     )
 )]
-#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, TreeHash)]
+#[derive(Debug, Clone, Serialize, Deserialize, Encode, TreeHash, Derivative)]
+#[derivative(PartialEq, Hash(bound = "E: EthSpec"))]
 #[serde(untagged)]
 #[serde(bound = "E: EthSpec")]
 #[cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))]

consensus/types/src/signed_beacon_block_header.rs

@@ -2,11 +2,8 @@ use crate::{
     test_utils::TestRandom, BeaconBlockHeader, ChainSpec, Domain, EthSpec, Fork, Hash256,
     PublicKey, Signature, SignedRoot,
 };
-use derivative::Derivative;
 use serde_derive::{Deserialize, Serialize};
-use ssz::Encode;
 use ssz_derive::{Decode, Encode};
-use std::hash::{Hash, Hasher};
 use test_random_derive::TestRandom;
 use tree_hash_derive::TreeHash;
 
@@ -15,26 +12,13 @@ use tree_hash_derive::TreeHash;
 /// Spec v0.12.1
 #[cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))]
 #[derive(
-    Derivative, Debug, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom,
+    Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom,
 )]
-#[derivative(PartialEq, Eq)]
 pub struct SignedBeaconBlockHeader {
     pub message: BeaconBlockHeader,
     pub signature: Signature,
 }
 
-/// Implementation of non-crypto-secure `Hash`, for use with `HashMap` and `HashSet`.
-///
-/// Guarantees `header1 == header2 -> hash(header1) == hash(header2)`.
-///
-/// Used in the slasher.
-impl Hash for SignedBeaconBlockHeader {
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        self.message.hash(state);
-        self.signature.as_ssz_bytes().hash(state);
-    }
-}
-
 impl SignedBeaconBlockHeader {
     /// Verify that this block header was signed by `pubkey`.
     pub fn verify_signature<E: EthSpec>(

consensus/types/src/signed_voluntary_exit.rs

@@ -10,7 +10,9 @@ use tree_hash_derive::TreeHash;
 ///
 /// Spec v0.12.1
 #[cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))]
-#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(
+    Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom,
+)]
 pub struct SignedVoluntaryExit {
     pub message: VoluntaryExit,
     pub signature: Signature,

consensus/types/src/sync_aggregate.rs

@@ -1,6 +1,7 @@
 use crate::consts::altair::SYNC_COMMITTEE_SUBNET_COUNT;
 use crate::test_utils::TestRandom;
 use crate::{AggregateSignature, BitVector, EthSpec, SyncCommitteeContribution};
+use derivative::Derivative;
 use safe_arith::{ArithError, SafeArith};
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
@@ -20,7 +21,10 @@ impl From<ArithError> for Error {
 }
 
 #[cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))]
-#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(
+    Debug, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, Derivative,
+)]
+#[derivative(PartialEq, Hash(bound = "T: EthSpec"))]
 #[serde(bound = "T: EthSpec")]
 pub struct SyncAggregate<T: EthSpec> {
     pub sync_committee_bits: BitVector<T::SyncCommitteeSize>,

consensus/types/src/voluntary_exit.rs

@@ -12,7 +12,9 @@ use tree_hash_derive::TreeHash;
 ///
 /// Spec v0.12.1
 #[cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))]
-#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(
+    Debug, PartialEq, Hash, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom,
+)]
 pub struct VoluntaryExit {
     /// Earliest epoch when voluntary exit can be processed.
     pub epoch: Epoch,

crypto/bls/src/generic_aggregate_signature.rs

@@ -9,6 +9,7 @@ use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use ssz::{Decode, Encode};
 use std::fmt;
+use std::hash::{Hash, Hasher};
 use std::marker::PhantomData;
 use tree_hash::TreeHash;
 
@@ -264,6 +265,18 @@ where
     impl_tree_hash!(SIGNATURE_BYTES_LEN);
 }
 
+/// Hashes the `self.serialize()` bytes.
+#[allow(clippy::derive_hash_xor_eq)]
+impl<Pub, AggPub, Sig, AggSig> Hash for GenericAggregateSignature<Pub, AggPub, Sig, AggSig>
+where
+    Sig: TSignature<Pub>,
+    AggSig: TAggregateSignature<Pub, AggPub, Sig>,
+{
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.serialize().hash(state);
+    }
+}
+
 impl<Pub, AggPub, Sig, AggSig> fmt::Display for GenericAggregateSignature<Pub, AggPub, Sig, AggSig>
 where
     Sig: TSignature<Pub>,

crypto/bls/src/generic_signature.rs

@@ -7,6 +7,7 @@ use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use ssz::{Decode, Encode};
 use std::fmt;
+use std::hash::{Hash, Hasher};
 use std::marker::PhantomData;
 use tree_hash::TreeHash;
 
@@ -145,6 +146,13 @@ impl<PublicKey, T: TSignature<PublicKey>> TreeHash for GenericSignature<PublicKe
     impl_tree_hash!(SIGNATURE_BYTES_LEN);
 }
 
+/// Hashes the `self.serialize()` bytes.
+impl<PublicKey, T: TSignature<PublicKey>> Hash for GenericSignature<PublicKey, T> {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.serialize().hash(state);
+    }
+}
+
 impl<PublicKey, T: TSignature<PublicKey>> fmt::Display for GenericSignature<PublicKey, T> {
     impl_display!();
 }

crypto/bls/src/generic_signature_bytes.rs

@@ -9,6 +9,7 @@ use serde::ser::{Serialize, Serializer};
 use ssz::{Decode, Encode};
 use std::convert::TryInto;
 use std::fmt;
+use std::hash::{Hash, Hasher};
 use std::marker::PhantomData;
 use tree_hash::TreeHash;
 
@@ -84,6 +85,12 @@ impl<Pub, Sig> PartialEq for GenericSignatureBytes<Pub, Sig> {
     }
 }
 
+impl<Pub, Sig> Hash for GenericSignatureBytes<Pub, Sig> {
+    fn hash<H: Hasher>(&self, hasher: &mut H) {
+        self.bytes.hash(hasher);
+    }
+}
+
 /// Serializes the `GenericSignature` in compressed form, storing the bytes in the newly created `Self`.
 impl<Pub, Sig> From<GenericSignature<Pub, Sig>> for GenericSignatureBytes<Pub, Sig>
 where

crypto/bls/src/impls/fake_crypto.rs

@@ -113,6 +113,14 @@ impl PartialEq for Signature {
     }
 }
 
+impl Eq for Signature {}
+
+impl std::hash::Hash for Signature {
+    fn hash<H: std::hash::Hasher>(&self, hasher: &mut H) {
+        self.0.hash(hasher);
+    }
+}
+
 #[derive(Clone)]
 pub struct AggregateSignature([u8; SIGNATURE_BYTES_LEN]);