Add vector type to tree hashing

eth2/types/src/beacon_state.rs

@@ -60,7 +60,7 @@ pub struct BeaconState {
     pub validator_registry_update_epoch: Epoch,
 
     // Randomness and committees
-    pub latest_randao_mixes: Vec<Hash256>,
+    pub latest_randao_mixes: TreeHashVector<Hash256>,
     pub previous_shuffling_start_shard: u64,
     pub current_shuffling_start_shard: u64,
     pub previous_shuffling_epoch: Epoch,
@@ -80,11 +80,11 @@ pub struct BeaconState {
     pub finalized_root: Hash256,
 
     // Recent state
-    pub latest_crosslinks: Vec<Crosslink>,
+    pub latest_crosslinks: TreeHashVector<Crosslink>,
-    latest_block_roots: Vec<Hash256>,
+    latest_block_roots: TreeHashVector<Hash256>,
-    latest_state_roots: Vec<Hash256>,
+    latest_state_roots: TreeHashVector<Hash256>,
-    latest_active_index_roots: Vec<Hash256>,
+    latest_active_index_roots: TreeHashVector<Hash256>,
-    latest_slashed_balances: Vec<u64>,
+    latest_slashed_balances: TreeHashVector<u64>,
     pub latest_block_header: BeaconBlockHeader,
     pub historical_roots: Vec<Hash256>,
 
@@ -139,7 +139,8 @@ impl BeaconState {
             validator_registry_update_epoch: spec.genesis_epoch,
 
             // Randomness and committees
-            latest_randao_mixes: vec![spec.zero_hash; spec.latest_randao_mixes_length as usize],
+            latest_randao_mixes: vec![spec.zero_hash; spec.latest_randao_mixes_length as usize]
+                .into(),
             previous_shuffling_start_shard: spec.genesis_start_shard,
             current_shuffling_start_shard: spec.genesis_start_shard,
             previous_shuffling_epoch: spec.genesis_epoch,
@@ -159,11 +160,12 @@ impl BeaconState {
             finalized_root: spec.zero_hash,
 
             // Recent state
-            latest_crosslinks: vec![initial_crosslink; spec.shard_count as usize],
+            latest_crosslinks: vec![initial_crosslink; spec.shard_count as usize].into(),
-            latest_block_roots: vec![spec.zero_hash; spec.slots_per_historical_root],
+            latest_block_roots: vec![spec.zero_hash; spec.slots_per_historical_root].into(),
-            latest_state_roots: vec![spec.zero_hash; spec.slots_per_historical_root],
+            latest_state_roots: vec![spec.zero_hash; spec.slots_per_historical_root].into(),
-            latest_active_index_roots: vec![spec.zero_hash; spec.latest_active_index_roots_length],
+            latest_active_index_roots: vec![spec.zero_hash; spec.latest_active_index_roots_length]
-            latest_slashed_balances: vec![0; spec.latest_slashed_exit_length],
+                .into(),
+            latest_slashed_balances: vec![0; spec.latest_slashed_exit_length].into(),
             latest_block_header: BeaconBlock::empty(spec).temporary_block_header(spec),
             historical_roots: vec![],
 
@@ -505,7 +507,7 @@ impl BeaconState {
     /// Spec v0.5.0
     pub fn fill_active_index_roots_with(&mut self, index_root: Hash256, spec: &ChainSpec) {
         self.latest_active_index_roots =
-            vec![index_root; spec.latest_active_index_roots_length as usize]
+            vec![index_root; spec.latest_active_index_roots_length as usize].into()
     }
 
     /// Safely obtains the index for latest state roots, given some `slot`.

eth2/types/src/historical_batch.rs

@@ -1,5 +1,5 @@
 use crate::test_utils::TestRandom;
-use crate::Hash256;
+use crate::{Hash256, TreeHashVector};
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
@@ -11,8 +11,8 @@ use tree_hash_derive::TreeHash;
 /// Spec v0.5.0
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
 pub struct HistoricalBatch {
-    pub block_roots: Vec<Hash256>,
+    pub block_roots: TreeHashVector<Hash256>,
-    pub state_roots: Vec<Hash256>,
+    pub state_roots: TreeHashVector<Hash256>,
 }
 
 #[cfg(test)]

eth2/types/src/lib.rs

@@ -27,6 +27,7 @@ pub mod pending_attestation;
 pub mod proposer_slashing;
 pub mod slashable_attestation;
 pub mod transfer;
+pub mod tree_hash_vector;
 pub mod voluntary_exit;
 #[macro_use]
 pub mod slot_epoch_macros;
@@ -65,6 +66,7 @@ pub use crate::slashable_attestation::SlashableAttestation;
 pub use crate::slot_epoch::{Epoch, Slot};
 pub use crate::slot_height::SlotHeight;
 pub use crate::transfer::Transfer;
+pub use crate::tree_hash_vector::TreeHashVector;
 pub use crate::validator::Validator;
 pub use crate::voluntary_exit::VoluntaryExit;
 

eth2/types/src/test_utils/mod.rs

@@ -17,7 +17,10 @@ mod testing_voluntary_exit_builder;
 
 pub use generate_deterministic_keypairs::generate_deterministic_keypairs;
 pub use keypairs_file::KeypairsFile;
-pub use rand::{prng::XorShiftRng, SeedableRng};
+pub use rand::{
+    RngCore,
+    {prng::XorShiftRng, SeedableRng},
+};
 pub use serde_utils::{fork_from_hex_str, u8_from_hex_str};
 pub use test_random::TestRandom;
 pub use testing_attestation_builder::TestingAttestationBuilder;

eth2/types/src/tree_hash_vector.rs

@@ -0,0 +1,82 @@
+use crate::test_utils::{RngCore, TestRandom};
+use serde_derive::{Deserialize, Serialize};
+use ssz::{Decodable, DecodeError, Encodable, SszStream};
+use std::ops::{Deref, DerefMut};
+use tree_hash::TreeHash;
+
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
+pub struct TreeHashVector<T>(Vec<T>);
+
+impl<T> From<Vec<T>> for TreeHashVector<T> {
+    fn from(vec: Vec<T>) -> TreeHashVector<T> {
+        TreeHashVector(vec)
+    }
+}
+
+impl<T> Into<Vec<T>> for TreeHashVector<T> {
+    fn into(self) -> Vec<T> {
+        self.0
+    }
+}
+
+impl<T> Deref for TreeHashVector<T> {
+    type Target = Vec<T>;
+
+    fn deref(&self) -> &Vec<T> {
+        &self.0
+    }
+}
+
+impl<T> DerefMut for TreeHashVector<T> {
+    fn deref_mut(&mut self) -> &mut Vec<T> {
+        &mut self.0
+    }
+}
+
+impl<T> tree_hash::TreeHash for TreeHashVector<T>
+where
+    T: TreeHash,
+{
+    fn tree_hash_type() -> tree_hash::TreeHashType {
+        tree_hash::TreeHashType::Vector
+    }
+
+    fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+        unreachable!("Vector should never be packed.")
+    }
+
+    fn tree_hash_packing_factor() -> usize {
+        unreachable!("Vector should never be packed.")
+    }
+
+    fn tree_hash_root(&self) -> Vec<u8> {
+        tree_hash::standard_tree_hash::vec_tree_hash_root(self)
+    }
+}
+
+impl<T> Encodable for TreeHashVector<T>
+where
+    T: Encodable,
+{
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append_vec(self)
+    }
+}
+
+impl<T> Decodable for TreeHashVector<T>
+where
+    T: Decodable,
+{
+    fn ssz_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), DecodeError> {
+        ssz::decode_ssz_list(bytes, index).and_then(|(vec, i)| Ok((vec.into(), i)))
+    }
+}
+
+impl<T: RngCore, U> TestRandom<T> for TreeHashVector<U>
+where
+    U: TestRandom<T>,
+{
+    fn random_for_test(rng: &mut T) -> Self {
+        Vec::random_for_test(rng).into()
+    }
+}

eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs

@@ -9,7 +9,7 @@ where
             TreeHashType::Basic => {
                 TreeHashCache::from_bytes(merkleize(get_packed_leaves(self)?), false)
             }
-            TreeHashType::Composite | TreeHashType::List => {
+            TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
                 let subtrees = self
                     .iter()
                     .map(|item| TreeHashCache::new(item))
@@ -23,7 +23,7 @@ where
     fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
         let lengths = match T::tree_hash_type() {
             TreeHashType::Basic => vec![1; self.len() / T::tree_hash_packing_factor()],
-            TreeHashType::Composite | TreeHashType::List => {
+            TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
                 let mut lengths = vec![];
 
                 for item in self {
@@ -97,7 +97,7 @@ where
                     TreeHashCache::from_bytes(leaves, true)?,
                 );
             }
-            TreeHashType::Composite | TreeHashType::List => {
+            TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
                 let mut i = offset_handler.num_leaf_nodes;
                 for &start_chunk in offset_handler.iter_leaf_nodes().rev() {
                     i -= 1;

eth2/utils/tree_hash/src/lib.rs

@@ -13,8 +13,9 @@ pub use standard_tree_hash::{merkle_root, TreeHash};
 #[derive(Debug, PartialEq, Clone)]
 pub enum TreeHashType {
     Basic,
+    Vector,
     List,
-    Composite,
+    Container,
 }
 
 fn num_sanitized_leaves(num_bytes: usize) -> usize {
@@ -31,15 +32,15 @@ macro_rules! impl_tree_hash_for_ssz_bytes {
     ($type: ident) => {
         impl tree_hash::TreeHash for $type {
             fn tree_hash_type() -> tree_hash::TreeHashType {
-                tree_hash::TreeHashType::List
+                tree_hash::TreeHashType::Vector
             }
 
             fn tree_hash_packed_encoding(&self) -> Vec<u8> {
-                panic!("bytesN should never be packed.")
+                unreachable!("Vector should never be packed.")
             }
 
             fn tree_hash_packing_factor() -> usize {
-                panic!("bytesN should never be packed.")
+                unreachable!("Vector should never be packed.")
             }
 
             fn tree_hash_root(&self) -> Vec<u8> {
@@ -48,3 +49,26 @@ macro_rules! impl_tree_hash_for_ssz_bytes {
         }
     };
 }
+
+#[macro_export]
+macro_rules! impl_vec_as_fixed_len {
+    ($type: ty) => {
+        impl tree_hash::TreeHash for $type {
+            fn tree_hash_type() -> tree_hash::TreeHashType {
+                tree_hash::TreeHashType::Vector
+            }
+
+            fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+                unreachable!("Vector should never be packed.")
+            }
+
+            fn tree_hash_packing_factor() -> usize {
+                unreachable!("Vector should never be packed.")
+            }
+
+            fn tree_hash_root(&self) -> Vec<u8> {
+                tree_hash::standard_tree_hash::vec_tree_hash_root(self)
+            }
+        }
+    };
+}

eth2/utils/tree_hash/src/standard_tree_hash.rs

@@ -3,6 +3,8 @@ use hashing::hash;
 use int_to_bytes::int_to_bytes32;
 use ssz::ssz_encode;
 
+pub use impls::vec_tree_hash_root;
+
 mod impls;
 
 pub trait TreeHash {
@@ -16,11 +18,18 @@ pub trait TreeHash {
 }
 
 pub fn merkle_root(bytes: &[u8]) -> Vec<u8> {
-    // TODO: replace this with a _more_ efficient fn which is more memory efficient.
+    // TODO: replace this with a more memory efficient method.
     efficient_merkleize(&bytes)[0..32].to_vec()
 }
 
 pub fn efficient_merkleize(bytes: &[u8]) -> Vec<u8> {
+    // If the bytes are just one chunk (or less than one chunk) just return them.
+    if bytes.len() <= HASHSIZE {
+        let mut o = bytes.to_vec();
+        o.resize(HASHSIZE, 0);
+        return o;
+    }
+
     let leaves = num_sanitized_leaves(bytes.len());
     let nodes = num_nodes(leaves);
     let internal_nodes = nodes - leaves;
@@ -29,10 +38,6 @@ pub fn efficient_merkleize(bytes: &[u8]) -> Vec<u8> {
 
     let mut o: Vec<u8> = vec![0; internal_nodes * HASHSIZE];
 
-    if o.len() < HASHSIZE {
-        o.resize(HASHSIZE, 0);
-    }
-
     o.append(&mut bytes.to_vec());
 
     assert_eq!(o.len(), num_bytes);

eth2/utils/tree_hash/src/standard_tree_hash/impls.rs

@@ -50,7 +50,7 @@ impl TreeHash for [u8; 4] {
 
 impl TreeHash for H256 {
     fn tree_hash_type() -> TreeHashType {
-        TreeHashType::Basic
+        TreeHashType::Vector
     }
 
     fn tree_hash_packed_encoding(&self) -> Vec<u8> {
@@ -62,7 +62,7 @@ impl TreeHash for H256 {
     }
 
     fn tree_hash_root(&self) -> Vec<u8> {
-        ssz_encode(self)
+        merkle_root(&ssz::ssz_encode(self))
     }
 }
 
@@ -83,37 +83,43 @@ where
     }
 
     fn tree_hash_root(&self) -> Vec<u8> {
-        let leaves = match T::tree_hash_type() {
-            TreeHashType::Basic => {
-                let mut leaves =
-                    Vec::with_capacity((HASHSIZE / T::tree_hash_packing_factor()) * self.len());
-
-                for item in self {
-                    leaves.append(&mut item.tree_hash_packed_encoding());
-                }
-
-                leaves
-            }
-            TreeHashType::Composite | TreeHashType::List => {
-                let mut leaves = Vec::with_capacity(self.len() * HASHSIZE);
-
-                for item in self {
-                    leaves.append(&mut item.tree_hash_root())
-                }
-
-                leaves
-            }
-        };
-
-        // Mix in the length
         let mut root_and_len = Vec::with_capacity(HASHSIZE * 2);
-        root_and_len.append(&mut merkle_root(&leaves));
+        root_and_len.append(&mut vec_tree_hash_root(self));
         root_and_len.append(&mut int_to_bytes32(self.len() as u64));
 
         hash(&root_and_len)
     }
 }
 
+pub fn vec_tree_hash_root<T>(vec: &[T]) -> Vec<u8>
+where
+    T: TreeHash,
+{
+    let leaves = match T::tree_hash_type() {
+        TreeHashType::Basic => {
+            let mut leaves =
+                Vec::with_capacity((HASHSIZE / T::tree_hash_packing_factor()) * vec.len());
+
+            for item in vec {
+                leaves.append(&mut item.tree_hash_packed_encoding());
+            }
+
+            leaves
+        }
+        TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
+            let mut leaves = Vec::with_capacity(vec.len() * HASHSIZE);
+
+            for item in vec {
+                leaves.append(&mut item.tree_hash_root())
+            }
+
+            leaves
+        }
+    };
+
+    merkle_root(&leaves)
+}
+
 #[cfg(test)]
 mod test {
     use super::*;

eth2/utils/tree_hash/tests/tests.rs

@@ -13,7 +13,7 @@ pub struct InternalCache {
 
 impl TreeHash for InternalCache {
     fn tree_hash_type() -> TreeHashType {
-        TreeHashType::Composite
+        TreeHashType::Container
     }
 
     fn tree_hash_packed_encoding(&self) -> Vec<u8> {
@@ -146,7 +146,7 @@ pub struct Inner {
 
 impl TreeHash for Inner {
     fn tree_hash_type() -> TreeHashType {
-        TreeHashType::Composite
+        TreeHashType::Container
     }
 
     fn tree_hash_packed_encoding(&self) -> Vec<u8> {
@@ -231,7 +231,7 @@ pub struct Outer {
 
 impl TreeHash for Outer {
     fn tree_hash_type() -> TreeHashType {
-        TreeHashType::Composite
+        TreeHashType::Container
     }
 
     fn tree_hash_packed_encoding(&self) -> Vec<u8> {
@@ -894,11 +894,39 @@ fn vec_of_u64_builds() {
 
     let my_vec = vec![1, 2, 3, 4, 5];
 
+    //
+    // Note: the length is not mixed-in in this example. The user must ensure the length is
+    // mixed-in.
+    //
+
     let cache: Vec<u8> = TreeHashCache::new(&my_vec).unwrap().into();
 
     assert_eq!(expected, cache);
 }
 
+#[test]
+fn vec_does_mix_in_len() {
+    let data = join(vec![
+        int_to_bytes8(1),
+        int_to_bytes8(2),
+        int_to_bytes8(3),
+        int_to_bytes8(4),
+        int_to_bytes8(5),
+        vec![0; 32 - 8], // padding
+    ]);
+
+    let tree = merkleize(data);
+
+    let my_vec: Vec<u64> = vec![1, 2, 3, 4, 5];
+
+    let mut expected = vec![0; 32];
+    expected.copy_from_slice(&tree[0..HASHSIZE]);
+    expected.append(&mut int_to_bytes32(my_vec.len() as u64));
+    let expected = hash(&expected);
+
+    assert_eq!(&expected[0..HASHSIZE], &my_vec.tree_hash_root()[..]);
+}
+
 #[test]
 fn merkleize_odd() {
     let data = join(vec![

eth2/utils/tree_hash_derive/src/lib.rs

@@ -129,7 +129,7 @@ pub fn tree_hash_derive(input: TokenStream) -> TokenStream {
     let output = quote! {
         impl tree_hash::TreeHash for #name {
             fn tree_hash_type() -> tree_hash::TreeHashType {
-                tree_hash::TreeHashType::Composite
+                tree_hash::TreeHashType::Container
             }
 
             fn tree_hash_packed_encoding(&self) -> Vec<u8> {
@@ -154,19 +154,6 @@ pub fn tree_hash_derive(input: TokenStream) -> TokenStream {
     output.into()
 }
 
-/// Implements `tree_hash::TreeHash` for some `struct`, whilst excluding any fields following and
-/// including a field that is of type "Signature" or "AggregateSignature".
-///
-/// See:
-/// https://github.com/ethereum/eth2.0-specs/blob/master/specs/simple-serialize.md#signed-roots
-///
-/// This is a rather horrendous macro, it will read the type of the object as a string and decide
-/// if it's a signature by matching that string against "Signature" or "AggregateSignature". So,
-/// it's important that you use those exact words as your type -- don't alias it to something else.
-///
-/// If you can think of a better way to do this, please make an issue!
-///
-/// Fields are processed in the order they are defined.
 #[proc_macro_derive(SignedRoot, attributes(signed_root))]
 pub fn tree_hash_signed_root_derive(input: TokenStream) -> TokenStream {
     let item = parse_macro_input!(input as DeriveInput);