From e28e97d3c71c47762905340f5352759ba9afa282 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 27 Mar 2019 15:59:27 +1100
Subject: [PATCH 001/137] Add initial work on tree hash caching

---
 eth2/utils/ssz/src/cached_tree_hash.rs | 331 +++++++++++++++++++++++++
 eth2/utils/ssz/src/lib.rs              |   1 +
 2 files changed, 332 insertions(+)
 create mode 100644 eth2/utils/ssz/src/cached_tree_hash.rs

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
new file mode 100644
index 000000000..7a5b1c527
--- /dev/null
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -0,0 +1,331 @@
+use crate::ssz_encode;
+use hashing::hash;
+
+const BYTES_PER_CHUNK: usize = 32;
+const HASHSIZE: usize = 32;
+const MERKLE_HASH_CHUNCK: usize = 2 * BYTES_PER_CHUNK;
+
+pub trait CachedTreeHash {
+    fn cached_hash_tree_root(
+        &self,
+        other: &Self,
+        cache: &mut [u8],
+        i: usize,
+        changes: Vec<bool>,
+    ) -> Option<(usize, Vec<bool>)>;
+}
+
+impl CachedTreeHash for u64 {
+    fn cached_hash_tree_root(
+        &self,
+        other: &Self,
+        cache: &mut [u8],
+        i: usize,
+        mut changes: Vec<bool>,
+    ) -> Option<(usize, Vec<bool>)> {
+        if self != other {
+            cache
+                .get_mut(i..i + HASHSIZE)?
+                .copy_from_slice(&mut hash(&ssz_encode(self)));
+            changes.push(true);
+        } else {
+            changes.push(false);
+        };
+
+        Some((i + HASHSIZE, changes))
+    }
+}
+
+pub struct Inner {
+    pub a: u64,
+    pub b: u64,
+    pub c: u64,
+    pub d: u64,
+}
+
+impl CachedTreeHash for Inner {
+    fn cached_hash_tree_root(
+        &self,
+        other: &Self,
+        cache: &mut [u8],
+        i: usize,
+        mut changes: Vec<bool>,
+    ) -> Option<(usize, Vec<bool>)> {
+        let original_start = i;
+
+        let leaves = 4;
+        let nodes = num_nodes(leaves);
+        let internal = nodes - leaves;
+        let leaves_start = i + internal * HASHSIZE;
+
+        let mut leaf_changes = {
+            let leaf_changes = Vec::with_capacity(leaves);
+            let leaf_start = leaves_start;
+
+            let (leaf_start, leaf_changes) =
+                self.a
+                    .cached_hash_tree_root(&other.a, cache, leaf_start, leaf_changes)?;
+            let (leaf_start, leaf_changes) =
+                self.b
+                    .cached_hash_tree_root(&other.b, cache, leaf_start, leaf_changes)?;
+            let (leaf_start, leaf_changes) =
+                self.c
+                    .cached_hash_tree_root(&other.c, cache, leaf_start, leaf_changes)?;
+            let (_leaf_start, leaf_changes) =
+                self.d
+                    .cached_hash_tree_root(&other.d, cache, leaf_start, leaf_changes)?;
+
+            leaf_changes
+        };
+
+        let any_changes = leaf_changes.iter().any(|&c| c);
+
+        changes.resize(changes.len() + internal, false);
+        changes.append(&mut leaf_changes);
+
+        if any_changes {
+            let mut i = internal;
+
+            while i > 0 {
+                let children = children(i);
+
+                if changes[children.0] | changes[children.1] {
+                    changes[parent(i)] = true;
+
+                    let children_start = children.0 * HASHSIZE;
+                    let children_end = children_start + 2 * HASHSIZE;
+                    let hash = hash(&cache.get(children_start..children_end)?);
+
+                    cache
+                        .get_mut(i * HASHSIZE..(i + 1) * HASHSIZE)?
+                        .copy_from_slice(&hash);
+                }
+                i += 1
+            }
+        }
+
+        Some((42, vec![any_changes]))
+    }
+}
+
+/// Get merkle root of some hashed values - the input leaf nodes is expected to already be hashed
+/// Outputs a `Vec<u8>` byte array of the merkle root given a set of leaf node values.
+pub fn cache_builder(values: &[u8]) -> Option<Vec<u8>> {
+    let leaves = values.len() / HASHSIZE;
+
+    if leaves == 0 || !leaves.is_power_of_two() {
+        return None;
+    }
+
+    let mut o: Vec<u8> = vec![0; (num_nodes(leaves) - leaves) * HASHSIZE];
+    o.append(&mut values.to_vec());
+
+    let mut i = o.len();
+    let mut j = o.len() - values.len();
+
+    while i >= MERKLE_HASH_CHUNCK {
+        i -= MERKLE_HASH_CHUNCK;
+        let hash = hash(&o[i..i + MERKLE_HASH_CHUNCK]);
+
+        j -= HASHSIZE;
+        o.get_mut(j..j + HASHSIZE)?.copy_from_slice(&hash);
+    }
+
+    return Some(o);
+}
+
+fn parent(child: usize) -> usize {
+    (child - 1) / 2
+}
+
+fn children(parent: usize) -> (usize, usize) {
+    ((2 * parent + 1), (2 * parent + 2))
+}
+
+fn num_nodes(num_leaves: usize) -> usize {
+    2 * num_leaves - 1
+}
+
+pub struct Outer {
+    pub a: u64,
+    pub b: u64,
+    pub inner: Inner,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn join(many: Vec<&[u8]>) -> Vec<u8> {
+        let mut all = vec![];
+        for one in many {
+            all.extend_from_slice(&mut one.clone())
+        }
+        all
+    }
+
+    /*
+    #[test]
+    fn container() {
+        let data1 = hash(&vec![1; 32]);
+        let data2 = hash(&vec![2; 32]);
+        let data3 = hash(&vec![3; 32]);
+        let data4 = hash(&vec![4; 32]);
+
+        let data = join(vec![&data1, &data2, &data3, &data4]);
+
+        let cache = cache_builder(&data).unwrap();
+    }
+    */
+
+    #[test]
+    fn can_build_cache() {
+        let data1 = hash(&vec![1; 32]);
+        let data2 = hash(&vec![2; 32]);
+        let data3 = hash(&vec![3; 32]);
+        let data4 = hash(&vec![4; 32]);
+
+        let data = join(vec![&data1, &data2, &data3, &data4]);
+
+        let cache = cache_builder(&data).unwrap();
+
+        let hash_12 = {
+            let mut joined = vec![];
+            joined.append(&mut data1.clone());
+            joined.append(&mut data2.clone());
+            hash(&joined)
+        };
+        let hash_34 = {
+            let mut joined = vec![];
+            joined.append(&mut data3.clone());
+            joined.append(&mut data4.clone());
+            hash(&joined)
+        };
+        let hash_hash12_hash_34 = {
+            let mut joined = vec![];
+            joined.append(&mut hash_12.clone());
+            joined.append(&mut hash_34.clone());
+            hash(&joined)
+        };
+
+        for (i, chunk) in cache.chunks(HASHSIZE).enumerate().rev() {
+            let expected = match i {
+                0 => hash_hash12_hash_34.clone(),
+                1 => hash_12.clone(),
+                2 => hash_34.clone(),
+                3 => data1.clone(),
+                4 => data2.clone(),
+                5 => data3.clone(),
+                6 => data4.clone(),
+                _ => vec![],
+            };
+
+            assert_eq!(chunk, &expected[..], "failed at {}", i);
+        }
+    }
+}
+
+/*
+pub trait TreeHash {
+    fn hash_tree_root(&self) -> Vec<u8>;
+}
+
+/// Returns a 32 byte hash of 'list' - a vector of byte vectors.
+/// Note that this will consume 'list'.
+pub fn merkle_hash(list: &mut Vec<Vec<u8>>) -> Vec<u8> {
+    // flatten list
+    let mut chunkz = list_to_blob(list);
+
+    // get data_len as bytes. It will hashed will the merkle root
+    let mut datalen = list.len().to_le_bytes().to_vec();
+    zpad(&mut datalen, 32);
+
+    // merklelize
+    while chunkz.len() > HASHSIZE {
+        let mut new_chunkz: Vec<u8> = Vec::new();
+
+        for two_chunks in chunkz.chunks(BYTES_PER_CHUNK * 2) {
+            // Hash two chuncks together
+            new_chunkz.append(&mut hash(two_chunks));
+        }
+
+        chunkz = new_chunkz;
+    }
+
+    chunkz.append(&mut datalen);
+    hash(&chunkz)
+}
+
+fn list_to_blob(list: &mut Vec<Vec<u8>>) -> Vec<u8> {
+    // pack - fit as many many items per chunk as we can and then
+    // right pad to BYTES_PER_CHUNCK
+    let (items_per_chunk, chunk_count) = if list.is_empty() {
+        (1, 1)
+    } else {
+        let items_per_chunk = BYTES_PER_CHUNK / list[0].len();
+        let chunk_count = list.len() / items_per_chunk;
+        (items_per_chunk, chunk_count)
+    };
+
+    let mut chunkz = Vec::new();
+    if list.is_empty() {
+        // handle and empty list
+        chunkz.append(&mut vec![0; BYTES_PER_CHUNK * 2]);
+    } else if list[0].len() <= BYTES_PER_CHUNK {
+        // just create a blob here; we'll divide into
+        // chunked slices when we merklize
+        let mut chunk = Vec::with_capacity(BYTES_PER_CHUNK);
+        let mut item_count_in_chunk = 0;
+        chunkz.reserve(chunk_count * BYTES_PER_CHUNK);
+        for item in list.iter_mut() {
+            item_count_in_chunk += 1;
+            chunk.append(item);
+
+            // completed chunk?
+            if item_count_in_chunk == items_per_chunk {
+                zpad(&mut chunk, BYTES_PER_CHUNK);
+                chunkz.append(&mut chunk);
+                item_count_in_chunk = 0;
+            }
+        }
+
+        // left-over uncompleted chunk?
+        if item_count_in_chunk != 0 {
+            zpad(&mut chunk, BYTES_PER_CHUNK);
+            chunkz.append(&mut chunk);
+        }
+    }
+
+    // extend the number of chunks to a power of two if necessary
+    if !chunk_count.is_power_of_two() {
+        let zero_chunks_count = chunk_count.next_power_of_two() - chunk_count;
+        chunkz.append(&mut vec![0; zero_chunks_count * BYTES_PER_CHUNK]);
+    }
+
+    chunkz
+}
+
+/// right pads with zeros making 'bytes' 'size' in length
+fn zpad(bytes: &mut Vec<u8>, size: usize) {
+    if bytes.len() < size {
+        bytes.resize(size, 0);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_merkle_hash() {
+        let data1 = vec![1; 32];
+        let data2 = vec![2; 32];
+        let data3 = vec![3; 32];
+        let mut list = vec![data1, data2, data3];
+        let result = merkle_hash(&mut list);
+
+        //note: should test againt a known test hash value
+        assert_eq!(HASHSIZE, result.len());
+    }
+}
+*/
diff --git a/eth2/utils/ssz/src/lib.rs b/eth2/utils/ssz/src/lib.rs
index 7c29667af..9bf441249 100644
--- a/eth2/utils/ssz/src/lib.rs
+++ b/eth2/utils/ssz/src/lib.rs
@@ -10,6 +10,7 @@
 extern crate bytes;
 extern crate ethereum_types;
 
+mod cached_tree_hash;
 pub mod decode;
 pub mod encode;
 mod signed_root;

From 35ceb92f2e94f70a48cfa7e0155d31ce55da531c Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 27 Mar 2019 17:45:27 +1100
Subject: [PATCH 002/137] Refactor with TreeHashCache struct

---
 eth2/utils/ssz/Cargo.toml              |   1 +
 eth2/utils/ssz/src/cached_tree_hash.rs | 358 +++++++++++++++++--------
 2 files changed, 243 insertions(+), 116 deletions(-)

diff --git a/eth2/utils/ssz/Cargo.toml b/eth2/utils/ssz/Cargo.toml
index f13db5def..21b726e9f 100644
--- a/eth2/utils/ssz/Cargo.toml
+++ b/eth2/utils/ssz/Cargo.toml
@@ -8,3 +8,4 @@ edition = "2018"
 bytes = "0.4.9"
 ethereum-types = "0.5"
 hashing = { path = "../hashing" }
+int_to_bytes = { path = "../int_to_bytes" }
diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index 7a5b1c527..99fd49221 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -1,38 +1,97 @@
 use crate::ssz_encode;
 use hashing::hash;
+use int_to_bytes::int_to_bytes32;
 
 const BYTES_PER_CHUNK: usize = 32;
 const HASHSIZE: usize = 32;
 const MERKLE_HASH_CHUNCK: usize = 2 * BYTES_PER_CHUNK;
 
+pub struct TreeHashCache<'a> {
+    chunk_offset: usize,
+    cache: &'a mut [u8],
+    chunk_modified: &'a mut [bool],
+}
+
+impl<'a> TreeHashCache<'a> {
+    pub fn increment(&mut self) {
+        self.chunk_offset += 1
+    }
+
+    pub fn modify_current_chunk(&mut self, to: &[u8]) -> Option<()> {
+        self.modify_chunk(0, to)
+    }
+
+    pub fn modify_chunk(&mut self, chunk: usize, to: &[u8]) -> Option<()> {
+        let start = chunk * BYTES_PER_CHUNK;
+        let end = start + BYTES_PER_CHUNK;
+        self.cache.get_mut(start..end)?.copy_from_slice(to);
+
+        self.chunk_modified[chunk] = true;
+
+        Some(())
+    }
+
+    pub fn changed(&self, chunk: usize) -> Option<bool> {
+        self.chunk_modified.get(chunk).cloned()
+    }
+
+    pub fn children_modified(&self, parent_chunk: usize) -> Option<bool> {
+        let children = children(parent_chunk);
+
+        Some(self.changed(children.0)? | self.changed(children.1)?)
+    }
+
+    pub fn hash_children(&self, parent_chunk: usize) -> Option<Vec<u8>> {
+        let children = children(parent_chunk);
+
+        let start = children.0 * BYTES_PER_CHUNK;
+        let end = start + BYTES_PER_CHUNK * 2;
+
+        Some(hash(&self.cache.get(start..end)?))
+    }
+
+    pub fn just_the_leaves(&mut self, leaves: usize) -> Option<TreeHashCache> {
+        let nodes = num_nodes(leaves);
+        let internal = nodes - leaves;
+        let leaves_start = (self.chunk_offset + internal) * HASHSIZE;
+
+        Some(TreeHashCache {
+            chunk_offset: self.chunk_offset + internal,
+            cache: self.cache.get_mut(leaves_start..leaves * HASHSIZE)?,
+            chunk_modified: self
+                .chunk_modified
+                .get_mut(self.chunk_offset..self.chunk_offset + leaves)?,
+        })
+    }
+}
+
+fn children(parent: usize) -> (usize, usize) {
+    ((2 * parent + 1), (2 * parent + 2))
+}
+
+fn num_nodes(num_leaves: usize) -> usize {
+    2 * num_leaves - 1
+}
+
 pub trait CachedTreeHash {
-    fn cached_hash_tree_root(
-        &self,
-        other: &Self,
-        cache: &mut [u8],
-        i: usize,
-        changes: Vec<bool>,
-    ) -> Option<(usize, Vec<bool>)>;
+    fn build_cache_bytes(&self) -> Vec<u8>;
+
+    fn cached_hash_tree_root(&self, other: &Self, cache: &mut TreeHashCache) -> Option<()>;
 }
 
 impl CachedTreeHash for u64 {
-    fn cached_hash_tree_root(
-        &self,
-        other: &Self,
-        cache: &mut [u8],
-        i: usize,
-        mut changes: Vec<bool>,
-    ) -> Option<(usize, Vec<bool>)> {
-        if self != other {
-            cache
-                .get_mut(i..i + HASHSIZE)?
-                .copy_from_slice(&mut hash(&ssz_encode(self)));
-            changes.push(true);
-        } else {
-            changes.push(false);
-        };
+    fn build_cache_bytes(&self) -> Vec<u8> {
+        merkleize(&ssz_encode(self))
+    }
 
-        Some((i + HASHSIZE, changes))
+    fn cached_hash_tree_root(&self, other: &Self, cache: &mut TreeHashCache) -> Option<()> {
+        if self != other {
+            cache.modify_current_chunk(&hash(&ssz_encode(self)));
+        }
+
+        cache.increment();
+
+        Some(())
     }
 }
 
@@ -44,70 +103,137 @@ pub struct Inner {
 }
 
 impl CachedTreeHash for Inner {
-    fn cached_hash_tree_root(
-        &self,
-        other: &Self,
-        cache: &mut [u8],
-        i: usize,
-        mut changes: Vec<bool>,
-    ) -> Option<(usize, Vec<bool>)> {
-        let original_start = i;
+    fn build_cache_bytes(&self) -> Vec<u8> {
+        let mut leaves = vec![];
 
-        let leaves = 4;
-        let nodes = num_nodes(leaves);
-        let internal = nodes - leaves;
-        let leaves_start = i + internal * HASHSIZE;
+        leaves.append(&mut self.a.build_cache_bytes());
+        leaves.append(&mut self.b.build_cache_bytes());
+        leaves.append(&mut self.c.build_cache_bytes());
+        leaves.append(&mut self.d.build_cache_bytes());
 
-        let mut leaf_changes = {
-            let leaf_changes = Vec::with_capacity(leaves);
-            let leaf_start = leaves_start;
+        merkleize(&leaves)
+    }
 
-            let (leaf_start, leaf_changes) =
-                self.a
-                    .cached_hash_tree_root(&other.a, cache, leaf_start, leaf_changes)?;
-            let (leaf_start, leaf_changes) =
-                self.b
-                    .cached_hash_tree_root(&other.b, cache, leaf_start, leaf_changes)?;
-            let (leaf_start, leaf_changes) =
-                self.c
-                    .cached_hash_tree_root(&other.c, cache, leaf_start, leaf_changes)?;
-            let (_leaf_start, leaf_changes) =
-                self.d
-                    .cached_hash_tree_root(&other.d, cache, leaf_start, leaf_changes)?;
+    fn cached_hash_tree_root(&self, other: &Self, cache: &mut TreeHashCache) -> Option<()> {
+        let num_leaves = 4;
 
-            leaf_changes
-        };
+        let mut leaf_cache = cache.just_the_leaves(num_leaves)?;
+        self.a.cached_hash_tree_root(&other.a, &mut leaf_cache)?;
+        self.b.cached_hash_tree_root(&other.b, &mut leaf_cache)?;
+        self.c.cached_hash_tree_root(&other.c, &mut leaf_cache)?;
+        self.d.cached_hash_tree_root(&other.d, &mut leaf_cache)?;
 
-        let any_changes = leaf_changes.iter().any(|&c| c);
+        let nodes = num_nodes(num_leaves);
+        let internal_chunks = nodes - num_leaves;
 
-        changes.resize(changes.len() + internal, false);
-        changes.append(&mut leaf_changes);
-
-        if any_changes {
-            let mut i = internal;
-
-            while i > 0 {
-                let children = children(i);
-
-                if changes[children.0] | changes[children.1] {
-                    changes[parent(i)] = true;
-
-                    let children_start = children.0 * HASHSIZE;
-                    let children_end = children_start + 2 * HASHSIZE;
-                    let hash = hash(&cache.get(children_start..children_end)?);
-
-                    cache
-                        .get_mut(i * HASHSIZE..(i + 1) * HASHSIZE)?
-                        .copy_from_slice(&hash);
-                }
-                i += 1
+        for chunk in 0..internal_chunks {
+            if cache.children_modified(chunk)? {
+                cache.modify_chunk(chunk, &cache.hash_children(chunk)?)?;
             }
         }
 
-        Some((42, vec![any_changes]))
+        Some(())
     }
 }
 
+/// A reference function to test against.
+pub fn merkleize(values: &[u8]) -> Vec<u8> {
+    let leaves = values.len() / HASHSIZE;
+
+    if leaves == 0 || !leaves.is_power_of_two() {
+        panic!("Handle bad leaf count");
+    }
+
+    let mut o: Vec<u8> = vec![0; (num_nodes(leaves) - leaves) * HASHSIZE];
+    o.append(&mut values.to_vec());
+
+    let mut i = o.len();
+    let mut j = o.len() - values.len();
+
+    while i >= MERKLE_HASH_CHUNCK {
+        i -= MERKLE_HASH_CHUNCK;
+        let hash = hash(&o[i..i + MERKLE_HASH_CHUNCK]);
+
+        j -= HASHSIZE;
+        o[j..j + HASHSIZE].copy_from_slice(&hash);
+    }
+
+    o
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn join(many: Vec<&[u8]>) -> Vec<u8> {
+        let mut all = vec![];
+        for one in many {
+            all.extend_from_slice(&mut one.clone())
+        }
+        all
+    }
+
+    /*
+    #[test]
+    fn container() {
+        let data1 = hash(&vec![1; 32]);
+        let data2 = hash(&vec![2; 32]);
+        let data3 = hash(&vec![3; 32]);
+        let data4 = hash(&vec![4; 32]);
+
+        let data = join(vec![&data1, &data2, &data3, &data4]);
+
+        let cache = cache_builder(&data).unwrap();
+    }
+    */
+    #[test]
+    fn merkleize_4_leaves() {
+        let data1 = hash(&int_to_bytes32(1));
+        let data2 = hash(&int_to_bytes32(2));
+        let data3 = hash(&int_to_bytes32(3));
+        let data4 = hash(&int_to_bytes32(4));
+
+        let data = join(vec![&data1, &data2, &data3, &data4]);
+
+        let cache = merkleize(&data);
+
+        let hash_12 = {
+            let mut joined = vec![];
+            joined.append(&mut data1.clone());
+            joined.append(&mut data2.clone());
+            hash(&joined)
+        };
+        let hash_34 = {
+            let mut joined = vec![];
+            joined.append(&mut data3.clone());
+            joined.append(&mut data4.clone());
+            hash(&joined)
+        };
+        let hash_hash12_hash_34 = {
+            let mut joined = vec![];
+            joined.append(&mut hash_12.clone());
+            joined.append(&mut hash_34.clone());
+            hash(&joined)
+        };
+
+        for (i, chunk) in cache.chunks(HASHSIZE).enumerate().rev() {
+            let expected = match i {
+                0 => hash_hash12_hash_34.clone(),
+                1 => hash_12.clone(),
+                2 => hash_34.clone(),
+                3 => data1.clone(),
+                4 => data2.clone(),
+                5 => data3.clone(),
+                6 => data4.clone(),
+                _ => vec![],
+            };
+
+            assert_eq!(chunk, &expected[..], "failed at {}", i);
+        }
+    }
+}
+/*
+
 /// Get merkle root of some hashed values - the input leaf nodes is expected to already be hashed
 /// Outputs a `Vec<u8>` byte array of the merkle root given a set of leaf node values.
 pub fn cache_builder(values: &[u8]) -> Option<Vec<u8>> {
@@ -177,52 +303,51 @@ mod tests {
         let cache = cache_builder(&data).unwrap();
     }
     */
+#[test]
+fn can_build_cache() {
+let data1 = hash(&vec![1; 32]);
+let data2 = hash(&vec![2; 32]);
+let data3 = hash(&vec![3; 32]);
+let data4 = hash(&vec![4; 32]);
 
-    #[test]
-    fn can_build_cache() {
-        let data1 = hash(&vec![1; 32]);
-        let data2 = hash(&vec![2; 32]);
-        let data3 = hash(&vec![3; 32]);
-        let data4 = hash(&vec![4; 32]);
+let data = join(vec![&data1, &data2, &data3, &data4]);
 
-        let data = join(vec![&data1, &data2, &data3, &data4]);
+let cache = cache_builder(&data).unwrap();
 
-        let cache = cache_builder(&data).unwrap();
+let hash_12 = {
+let mut joined = vec![];
+joined.append(&mut data1.clone());
+joined.append(&mut data2.clone());
+hash(&joined)
+};
+let hash_34 = {
+let mut joined = vec![];
+joined.append(&mut data3.clone());
+joined.append(&mut data4.clone());
+hash(&joined)
+};
+let hash_hash12_hash_34 = {
+let mut joined = vec![];
+joined.append(&mut hash_12.clone());
+joined.append(&mut hash_34.clone());
+hash(&joined)
+};
 
-        let hash_12 = {
-            let mut joined = vec![];
-            joined.append(&mut data1.clone());
-            joined.append(&mut data2.clone());
-            hash(&joined)
-        };
-        let hash_34 = {
-            let mut joined = vec![];
-            joined.append(&mut data3.clone());
-            joined.append(&mut data4.clone());
-            hash(&joined)
-        };
-        let hash_hash12_hash_34 = {
-            let mut joined = vec![];
-            joined.append(&mut hash_12.clone());
-            joined.append(&mut hash_34.clone());
-            hash(&joined)
-        };
+for (i, chunk) in cache.chunks(HASHSIZE).enumerate().rev() {
+let expected = match i {
+0 => hash_hash12_hash_34.clone(),
+1 => hash_12.clone(),
+2 => hash_34.clone(),
+3 => data1.clone(),
+4 => data2.clone(),
+5 => data3.clone(),
+6 => data4.clone(),
+_ => vec![],
+};
 
-        for (i, chunk) in cache.chunks(HASHSIZE).enumerate().rev() {
-            let expected = match i {
-                0 => hash_hash12_hash_34.clone(),
-                1 => hash_12.clone(),
-                2 => hash_34.clone(),
-                3 => data1.clone(),
-                4 => data2.clone(),
-                5 => data3.clone(),
-                6 => data4.clone(),
-                _ => vec![],
-            };
-
-            assert_eq!(chunk, &expected[..], "failed at {}", i);
-        }
-    }
+assert_eq!(chunk, &expected[..], "failed at {}", i);
+}
+}
 }
 
 /*
@@ -329,3 +454,4 @@ mod tests {
     }
 }
 */
+*/

From ad4000cbdf2fa0c23560ae005c10360229115cf5 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 27 Mar 2019 17:46:12 +1100
Subject: [PATCH 003/137] Remove unused code

---
 eth2/utils/ssz/src/cached_tree_hash.rs | 236 -------------------------
 1 file changed, 236 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index 99fd49221..756f97232 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -173,19 +173,6 @@ mod tests {
         all
     }
 
-    /*
-    #[test]
-    fn container() {
-        let data1 = hash(&vec![1; 32]);
-        let data2 = hash(&vec![2; 32]);
-        let data3 = hash(&vec![3; 32]);
-        let data4 = hash(&vec![4; 32]);
-
-        let data = join(vec![&data1, &data2, &data3, &data4]);
-
-        let cache = cache_builder(&data).unwrap();
-    }
-    */
     #[test]
     fn merkleize_4_leaves() {
         let data1 = hash(&int_to_bytes32(1));
@@ -232,226 +219,3 @@ mod tests {
         }
     }
 }
-/*
-
-/// Get merkle root of some hashed values - the input leaf nodes is expected to already be hashed
-/// Outputs a `Vec<u8>` byte array of the merkle root given a set of leaf node values.
-pub fn cache_builder(values: &[u8]) -> Option<Vec<u8>> {
-    let leaves = values.len() / HASHSIZE;
-
-    if leaves == 0 || !leaves.is_power_of_two() {
-        return None;
-    }
-
-    let mut o: Vec<u8> = vec![0; (num_nodes(leaves) - leaves) * HASHSIZE];
-    o.append(&mut values.to_vec());
-
-    let mut i = o.len();
-    let mut j = o.len() - values.len();
-
-    while i >= MERKLE_HASH_CHUNCK {
-        i -= MERKLE_HASH_CHUNCK;
-        let hash = hash(&o[i..i + MERKLE_HASH_CHUNCK]);
-
-        j -= HASHSIZE;
-        o.get_mut(j..j + HASHSIZE)?.copy_from_slice(&hash);
-    }
-
-    return Some(o);
-}
-
-fn parent(child: usize) -> usize {
-    (child - 1) / 2
-}
-
-fn children(parent: usize) -> (usize, usize) {
-    ((2 * parent + 1), (2 * parent + 2))
-}
-
-fn num_nodes(num_leaves: usize) -> usize {
-    2 * num_leaves - 1
-}
-
-pub struct Outer {
-    pub a: u64,
-    pub b: u64,
-    pub inner: Inner,
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    fn join(many: Vec<&[u8]>) -> Vec<u8> {
-        let mut all = vec![];
-        for one in many {
-            all.extend_from_slice(&mut one.clone())
-        }
-        all
-    }
-
-    /*
-    #[test]
-    fn container() {
-        let data1 = hash(&vec![1; 32]);
-        let data2 = hash(&vec![2; 32]);
-        let data3 = hash(&vec![3; 32]);
-        let data4 = hash(&vec![4; 32]);
-
-        let data = join(vec![&data1, &data2, &data3, &data4]);
-
-        let cache = cache_builder(&data).unwrap();
-    }
-    */
-#[test]
-fn can_build_cache() {
-let data1 = hash(&vec![1; 32]);
-let data2 = hash(&vec![2; 32]);
-let data3 = hash(&vec![3; 32]);
-let data4 = hash(&vec![4; 32]);
-
-let data = join(vec![&data1, &data2, &data3, &data4]);
-
-let cache = cache_builder(&data).unwrap();
-
-let hash_12 = {
-let mut joined = vec![];
-joined.append(&mut data1.clone());
-joined.append(&mut data2.clone());
-hash(&joined)
-};
-let hash_34 = {
-let mut joined = vec![];
-joined.append(&mut data3.clone());
-joined.append(&mut data4.clone());
-hash(&joined)
-};
-let hash_hash12_hash_34 = {
-let mut joined = vec![];
-joined.append(&mut hash_12.clone());
-joined.append(&mut hash_34.clone());
-hash(&joined)
-};
-
-for (i, chunk) in cache.chunks(HASHSIZE).enumerate().rev() {
-let expected = match i {
-0 => hash_hash12_hash_34.clone(),
-1 => hash_12.clone(),
-2 => hash_34.clone(),
-3 => data1.clone(),
-4 => data2.clone(),
-5 => data3.clone(),
-6 => data4.clone(),
-_ => vec![],
-};
-
-assert_eq!(chunk, &expected[..], "failed at {}", i);
-}
-}
-}
-
-/*
-pub trait TreeHash {
-    fn hash_tree_root(&self) -> Vec<u8>;
-}
-
-/// Returns a 32 byte hash of 'list' - a vector of byte vectors.
-/// Note that this will consume 'list'.
-pub fn merkle_hash(list: &mut Vec<Vec<u8>>) -> Vec<u8> {
-    // flatten list
-    let mut chunkz = list_to_blob(list);
-
-    // get data_len as bytes. It will hashed will the merkle root
-    let mut datalen = list.len().to_le_bytes().to_vec();
-    zpad(&mut datalen, 32);
-
-    // merklelize
-    while chunkz.len() > HASHSIZE {
-        let mut new_chunkz: Vec<u8> = Vec::new();
-
-        for two_chunks in chunkz.chunks(BYTES_PER_CHUNK * 2) {
-            // Hash two chuncks together
-            new_chunkz.append(&mut hash(two_chunks));
-        }
-
-        chunkz = new_chunkz;
-    }
-
-    chunkz.append(&mut datalen);
-    hash(&chunkz)
-}
-
-fn list_to_blob(list: &mut Vec<Vec<u8>>) -> Vec<u8> {
-    // pack - fit as many many items per chunk as we can and then
-    // right pad to BYTES_PER_CHUNCK
-    let (items_per_chunk, chunk_count) = if list.is_empty() {
-        (1, 1)
-    } else {
-        let items_per_chunk = BYTES_PER_CHUNK / list[0].len();
-        let chunk_count = list.len() / items_per_chunk;
-        (items_per_chunk, chunk_count)
-    };
-
-    let mut chunkz = Vec::new();
-    if list.is_empty() {
-        // handle and empty list
-        chunkz.append(&mut vec![0; BYTES_PER_CHUNK * 2]);
-    } else if list[0].len() <= BYTES_PER_CHUNK {
-        // just create a blob here; we'll divide into
-        // chunked slices when we merklize
-        let mut chunk = Vec::with_capacity(BYTES_PER_CHUNK);
-        let mut item_count_in_chunk = 0;
-        chunkz.reserve(chunk_count * BYTES_PER_CHUNK);
-        for item in list.iter_mut() {
-            item_count_in_chunk += 1;
-            chunk.append(item);
-
-            // completed chunk?
-            if item_count_in_chunk == items_per_chunk {
-                zpad(&mut chunk, BYTES_PER_CHUNK);
-                chunkz.append(&mut chunk);
-                item_count_in_chunk = 0;
-            }
-        }
-
-        // left-over uncompleted chunk?
-        if item_count_in_chunk != 0 {
-            zpad(&mut chunk, BYTES_PER_CHUNK);
-            chunkz.append(&mut chunk);
-        }
-    }
-
-    // extend the number of chunks to a power of two if necessary
-    if !chunk_count.is_power_of_two() {
-        let zero_chunks_count = chunk_count.next_power_of_two() - chunk_count;
-        chunkz.append(&mut vec![0; zero_chunks_count * BYTES_PER_CHUNK]);
-    }
-
-    chunkz
-}
-
-/// right pads with zeros making 'bytes' 'size' in length
-fn zpad(bytes: &mut Vec<u8>, size: usize) {
-    if bytes.len() < size {
-        bytes.resize(size, 0);
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_merkle_hash() {
-        let data1 = vec![1; 32];
-        let data2 = vec![2; 32];
-        let data3 = vec![3; 32];
-        let mut list = vec![data1, data2, data3];
-        let result = merkle_hash(&mut list);
-
-        //note: should test againt a known test hash value
-        assert_eq!(HASHSIZE, result.len());
-    }
-}
-*/
-*/

From 839ff3ea3b4885126074e723e8fed1c448c25fe1 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 27 Mar 2019 18:34:10 +1100
Subject: [PATCH 004/137] Implement (failing) cached tree hash tests

---
 eth2/utils/ssz/src/cached_tree_hash.rs | 91 +++++++++++++++++++++++++-
 1 file changed, 88 insertions(+), 3 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index 756f97232..328fdb394 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -13,6 +13,24 @@ pub struct TreeHashCache<'a> {
 }
 
 impl<'a> TreeHashCache<'a> {
+    pub fn build_changes_vec(bytes: &[u8]) -> Vec<bool> {
+        vec![false; bytes.len() / BYTES_PER_CHUNK]
+    }
+
+    pub fn from_mut_slice(bytes: &'a mut [u8], changes: &'a mut [bool]) -> Option<Self> {
+        if bytes.len() % BYTES_PER_CHUNK > 0 {
+            return None;
+        }
+
+        let chunk_modified = vec![false; bytes.len() / BYTES_PER_CHUNK];
+
+        Some(Self {
+            chunk_offset: 0,
+            cache: bytes,
+            chunk_modified: changes,
+        })
+    }
+
     pub fn increment(&mut self) {
         self.chunk_offset += 1
     }
@@ -63,6 +81,10 @@ impl<'a> TreeHashCache<'a> {
                 .get_mut(self.chunk_offset..self.chunk_offset + leaves)?,
         })
     }
+
+    pub fn into_slice(self) -> &'a [u8] {
+        self.cache
+    }
 }
 
 fn children(parent: usize) -> (usize, usize) {
@@ -81,7 +103,7 @@ pub trait CachedTreeHash {
 
 impl CachedTreeHash for u64 {
     fn build_cache_bytes(&self) -> Vec<u8> {
-        merkleize(&ssz_encode(self))
+        merkleize(&int_to_bytes32(*self))
     }
 
     fn cached_hash_tree_root(&self, other: &Self, cache: &mut TreeHashCache) -> Option<()> {
@@ -95,6 +117,7 @@ impl CachedTreeHash for u64 {
     }
 }
 
+#[derive(Clone)]
 pub struct Inner {
     pub a: u64,
     pub b: u64,
@@ -123,6 +146,8 @@ impl CachedTreeHash for Inner {
         self.c.cached_hash_tree_root(&other.c, &mut leaf_cache)?;
         self.d.cached_hash_tree_root(&other.d, &mut leaf_cache)?;
 
+        dbg!(leaf_cache.into_slice());
+
         let nodes = num_nodes(num_leaves);
         let internal_chunks = nodes - num_leaves;
 
@@ -140,8 +165,12 @@ impl CachedTreeHash for Inner {
 pub fn merkleize(values: &[u8]) -> Vec<u8> {
     let leaves = values.len() / HASHSIZE;
 
-    if leaves == 0 || !leaves.is_power_of_two() {
-        panic!("Handle bad leaf count");
+    if leaves == 0 {
+        panic!("No full leaves");
+    }
+
+    if !leaves.is_power_of_two() {
+        panic!("leaves is not power of two");
     }
 
     let mut o: Vec<u8> = vec![0; (num_nodes(leaves) - leaves) * HASHSIZE];
@@ -173,6 +202,62 @@ mod tests {
         all
     }
 
+    #[test]
+    fn cached_hash_on_inner() {
+        let inner = Inner {
+            a: 1,
+            b: 2,
+            c: 3,
+            d: 4,
+        };
+
+        let mut cache = inner.build_cache_bytes();
+
+        let changed_inner = Inner {
+            a: 42,
+            ..inner.clone()
+        };
+
+        let mut changes = TreeHashCache::build_changes_vec(&cache);
+        let mut cache_struct = TreeHashCache::from_mut_slice(&mut cache, &mut changes).unwrap();
+
+        changed_inner.cached_hash_tree_root(&inner, &mut cache_struct);
+
+        let new_cache = cache_struct.into_slice();
+
+        let data1 = &int_to_bytes32(42);
+        let data2 = &int_to_bytes32(2);
+        let data3 = &int_to_bytes32(3);
+        let data4 = &int_to_bytes32(4);
+
+        let data = join(vec![&data1, &data2, &data3, &data4]);
+        let expected = merkleize(&data);
+
+        assert_eq!(expected, new_cache);
+    }
+
+    #[test]
+    fn build_cache_matches_merkelize() {
+        let data1 = &int_to_bytes32(1);
+        let data2 = &int_to_bytes32(2);
+        let data3 = &int_to_bytes32(3);
+        let data4 = &int_to_bytes32(4);
+
+        let data = join(vec![&data1, &data2, &data3, &data4]);
+        let expected = merkleize(&data);
+
+        let inner = Inner {
+            a: 1,
+            b: 2,
+            c: 3,
+            d: 4,
+        };
+
+        let cache = inner.build_cache_bytes();
+
+        assert_eq!(expected, cache);
+    }
+
     #[test]
     fn merkleize_4_leaves() {
         let data1 = hash(&int_to_bytes32(1));

From e33d1d0ebb69a362d6a3e3fd47efc1b538dd3d92 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 27 Mar 2019 18:55:39 +1100
Subject: [PATCH 005/137] First passing tree hash test

---
 eth2/utils/ssz/src/cached_tree_hash.rs | 31 ++++++++++++++------------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index 328fdb394..a85da8fd9 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -71,14 +71,17 @@ impl<'a> TreeHashCache<'a> {
     pub fn just_the_leaves(&mut self, leaves: usize) -> Option<TreeHashCache> {
         let nodes = num_nodes(leaves);
         let internal = nodes - leaves;
-        let leaves_start = (self.chunk_offset + internal) * HASHSIZE;
+
+        let leaves_start = (self.chunk_offset + internal) * BYTES_PER_CHUNK;
+        let leaves_end = leaves_start + leaves * BYTES_PER_CHUNK;
+
+        let modified_start = self.chunk_offset + internal;
+        let modified_end = modified_start + leaves;
 
         Some(TreeHashCache {
             chunk_offset: self.chunk_offset + internal,
-            cache: self.cache.get_mut(leaves_start..leaves * HASHSIZE)?,
-            chunk_modified: self
-                .chunk_modified
-                .get_mut(self.chunk_offset..self.chunk_offset + leaves)?,
+            cache: self.cache.get_mut(leaves_start..leaves_end)?,
+            chunk_modified: self.chunk_modified.get_mut(modified_start..modified_end)?,
         })
     }
 
@@ -108,7 +111,7 @@ impl CachedTreeHash for u64 {
 
     fn cached_hash_tree_root(&self, other: &Self, cache: &mut TreeHashCache) -> Option<()> {
         if self != other {
-            cache.modify_current_chunk(&hash(&ssz_encode(self)));
+            cache.modify_current_chunk(&merkleize(&int_to_bytes32(*self)));
         }
 
         cache.increment();
@@ -140,18 +143,18 @@ impl CachedTreeHash for Inner {
     fn cached_hash_tree_root(&self, other: &Self, cache: &mut TreeHashCache) -> Option<()> {
         let num_leaves = 4;
 
-        let mut leaf_cache = cache.just_the_leaves(num_leaves)?;
-        self.a.cached_hash_tree_root(&other.a, &mut leaf_cache)?;
-        self.b.cached_hash_tree_root(&other.b, &mut leaf_cache)?;
-        self.c.cached_hash_tree_root(&other.c, &mut leaf_cache)?;
-        self.d.cached_hash_tree_root(&other.d, &mut leaf_cache)?;
-
-        dbg!(leaf_cache.into_slice());
+        {
+            let mut leaf_cache = cache.just_the_leaves(num_leaves)?;
+            self.a.cached_hash_tree_root(&other.a, &mut leaf_cache)?;
+            self.b.cached_hash_tree_root(&other.b, &mut leaf_cache)?;
+            self.c.cached_hash_tree_root(&other.c, &mut leaf_cache)?;
+            self.d.cached_hash_tree_root(&other.d, &mut leaf_cache)?;
+        }
 
         let nodes = num_nodes(num_leaves);
         let internal_chunks = nodes - num_leaves;
 
-        for chunk in 0..internal_chunks {
+        for chunk in (0..internal_chunks).into_iter().rev() {
             if cache.children_modified(chunk)? {
                 cache.modify_chunk(chunk, &cache.hash_children(chunk)?)?;
             }

From acb1dd47cd71187eb34ab74479b07299f3eab3ef Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 27 Mar 2019 19:31:02 +1100
Subject: [PATCH 006/137] Make tree hash pass tests

---
 eth2/utils/ssz/src/cached_tree_hash.rs | 98 +++++++++++++++++++-------
 1 file changed, 71 insertions(+), 27 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index a85da8fd9..757bfa9f7 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -10,6 +10,7 @@ pub struct TreeHashCache<'a> {
     chunk_offset: usize,
     cache: &'a mut [u8],
     chunk_modified: &'a mut [bool],
+    hash_count: &'a mut usize,
 }
 
 impl<'a> TreeHashCache<'a> {
@@ -17,17 +18,20 @@ impl<'a> TreeHashCache<'a> {
         vec![false; bytes.len() / BYTES_PER_CHUNK]
     }
 
-    pub fn from_mut_slice(bytes: &'a mut [u8], changes: &'a mut [bool]) -> Option<Self> {
+    pub fn from_mut_slice(
+        bytes: &'a mut [u8],
+        changes: &'a mut [bool],
+        hash_count: &'a mut usize,
+    ) -> Option<Self> {
         if bytes.len() % BYTES_PER_CHUNK > 0 {
             return None;
         }
 
-        let chunk_modified = vec![false; bytes.len() / BYTES_PER_CHUNK];
-
         Some(Self {
             chunk_offset: 0,
             cache: bytes,
             chunk_modified: changes,
+            hash_count,
         })
     }
 
@@ -36,12 +40,13 @@ impl<'a> TreeHashCache<'a> {
     }
 
     pub fn modify_current_chunk(&mut self, to: &[u8]) -> Option<()> {
-        self.modify_chunk(0, to)
+        self.modify_chunk(self.chunk_offset, to)
     }
 
     pub fn modify_chunk(&mut self, chunk: usize, to: &[u8]) -> Option<()> {
         let start = chunk * BYTES_PER_CHUNK;
         let end = start + BYTES_PER_CHUNK;
+
         self.cache.get_mut(start..end)?.copy_from_slice(to);
 
         self.chunk_modified[chunk] = true;
@@ -79,9 +84,10 @@ impl<'a> TreeHashCache<'a> {
         let modified_end = modified_start + leaves;
 
         Some(TreeHashCache {
-            chunk_offset: self.chunk_offset + internal,
+            chunk_offset: 0,
             cache: self.cache.get_mut(leaves_start..leaves_end)?,
             chunk_modified: self.chunk_modified.get_mut(modified_start..modified_end)?,
+            hash_count: self.hash_count,
         })
     }
 
@@ -111,7 +117,8 @@ impl CachedTreeHash for u64 {
 
     fn cached_hash_tree_root(&self, other: &Self, cache: &mut TreeHashCache) -> Option<()> {
         if self != other {
-            cache.modify_current_chunk(&merkleize(&int_to_bytes32(*self)));
+            *cache.hash_count += 1;
+            cache.modify_current_chunk(&merkleize(&int_to_bytes32(*self)))?;
         }
 
         cache.increment();
@@ -156,6 +163,7 @@ impl CachedTreeHash for Inner {
 
         for chunk in (0..internal_chunks).into_iter().rev() {
             if cache.children_modified(chunk)? {
+                *cache.hash_count += 1;
                 cache.modify_chunk(chunk, &cache.hash_children(chunk)?)?;
             }
         }
@@ -197,7 +205,7 @@ pub fn merkleize(values: &[u8]) -> Vec<u8> {
 mod tests {
     use super::*;
 
-    fn join(many: Vec<&[u8]>) -> Vec<u8> {
+    fn join(many: Vec<Vec<u8>>) -> Vec<u8> {
         let mut all = vec![];
         for one in many {
             all.extend_from_slice(&mut one.clone())
@@ -205,8 +213,7 @@ mod tests {
         all
     }
 
-    #[test]
-    fn cached_hash_on_inner() {
+    fn generic_test(index: usize) {
         let inner = Inner {
             a: 1,
             b: 2,
@@ -216,37 +223,69 @@ mod tests {
 
         let mut cache = inner.build_cache_bytes();
 
-        let changed_inner = Inner {
-            a: 42,
-            ..inner.clone()
+        let changed_inner = match index {
+            0 => Inner {
+                a: 42,
+                ..inner.clone()
+            },
+            1 => Inner {
+                b: 42,
+                ..inner.clone()
+            },
+            2 => Inner {
+                c: 42,
+                ..inner.clone()
+            },
+            3 => Inner {
+                d: 42,
+                ..inner.clone()
+            },
+            _ => panic!("bad index"),
         };
 
         let mut changes = TreeHashCache::build_changes_vec(&cache);
-        let mut cache_struct = TreeHashCache::from_mut_slice(&mut cache, &mut changes).unwrap();
+        let mut hash_count = 0;
+        let mut cache_struct =
+            TreeHashCache::from_mut_slice(&mut cache, &mut changes, &mut hash_count).unwrap();
 
-        changed_inner.cached_hash_tree_root(&inner, &mut cache_struct);
+        changed_inner
+            .cached_hash_tree_root(&inner, &mut cache_struct)
+            .unwrap();
+
+        assert_eq!(*cache_struct.hash_count, 3);
 
         let new_cache = cache_struct.into_slice();
 
-        let data1 = &int_to_bytes32(42);
-        let data2 = &int_to_bytes32(2);
-        let data3 = &int_to_bytes32(3);
-        let data4 = &int_to_bytes32(4);
+        let data1 = int_to_bytes32(1);
+        let data2 = int_to_bytes32(2);
+        let data3 = int_to_bytes32(3);
+        let data4 = int_to_bytes32(4);
 
-        let data = join(vec![&data1, &data2, &data3, &data4]);
-        let expected = merkleize(&data);
+        let mut data = vec![data1, data2, data3, data4];
+
+        data[index] = int_to_bytes32(42);
+
+        let expected = merkleize(&join(data));
 
         assert_eq!(expected, new_cache);
     }
 
     #[test]
-    fn build_cache_matches_merkelize() {
-        let data1 = &int_to_bytes32(1);
-        let data2 = &int_to_bytes32(2);
-        let data3 = &int_to_bytes32(3);
-        let data4 = &int_to_bytes32(4);
+    fn cached_hash_on_inner() {
+        generic_test(0);
+        generic_test(1);
+        generic_test(2);
+        generic_test(3);
+    }
 
-        let data = join(vec![&data1, &data2, &data3, &data4]);
+    #[test]
+    fn build_cache_matches_merkelize() {
+        let data1 = int_to_bytes32(1);
+        let data2 = int_to_bytes32(2);
+        let data3 = int_to_bytes32(3);
+        let data4 = int_to_bytes32(4);
+
+        let data = join(vec![data1, data2, data3, data4]);
         let expected = merkleize(&data);
 
         let inner = Inner {
@@ -268,7 +307,12 @@ mod tests {
         let data3 = hash(&int_to_bytes32(3));
         let data4 = hash(&int_to_bytes32(4));
 
-        let data = join(vec![&data1, &data2, &data3, &data4]);
+        let data = join(vec![
+            data1.clone(),
+            data2.clone(),
+            data3.clone(),
+            data4.clone(),
+        ]);
 
         let cache = merkleize(&data);
 

From b05787207fb6cb79b7918071dc65e3ed140e018b Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Thu, 28 Mar 2019 09:33:44 +1100
Subject: [PATCH 007/137] Refactor CachedTreeHash into owned bytes

Instead of slices
---
 eth2/utils/ssz/src/cached_tree_hash.rs | 150 +++++++++++++------------
 1 file changed, 77 insertions(+), 73 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index 757bfa9f7..e72ff1ffd 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -6,43 +6,29 @@ const BYTES_PER_CHUNK: usize = 32;
 const HASHSIZE: usize = 32;
 const MERKLE_HASH_CHUNCK: usize = 2 * BYTES_PER_CHUNK;
 
-pub struct TreeHashCache<'a> {
-    chunk_offset: usize,
-    cache: &'a mut [u8],
-    chunk_modified: &'a mut [bool],
-    hash_count: &'a mut usize,
+pub struct TreeHashCache {
+    cache: Vec<u8>,
+    chunk_modified: Vec<bool>,
 }
 
-impl<'a> TreeHashCache<'a> {
-    pub fn build_changes_vec(bytes: &[u8]) -> Vec<bool> {
-        vec![false; bytes.len() / BYTES_PER_CHUNK]
+impl Into<Vec<u8>> for TreeHashCache {
+    fn into(self) -> Vec<u8> {
+        self.cache
     }
+}
 
-    pub fn from_mut_slice(
-        bytes: &'a mut [u8],
-        changes: &'a mut [bool],
-        hash_count: &'a mut usize,
-    ) -> Option<Self> {
+impl TreeHashCache {
+    pub fn from_bytes(bytes: Vec<u8>) -> Option<Self> {
         if bytes.len() % BYTES_PER_CHUNK > 0 {
             return None;
         }
 
         Some(Self {
-            chunk_offset: 0,
+            chunk_modified: vec![false; bytes.len() / BYTES_PER_CHUNK],
             cache: bytes,
-            chunk_modified: changes,
-            hash_count,
         })
     }
 
-    pub fn increment(&mut self) {
-        self.chunk_offset += 1
-    }
-
-    pub fn modify_current_chunk(&mut self, to: &[u8]) -> Option<()> {
-        self.modify_chunk(self.chunk_offset, to)
-    }
-
     pub fn modify_chunk(&mut self, chunk: usize, to: &[u8]) -> Option<()> {
         let start = chunk * BYTES_PER_CHUNK;
         let end = start + BYTES_PER_CHUNK;
@@ -72,28 +58,6 @@ impl<'a> TreeHashCache<'a> {
 
         Some(hash(&self.cache.get(start..end)?))
     }
-
-    pub fn just_the_leaves(&mut self, leaves: usize) -> Option<TreeHashCache> {
-        let nodes = num_nodes(leaves);
-        let internal = nodes - leaves;
-
-        let leaves_start = (self.chunk_offset + internal) * BYTES_PER_CHUNK;
-        let leaves_end = leaves_start + leaves * BYTES_PER_CHUNK;
-
-        let modified_start = self.chunk_offset + internal;
-        let modified_end = modified_start + leaves;
-
-        Some(TreeHashCache {
-            chunk_offset: 0,
-            cache: self.cache.get_mut(leaves_start..leaves_end)?,
-            chunk_modified: self.chunk_modified.get_mut(modified_start..modified_end)?,
-            hash_count: self.hash_count,
-        })
-    }
-
-    pub fn into_slice(self) -> &'a [u8] {
-        self.cache
-    }
 }
 
 fn children(parent: usize) -> (usize, usize) {
@@ -107,7 +71,16 @@ fn num_nodes(num_leaves: usize) -> usize {
 pub trait CachedTreeHash {
     fn build_cache_bytes(&self) -> Vec<u8>;
 
-    fn cached_hash_tree_root(&self, other: &Self, cache: &mut TreeHashCache) -> Option<()>;
+    fn num_bytes(&self) -> usize;
+
+    fn max_num_leaves(&self) -> usize;
+
+    fn cached_hash_tree_root(
+        &self,
+        other: &Self,
+        cache: &mut TreeHashCache,
+        chunk: usize,
+    ) -> Option<usize>;
 }
 
 impl CachedTreeHash for u64 {
@@ -115,15 +88,25 @@ impl CachedTreeHash for u64 {
         merkleize(&int_to_bytes32(*self))
     }
 
-    fn cached_hash_tree_root(&self, other: &Self, cache: &mut TreeHashCache) -> Option<()> {
+    fn num_bytes(&self) -> usize {
+        8
+    }
+
+    fn max_num_leaves(&self) -> usize {
+        1
+    }
+
+    fn cached_hash_tree_root(
+        &self,
+        other: &Self,
+        cache: &mut TreeHashCache,
+        chunk: usize,
+    ) -> Option<usize> {
         if self != other {
-            *cache.hash_count += 1;
-            cache.modify_current_chunk(&merkleize(&int_to_bytes32(*self)))?;
+            cache.modify_chunk(chunk, &merkleize(&int_to_bytes32(*self)))?;
         }
 
-        cache.increment();
-
-        Some(())
+        Some(chunk + 1)
     }
 }
 
@@ -147,28 +130,52 @@ impl CachedTreeHash for Inner {
         merkleize(&leaves)
     }
 
-    fn cached_hash_tree_root(&self, other: &Self, cache: &mut TreeHashCache) -> Option<()> {
-        let num_leaves = 4;
+    fn max_num_leaves(&self) -> usize {
+        let mut leaves = 0;
+        leaves += self.a.max_num_leaves();
+        leaves += self.b.max_num_leaves();
+        leaves += self.c.max_num_leaves();
+        leaves += self.d.max_num_leaves();
+        leaves
+    }
 
+    fn num_bytes(&self) -> usize {
+        let mut bytes = 0;
+        bytes += self.a.num_bytes();
+        bytes += self.b.num_bytes();
+        bytes += self.c.num_bytes();
+        bytes += self.d.num_bytes();
+        bytes
+    }
+
+    fn cached_hash_tree_root(
+        &self,
+        other: &Self,
+        cache: &mut TreeHashCache,
+        chunk: usize,
+    ) -> Option<usize> {
+        let num_leaves = self.max_num_leaves();
+        let num_nodes = num_nodes(num_leaves);
+        let num_internal_nodes = num_nodes - num_leaves;
+
+        // Skip past the internal nodes and update any changed leaf nodes.
         {
-            let mut leaf_cache = cache.just_the_leaves(num_leaves)?;
-            self.a.cached_hash_tree_root(&other.a, &mut leaf_cache)?;
-            self.b.cached_hash_tree_root(&other.b, &mut leaf_cache)?;
-            self.c.cached_hash_tree_root(&other.c, &mut leaf_cache)?;
-            self.d.cached_hash_tree_root(&other.d, &mut leaf_cache)?;
+            let chunk = chunk + num_internal_nodes;
+            let chunk = self.a.cached_hash_tree_root(&other.a, cache, chunk)?;
+            let chunk = self.b.cached_hash_tree_root(&other.b, cache, chunk)?;
+            let chunk = self.c.cached_hash_tree_root(&other.c, cache, chunk)?;
+            let _chunk = self.d.cached_hash_tree_root(&other.d, cache, chunk)?;
         }
 
-        let nodes = num_nodes(num_leaves);
-        let internal_chunks = nodes - num_leaves;
-
-        for chunk in (0..internal_chunks).into_iter().rev() {
+        // Iterate backwards through the internal nodes, rehashing any node where it's children
+        // have changed.
+        for chunk in (0..num_internal_nodes).into_iter().rev() {
             if cache.children_modified(chunk)? {
-                *cache.hash_count += 1;
                 cache.modify_chunk(chunk, &cache.hash_children(chunk)?)?;
             }
         }
 
-        Some(())
+        Some(chunk + num_nodes)
     }
 }
 
@@ -243,18 +250,15 @@ mod tests {
             _ => panic!("bad index"),
         };
 
-        let mut changes = TreeHashCache::build_changes_vec(&cache);
-        let mut hash_count = 0;
-        let mut cache_struct =
-            TreeHashCache::from_mut_slice(&mut cache, &mut changes, &mut hash_count).unwrap();
+        let mut cache_struct = TreeHashCache::from_bytes(cache.clone()).unwrap();
 
         changed_inner
-            .cached_hash_tree_root(&inner, &mut cache_struct)
+            .cached_hash_tree_root(&inner, &mut cache_struct, 0)
             .unwrap();
 
-        assert_eq!(*cache_struct.hash_count, 3);
+        // assert_eq!(*cache_struct.hash_count, 3);
 
-        let new_cache = cache_struct.into_slice();
+        let new_cache: Vec<u8> = cache_struct.into();
 
         let data1 = int_to_bytes32(1);
         let data2 = int_to_bytes32(2);

From 3c7e18bdf3544bc049634aeca936ea42983b2636 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Thu, 28 Mar 2019 10:56:20 +1100
Subject: [PATCH 008/137] Sanitize for odd leaf count

---
 eth2/utils/ssz/src/cached_tree_hash.rs | 38 +++++++++++++++++++++++++-
 1 file changed, 37 insertions(+), 1 deletion(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index e72ff1ffd..525f35db3 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -179,6 +179,29 @@ impl CachedTreeHash for Inner {
     }
 }
 
+fn last_leaf_needs_padding(num_bytes: usize) -> bool {
+    num_bytes % HASHSIZE != 0
+}
+
+fn num_leaves(num_bytes: usize) -> usize {
+    num_bytes / HASHSIZE
+}
+
+fn num_bytes(num_leaves: usize) -> usize {
+    num_leaves * HASHSIZE
+}
+
+pub fn sanitise_bytes(mut bytes: Vec<u8>) -> Vec<u8> {
+    let present_leaves = num_leaves(bytes.len());
+    let required_leaves = present_leaves.next_power_of_two();
+
+    if (present_leaves != required_leaves) | last_leaf_needs_padding(bytes.len()) {
+        bytes.resize(num_bytes(required_leaves), 0);
+    }
+
+    bytes
+}
+
 /// A reference function to test against.
 pub fn merkleize(values: &[u8]) -> Vec<u8> {
     let leaves = values.len() / HASHSIZE;
@@ -220,6 +243,19 @@ mod tests {
         all
     }
 
+    #[test]
+    fn merkleize_odd() {
+        let data = join(vec![
+            int_to_bytes32(1),
+            int_to_bytes32(2),
+            int_to_bytes32(3),
+            int_to_bytes32(4),
+            int_to_bytes32(5),
+        ]);
+
+        merkleize(&sanitise_bytes(data));
+    }
+
     fn generic_test(index: usize) {
         let inner = Inner {
             a: 1,
@@ -228,7 +264,7 @@ mod tests {
             d: 4,
         };
 
-        let mut cache = inner.build_cache_bytes();
+        let cache = inner.build_cache_bytes();
 
         let changed_inner = match index {
             0 => Inner {

From 1285f1e9f8b6efd4fee83e6a9acbc3cc46c86a48 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Thu, 28 Mar 2019 11:11:20 +1100
Subject: [PATCH 009/137] Restructure cached tree hash files, breaks tests

---
 eth2/utils/ssz/src/cached_tree_hash.rs       | 335 +++----------------
 eth2/utils/ssz/src/cached_tree_hash/impls.rs |  30 ++
 eth2/utils/ssz/src/cached_tree_hash/tests.rs | 227 +++++++++++++
 3 files changed, 299 insertions(+), 293 deletions(-)
 create mode 100644 eth2/utils/ssz/src/cached_tree_hash/impls.rs
 create mode 100644 eth2/utils/ssz/src/cached_tree_hash/tests.rs

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index 525f35db3..6535e5cda 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -1,11 +1,27 @@
-use crate::ssz_encode;
 use hashing::hash;
-use int_to_bytes::int_to_bytes32;
+
+mod impls;
+mod tests;
 
 const BYTES_PER_CHUNK: usize = 32;
 const HASHSIZE: usize = 32;
 const MERKLE_HASH_CHUNCK: usize = 2 * BYTES_PER_CHUNK;
 
+pub trait CachedTreeHash {
+    fn build_cache_bytes(&self) -> Vec<u8>;
+
+    fn num_bytes(&self) -> usize;
+
+    fn max_num_leaves(&self) -> usize;
+
+    fn cached_hash_tree_root(
+        &self,
+        other: &Self,
+        cache: &mut TreeHashCache,
+        chunk: usize,
+    ) -> Option<usize>;
+}
+
 pub struct TreeHashCache {
     cache: Vec<u8>,
     chunk_modified: Vec<bool>,
@@ -68,142 +84,13 @@ fn num_nodes(num_leaves: usize) -> usize {
     2 * num_leaves - 1
 }
 
-pub trait CachedTreeHash {
-    fn build_cache_bytes(&self) -> Vec<u8>;
+/// Split `values` into a power-of-two, identical-length chunks (padding with `0`) and merkleize
+/// them, returning the entire merkle tree.
+///
+/// The root hash is `merkleize(values)[0..BYTES_PER_CHUNK]`.
+pub fn merkleize(values: Vec<u8>) -> Vec<u8> {
+    let values = sanitise_bytes(values);
 
-    fn num_bytes(&self) -> usize;
-
-    fn max_num_leaves(&self) -> usize;
-
-    fn cached_hash_tree_root(
-        &self,
-        other: &Self,
-        cache: &mut TreeHashCache,
-        chunk: usize,
-    ) -> Option<usize>;
-}
-
-impl CachedTreeHash for u64 {
-    fn build_cache_bytes(&self) -> Vec<u8> {
-        merkleize(&int_to_bytes32(*self))
-    }
-
-    fn num_bytes(&self) -> usize {
-        8
-    }
-
-    fn max_num_leaves(&self) -> usize {
-        1
-    }
-
-    fn cached_hash_tree_root(
-        &self,
-        other: &Self,
-        cache: &mut TreeHashCache,
-        chunk: usize,
-    ) -> Option<usize> {
-        if self != other {
-            cache.modify_chunk(chunk, &merkleize(&int_to_bytes32(*self)))?;
-        }
-
-        Some(chunk + 1)
-    }
-}
-
-#[derive(Clone)]
-pub struct Inner {
-    pub a: u64,
-    pub b: u64,
-    pub c: u64,
-    pub d: u64,
-}
-
-impl CachedTreeHash for Inner {
-    fn build_cache_bytes(&self) -> Vec<u8> {
-        let mut leaves = vec![];
-
-        leaves.append(&mut self.a.build_cache_bytes());
-        leaves.append(&mut self.b.build_cache_bytes());
-        leaves.append(&mut self.c.build_cache_bytes());
-        leaves.append(&mut self.d.build_cache_bytes());
-
-        merkleize(&leaves)
-    }
-
-    fn max_num_leaves(&self) -> usize {
-        let mut leaves = 0;
-        leaves += self.a.max_num_leaves();
-        leaves += self.b.max_num_leaves();
-        leaves += self.c.max_num_leaves();
-        leaves += self.d.max_num_leaves();
-        leaves
-    }
-
-    fn num_bytes(&self) -> usize {
-        let mut bytes = 0;
-        bytes += self.a.num_bytes();
-        bytes += self.b.num_bytes();
-        bytes += self.c.num_bytes();
-        bytes += self.d.num_bytes();
-        bytes
-    }
-
-    fn cached_hash_tree_root(
-        &self,
-        other: &Self,
-        cache: &mut TreeHashCache,
-        chunk: usize,
-    ) -> Option<usize> {
-        let num_leaves = self.max_num_leaves();
-        let num_nodes = num_nodes(num_leaves);
-        let num_internal_nodes = num_nodes - num_leaves;
-
-        // Skip past the internal nodes and update any changed leaf nodes.
-        {
-            let chunk = chunk + num_internal_nodes;
-            let chunk = self.a.cached_hash_tree_root(&other.a, cache, chunk)?;
-            let chunk = self.b.cached_hash_tree_root(&other.b, cache, chunk)?;
-            let chunk = self.c.cached_hash_tree_root(&other.c, cache, chunk)?;
-            let _chunk = self.d.cached_hash_tree_root(&other.d, cache, chunk)?;
-        }
-
-        // Iterate backwards through the internal nodes, rehashing any node where it's children
-        // have changed.
-        for chunk in (0..num_internal_nodes).into_iter().rev() {
-            if cache.children_modified(chunk)? {
-                cache.modify_chunk(chunk, &cache.hash_children(chunk)?)?;
-            }
-        }
-
-        Some(chunk + num_nodes)
-    }
-}
-
-fn last_leaf_needs_padding(num_bytes: usize) -> bool {
-    num_bytes % HASHSIZE != 0
-}
-
-fn num_leaves(num_bytes: usize) -> usize {
-    num_bytes / HASHSIZE
-}
-
-fn num_bytes(num_leaves: usize) -> usize {
-    num_leaves * HASHSIZE
-}
-
-pub fn sanitise_bytes(mut bytes: Vec<u8>) -> Vec<u8> {
-    let present_leaves = num_leaves(bytes.len());
-    let required_leaves = present_leaves.next_power_of_two();
-
-    if (present_leaves != required_leaves) | last_leaf_needs_padding(bytes.len()) {
-        bytes.resize(num_bytes(required_leaves), 0);
-    }
-
-    bytes
-}
-
-/// A reference function to test against.
-pub fn merkleize(values: &[u8]) -> Vec<u8> {
     let leaves = values.len() / HASHSIZE;
 
     if leaves == 0 {
@@ -231,163 +118,25 @@ pub fn merkleize(values: &[u8]) -> Vec<u8> {
     o
 }
 
-#[cfg(test)]
-mod tests {
-    use super::*;
+pub fn sanitise_bytes(mut bytes: Vec<u8>) -> Vec<u8> {
+    let present_leaves = num_leaves(bytes.len());
+    let required_leaves = present_leaves.next_power_of_two();
 
-    fn join(many: Vec<Vec<u8>>) -> Vec<u8> {
-        let mut all = vec![];
-        for one in many {
-            all.extend_from_slice(&mut one.clone())
-        }
-        all
+    if (present_leaves != required_leaves) | last_leaf_needs_padding(bytes.len()) {
+        bytes.resize(num_bytes(required_leaves), 0);
     }
 
-    #[test]
-    fn merkleize_odd() {
-        let data = join(vec![
-            int_to_bytes32(1),
-            int_to_bytes32(2),
-            int_to_bytes32(3),
-            int_to_bytes32(4),
-            int_to_bytes32(5),
-        ]);
-
-        merkleize(&sanitise_bytes(data));
-    }
-
-    fn generic_test(index: usize) {
-        let inner = Inner {
-            a: 1,
-            b: 2,
-            c: 3,
-            d: 4,
-        };
-
-        let cache = inner.build_cache_bytes();
-
-        let changed_inner = match index {
-            0 => Inner {
-                a: 42,
-                ..inner.clone()
-            },
-            1 => Inner {
-                b: 42,
-                ..inner.clone()
-            },
-            2 => Inner {
-                c: 42,
-                ..inner.clone()
-            },
-            3 => Inner {
-                d: 42,
-                ..inner.clone()
-            },
-            _ => panic!("bad index"),
-        };
-
-        let mut cache_struct = TreeHashCache::from_bytes(cache.clone()).unwrap();
-
-        changed_inner
-            .cached_hash_tree_root(&inner, &mut cache_struct, 0)
-            .unwrap();
-
-        // assert_eq!(*cache_struct.hash_count, 3);
-
-        let new_cache: Vec<u8> = cache_struct.into();
-
-        let data1 = int_to_bytes32(1);
-        let data2 = int_to_bytes32(2);
-        let data3 = int_to_bytes32(3);
-        let data4 = int_to_bytes32(4);
-
-        let mut data = vec![data1, data2, data3, data4];
-
-        data[index] = int_to_bytes32(42);
-
-        let expected = merkleize(&join(data));
-
-        assert_eq!(expected, new_cache);
-    }
-
-    #[test]
-    fn cached_hash_on_inner() {
-        generic_test(0);
-        generic_test(1);
-        generic_test(2);
-        generic_test(3);
-    }
-
-    #[test]
-    fn build_cache_matches_merkelize() {
-        let data1 = int_to_bytes32(1);
-        let data2 = int_to_bytes32(2);
-        let data3 = int_to_bytes32(3);
-        let data4 = int_to_bytes32(4);
-
-        let data = join(vec![data1, data2, data3, data4]);
-        let expected = merkleize(&data);
-
-        let inner = Inner {
-            a: 1,
-            b: 2,
-            c: 3,
-            d: 4,
-        };
-
-        let cache = inner.build_cache_bytes();
-
-        assert_eq!(expected, cache);
-    }
-
-    #[test]
-    fn merkleize_4_leaves() {
-        let data1 = hash(&int_to_bytes32(1));
-        let data2 = hash(&int_to_bytes32(2));
-        let data3 = hash(&int_to_bytes32(3));
-        let data4 = hash(&int_to_bytes32(4));
-
-        let data = join(vec![
-            data1.clone(),
-            data2.clone(),
-            data3.clone(),
-            data4.clone(),
-        ]);
-
-        let cache = merkleize(&data);
-
-        let hash_12 = {
-            let mut joined = vec![];
-            joined.append(&mut data1.clone());
-            joined.append(&mut data2.clone());
-            hash(&joined)
-        };
-        let hash_34 = {
-            let mut joined = vec![];
-            joined.append(&mut data3.clone());
-            joined.append(&mut data4.clone());
-            hash(&joined)
-        };
-        let hash_hash12_hash_34 = {
-            let mut joined = vec![];
-            joined.append(&mut hash_12.clone());
-            joined.append(&mut hash_34.clone());
-            hash(&joined)
-        };
-
-        for (i, chunk) in cache.chunks(HASHSIZE).enumerate().rev() {
-            let expected = match i {
-                0 => hash_hash12_hash_34.clone(),
-                1 => hash_12.clone(),
-                2 => hash_34.clone(),
-                3 => data1.clone(),
-                4 => data2.clone(),
-                5 => data3.clone(),
-                6 => data4.clone(),
-                _ => vec![],
-            };
-
-            assert_eq!(chunk, &expected[..], "failed at {}", i);
-        }
-    }
+    bytes
+}
+
+fn last_leaf_needs_padding(num_bytes: usize) -> bool {
+    num_bytes % HASHSIZE != 0
+}
+
+fn num_leaves(num_bytes: usize) -> usize {
+    num_bytes / HASHSIZE
+}
+
+fn num_bytes(num_leaves: usize) -> usize {
+    num_leaves * HASHSIZE
 }
diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
new file mode 100644
index 000000000..b6b0d463a
--- /dev/null
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -0,0 +1,30 @@
+use super::*;
+use crate::ssz_encode;
+
+impl CachedTreeHash for u64 {
+    fn build_cache_bytes(&self) -> Vec<u8> {
+        merkleize(ssz_encode(self))
+    }
+
+    fn num_bytes(&self) -> usize {
+        8
+    }
+
+    fn max_num_leaves(&self) -> usize {
+        1
+    }
+
+    fn cached_hash_tree_root(
+        &self,
+        other: &Self,
+        cache: &mut TreeHashCache,
+        chunk: usize,
+    ) -> Option<usize> {
+        if self != other {
+            let leaf = merkleize(ssz_encode(self));
+            cache.modify_chunk(chunk, &leaf)?;
+        }
+
+        Some(chunk + 1)
+    }
+}
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
new file mode 100644
index 000000000..79665f89d
--- /dev/null
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -0,0 +1,227 @@
+use super::*;
+use int_to_bytes::int_to_bytes32;
+
+#[derive(Clone)]
+pub struct Inner {
+    pub a: u64,
+    pub b: u64,
+    pub c: u64,
+    pub d: u64,
+}
+
+impl CachedTreeHash for Inner {
+    fn build_cache_bytes(&self) -> Vec<u8> {
+        let mut leaves = vec![];
+
+        leaves.append(&mut self.a.build_cache_bytes());
+        leaves.append(&mut self.b.build_cache_bytes());
+        leaves.append(&mut self.c.build_cache_bytes());
+        leaves.append(&mut self.d.build_cache_bytes());
+
+        merkleize(leaves)
+    }
+
+    fn max_num_leaves(&self) -> usize {
+        let mut leaves = 0;
+        leaves += self.a.max_num_leaves();
+        leaves += self.b.max_num_leaves();
+        leaves += self.c.max_num_leaves();
+        leaves += self.d.max_num_leaves();
+        leaves
+    }
+
+    fn num_bytes(&self) -> usize {
+        let mut bytes = 0;
+        bytes += self.a.num_bytes();
+        bytes += self.b.num_bytes();
+        bytes += self.c.num_bytes();
+        bytes += self.d.num_bytes();
+        bytes
+    }
+
+    fn cached_hash_tree_root(
+        &self,
+        other: &Self,
+        cache: &mut TreeHashCache,
+        chunk: usize,
+    ) -> Option<usize> {
+        let num_leaves = self.max_num_leaves();
+        let num_nodes = num_nodes(num_leaves);
+        let num_internal_nodes = num_nodes - num_leaves;
+
+        // Skip past the internal nodes and update any changed leaf nodes.
+        {
+            let chunk = chunk + num_internal_nodes;
+            let chunk = self.a.cached_hash_tree_root(&other.a, cache, chunk)?;
+            let chunk = self.b.cached_hash_tree_root(&other.b, cache, chunk)?;
+            let chunk = self.c.cached_hash_tree_root(&other.c, cache, chunk)?;
+            let _chunk = self.d.cached_hash_tree_root(&other.d, cache, chunk)?;
+        }
+
+        // Iterate backwards through the internal nodes, rehashing any node where it's children
+        // have changed.
+        for chunk in (0..num_internal_nodes).into_iter().rev() {
+            if cache.children_modified(chunk)? {
+                cache.modify_chunk(chunk, &cache.hash_children(chunk)?)?;
+            }
+        }
+
+        Some(chunk + num_nodes)
+    }
+}
+
+fn join(many: Vec<Vec<u8>>) -> Vec<u8> {
+    let mut all = vec![];
+    for one in many {
+        all.extend_from_slice(&mut one.clone())
+    }
+    all
+}
+
+#[test]
+fn merkleize_odd() {
+    let data = join(vec![
+        int_to_bytes32(1),
+        int_to_bytes32(2),
+        int_to_bytes32(3),
+        int_to_bytes32(4),
+        int_to_bytes32(5),
+    ]);
+
+    merkleize(sanitise_bytes(data));
+}
+
+fn generic_test(index: usize) {
+    let inner = Inner {
+        a: 1,
+        b: 2,
+        c: 3,
+        d: 4,
+    };
+
+    let cache = inner.build_cache_bytes();
+
+    let changed_inner = match index {
+        0 => Inner {
+            a: 42,
+            ..inner.clone()
+        },
+        1 => Inner {
+            b: 42,
+            ..inner.clone()
+        },
+        2 => Inner {
+            c: 42,
+            ..inner.clone()
+        },
+        3 => Inner {
+            d: 42,
+            ..inner.clone()
+        },
+        _ => panic!("bad index"),
+    };
+
+    let mut cache_struct = TreeHashCache::from_bytes(cache.clone()).unwrap();
+
+    changed_inner
+        .cached_hash_tree_root(&inner, &mut cache_struct, 0)
+        .unwrap();
+
+    // assert_eq!(*cache_struct.hash_count, 3);
+
+    let new_cache: Vec<u8> = cache_struct.into();
+
+    let data1 = int_to_bytes32(1);
+    let data2 = int_to_bytes32(2);
+    let data3 = int_to_bytes32(3);
+    let data4 = int_to_bytes32(4);
+
+    let mut data = vec![data1, data2, data3, data4];
+
+    data[index] = int_to_bytes32(42);
+
+    let expected = merkleize(join(data));
+
+    assert_eq!(expected, new_cache);
+}
+
+#[test]
+fn cached_hash_on_inner() {
+    generic_test(0);
+    generic_test(1);
+    generic_test(2);
+    generic_test(3);
+}
+
+#[test]
+fn build_cache_matches_merkelize() {
+    let data1 = int_to_bytes32(1);
+    let data2 = int_to_bytes32(2);
+    let data3 = int_to_bytes32(3);
+    let data4 = int_to_bytes32(4);
+
+    let data = join(vec![data1, data2, data3, data4]);
+    let expected = merkleize(data);
+
+    let inner = Inner {
+        a: 1,
+        b: 2,
+        c: 3,
+        d: 4,
+    };
+
+    let cache = inner.build_cache_bytes();
+
+    assert_eq!(expected, cache);
+}
+
+#[test]
+fn merkleize_4_leaves() {
+    let data1 = hash(&int_to_bytes32(1));
+    let data2 = hash(&int_to_bytes32(2));
+    let data3 = hash(&int_to_bytes32(3));
+    let data4 = hash(&int_to_bytes32(4));
+
+    let data = join(vec![
+        data1.clone(),
+        data2.clone(),
+        data3.clone(),
+        data4.clone(),
+    ]);
+
+    let cache = merkleize(data);
+
+    let hash_12 = {
+        let mut joined = vec![];
+        joined.append(&mut data1.clone());
+        joined.append(&mut data2.clone());
+        hash(&joined)
+    };
+    let hash_34 = {
+        let mut joined = vec![];
+        joined.append(&mut data3.clone());
+        joined.append(&mut data4.clone());
+        hash(&joined)
+    };
+    let hash_hash12_hash_34 = {
+        let mut joined = vec![];
+        joined.append(&mut hash_12.clone());
+        joined.append(&mut hash_34.clone());
+        hash(&joined)
+    };
+
+    for (i, chunk) in cache.chunks(HASHSIZE).enumerate().rev() {
+        let expected = match i {
+            0 => hash_hash12_hash_34.clone(),
+            1 => hash_12.clone(),
+            2 => hash_34.clone(),
+            3 => data1.clone(),
+            4 => data2.clone(),
+            5 => data3.clone(),
+            6 => data4.clone(),
+            _ => vec![],
+        };
+
+        assert_eq!(chunk, &expected[..], "failed at {}", i);
+    }
+}

From 224a967cce660866e6dd379787871eac1d2f532c Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Thu, 28 Mar 2019 13:05:24 +1100
Subject: [PATCH 010/137] Implement basic vec tree hash cache

---
 eth2/utils/ssz/src/cached_tree_hash.rs       | 40 +++++++++--
 eth2/utils/ssz/src/cached_tree_hash/impls.rs | 70 ++++++++++++++++++--
 eth2/utils/ssz/src/cached_tree_hash/tests.rs | 40 ++++++++---
 3 files changed, 128 insertions(+), 22 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index 6535e5cda..caafaa2cf 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -8,15 +8,17 @@ const HASHSIZE: usize = 32;
 const MERKLE_HASH_CHUNCK: usize = 2 * BYTES_PER_CHUNK;
 
 pub trait CachedTreeHash {
+    type Item: CachedTreeHash;
+
     fn build_cache_bytes(&self) -> Vec<u8>;
 
+    /// Return the number of bytes when this element is encoded as raw SSZ _without_ length
+    /// prefixes.
     fn num_bytes(&self) -> usize;
 
-    fn max_num_leaves(&self) -> usize;
-
     fn cached_hash_tree_root(
         &self,
-        other: &Self,
+        other: &Self::Item,
         cache: &mut TreeHashCache,
         chunk: usize,
     ) -> Option<usize>;
@@ -45,6 +47,18 @@ impl TreeHashCache {
         })
     }
 
+    pub fn maybe_update_chunk(&mut self, chunk: usize, to: &[u8]) -> Option<()> {
+        let start = chunk * BYTES_PER_CHUNK;
+        let end = start + BYTES_PER_CHUNK;
+
+        if !self.chunk_equals(chunk, to)? {
+            self.cache.get_mut(start..end)?.copy_from_slice(to);
+            self.chunk_modified[chunk] = true;
+        }
+
+        Some(())
+    }
+
     pub fn modify_chunk(&mut self, chunk: usize, to: &[u8]) -> Option<()> {
         let start = chunk * BYTES_PER_CHUNK;
         let end = start + BYTES_PER_CHUNK;
@@ -56,6 +70,13 @@ impl TreeHashCache {
         Some(())
     }
 
+    pub fn chunk_equals(&mut self, chunk: usize, other: &[u8]) -> Option<bool> {
+        let start = chunk * BYTES_PER_CHUNK;
+        let end = start + BYTES_PER_CHUNK;
+
+        Some(self.cache.get(start..end)? == other)
+    }
+
     pub fn changed(&self, chunk: usize) -> Option<bool> {
         self.chunk_modified.get(chunk).cloned()
     }
@@ -119,7 +140,7 @@ pub fn merkleize(values: Vec<u8>) -> Vec<u8> {
 }
 
 pub fn sanitise_bytes(mut bytes: Vec<u8>) -> Vec<u8> {
-    let present_leaves = num_leaves(bytes.len());
+    let present_leaves = num_unsanitized_leaves(bytes.len());
     let required_leaves = present_leaves.next_power_of_two();
 
     if (present_leaves != required_leaves) | last_leaf_needs_padding(bytes.len()) {
@@ -133,8 +154,15 @@ fn last_leaf_needs_padding(num_bytes: usize) -> bool {
     num_bytes % HASHSIZE != 0
 }
 
-fn num_leaves(num_bytes: usize) -> usize {
-    num_bytes / HASHSIZE
+/// Rounds up
+fn num_unsanitized_leaves(num_bytes: usize) -> usize {
+    (num_bytes + HASHSIZE - 1) / HASHSIZE
+}
+
+/// Rounds up
+fn num_sanitized_leaves(num_bytes: usize) -> usize {
+    let leaves = (num_bytes + HASHSIZE - 1) / HASHSIZE;
+    leaves.next_power_of_two()
 }
 
 fn num_bytes(num_leaves: usize) -> usize {
diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index b6b0d463a..b27d28c4b 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -1,7 +1,9 @@
 use super::*;
-use crate::ssz_encode;
+use crate::{ssz_encode, Encodable};
 
 impl CachedTreeHash for u64 {
+    type Item = Self;
+
     fn build_cache_bytes(&self) -> Vec<u8> {
         merkleize(ssz_encode(self))
     }
@@ -10,10 +12,6 @@ impl CachedTreeHash for u64 {
         8
     }
 
-    fn max_num_leaves(&self) -> usize {
-        1
-    }
-
     fn cached_hash_tree_root(
         &self,
         other: &Self,
@@ -28,3 +26,65 @@ impl CachedTreeHash for u64 {
         Some(chunk + 1)
     }
 }
+
+impl<T> CachedTreeHash for Vec<T>
+where
+    T: CachedTreeHash + Encodable,
+{
+    type Item = Self;
+
+    fn build_cache_bytes(&self) -> Vec<u8> {
+        let num_packed_bytes = self.num_bytes();
+        let num_leaves = num_sanitized_leaves(num_packed_bytes);
+
+        let mut packed = Vec::with_capacity(num_leaves * HASHSIZE);
+
+        for item in self {
+            packed.append(&mut ssz_encode(item));
+        }
+
+        let packed = sanitise_bytes(packed);
+
+        merkleize(packed)
+    }
+
+    fn num_bytes(&self) -> usize {
+        self.iter().fold(0, |acc, item| acc + item.num_bytes())
+    }
+
+    fn cached_hash_tree_root(
+        &self,
+        other: &Self::Item,
+        cache: &mut TreeHashCache,
+        chunk: usize,
+    ) -> Option<usize> {
+        let num_packed_bytes = self.num_bytes();
+        let num_leaves = num_sanitized_leaves(num_packed_bytes);
+
+        if num_leaves != num_sanitized_leaves(other.num_bytes()) {
+            panic!("Need to handle a change in leaf count");
+        }
+
+        let mut packed = Vec::with_capacity(num_leaves * HASHSIZE);
+
+        // TODO: try and avoid fully encoding the whole list
+        for item in self {
+            packed.append(&mut ssz_encode(item));
+        }
+
+        let packed = sanitise_bytes(packed);
+
+        let num_nodes = num_nodes(num_leaves);
+        let num_internal_nodes = num_nodes - num_leaves;
+
+        {
+            let mut chunk = chunk + num_internal_nodes;
+            for new_chunk_bytes in packed.chunks(HASHSIZE) {
+                cache.maybe_update_chunk(chunk, new_chunk_bytes)?;
+                chunk += 1;
+            }
+        }
+
+        Some(chunk + num_nodes)
+    }
+}
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index 79665f89d..f4a4b1d46 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -1,5 +1,5 @@
 use super::*;
-use int_to_bytes::int_to_bytes32;
+use int_to_bytes::{int_to_bytes32, int_to_bytes8};
 
 #[derive(Clone)]
 pub struct Inner {
@@ -10,6 +10,8 @@ pub struct Inner {
 }
 
 impl CachedTreeHash for Inner {
+    type Item = Self;
+
     fn build_cache_bytes(&self) -> Vec<u8> {
         let mut leaves = vec![];
 
@@ -21,15 +23,6 @@ impl CachedTreeHash for Inner {
         merkleize(leaves)
     }
 
-    fn max_num_leaves(&self) -> usize {
-        let mut leaves = 0;
-        leaves += self.a.max_num_leaves();
-        leaves += self.b.max_num_leaves();
-        leaves += self.c.max_num_leaves();
-        leaves += self.d.max_num_leaves();
-        leaves
-    }
-
     fn num_bytes(&self) -> usize {
         let mut bytes = 0;
         bytes += self.a.num_bytes();
@@ -45,7 +38,12 @@ impl CachedTreeHash for Inner {
         cache: &mut TreeHashCache,
         chunk: usize,
     ) -> Option<usize> {
-        let num_leaves = self.max_num_leaves();
+        let mut num_leaves: usize = 0;
+        num_leaves += num_unsanitized_leaves(self.a.num_bytes());
+        num_leaves += num_unsanitized_leaves(self.b.num_bytes());
+        num_leaves += num_unsanitized_leaves(self.c.num_bytes());
+        num_leaves += num_unsanitized_leaves(self.d.num_bytes());
+
         let num_nodes = num_nodes(num_leaves);
         let num_internal_nodes = num_nodes - num_leaves;
 
@@ -78,6 +76,26 @@ fn join(many: Vec<Vec<u8>>) -> Vec<u8> {
     all
 }
 
+#[test]
+fn vec_of_u64() {
+    let data = join(vec![
+        int_to_bytes8(1),
+        int_to_bytes8(2),
+        int_to_bytes8(3),
+        int_to_bytes8(4),
+        int_to_bytes8(5),
+        vec![0; 32 - 8], // padding
+    ]);
+
+    let expected = merkleize(data);
+
+    let my_vec = vec![1, 2, 3, 4, 5];
+
+    let cache = my_vec.build_cache_bytes();
+
+    assert_eq!(expected, cache);
+}
+
 #[test]
 fn merkleize_odd() {
     let data = join(vec![

From 0d8d3385bef62d4bcb2c094fd7b6ad9d7b6e0801 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Thu, 28 Mar 2019 14:17:25 +1100
Subject: [PATCH 011/137] Pass tree hash caching tests

---
 eth2/utils/ssz/src/cached_tree_hash/impls.rs |  8 +++
 eth2/utils/ssz/src/cached_tree_hash/tests.rs | 53 +++++++++++++++++++-
 2 files changed, 59 insertions(+), 2 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index b27d28c4b..1c2bf342e 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -85,6 +85,14 @@ where
             }
         }
 
+        // Iterate backwards through the internal nodes, rehashing any node where it's children
+        // have changed.
+        for chunk in (chunk..chunk + num_internal_nodes).into_iter().rev() {
+            if cache.children_modified(chunk)? {
+                cache.modify_chunk(chunk, &cache.hash_children(chunk)?)?;
+            }
+        }
+
         Some(chunk + num_nodes)
     }
 }
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index f4a4b1d46..6c7567250 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -58,7 +58,7 @@ impl CachedTreeHash for Inner {
 
         // Iterate backwards through the internal nodes, rehashing any node where it's children
         // have changed.
-        for chunk in (0..num_internal_nodes).into_iter().rev() {
+        for chunk in (chunk..chunk + num_internal_nodes).into_iter().rev() {
             if cache.children_modified(chunk)? {
                 cache.modify_chunk(chunk, &cache.hash_children(chunk)?)?;
             }
@@ -77,7 +77,56 @@ fn join(many: Vec<Vec<u8>>) -> Vec<u8> {
 }
 
 #[test]
-fn vec_of_u64() {
+fn partial_modification_u64_vec() {
+    let n: u64 = 50;
+
+    let original_vec: Vec<u64> = (0..n).collect();
+
+    // Generate initial cache.
+    let original_cache = original_vec.build_cache_bytes();
+
+    // Modify the vec
+    let mut modified_vec = original_vec.clone();
+    modified_vec[n as usize - 1] = 42;
+
+    // Perform a differential hash
+    let mut cache_struct = TreeHashCache::from_bytes(original_cache.clone()).unwrap();
+    modified_vec.cached_hash_tree_root(&original_vec, &mut cache_struct, 0);
+    let modified_cache: Vec<u8> = cache_struct.into();
+
+    // Generate reference data.
+    let mut data = vec![];
+    for i in &modified_vec {
+        data.append(&mut int_to_bytes8(*i));
+    }
+    let data = sanitise_bytes(data);
+    let expected = merkleize(data);
+
+    assert_eq!(expected, modified_cache);
+}
+
+#[test]
+fn large_vec_of_u64_builds() {
+    let n: u64 = 50;
+
+    let my_vec: Vec<u64> = (0..n).collect();
+
+    // Generate function output.
+    let cache = my_vec.build_cache_bytes();
+
+    // Generate reference data.
+    let mut data = vec![];
+    for i in &my_vec {
+        data.append(&mut int_to_bytes8(*i));
+    }
+    let data = sanitise_bytes(data);
+    let expected = merkleize(data);
+
+    assert_eq!(expected, cache);
+}
+
+#[test]
+fn vec_of_u64_builds() {
     let data = join(vec![
         int_to_bytes8(1),
         int_to_bytes8(2),

From f21409fee1d050dabb60752a6df63de86411dd00 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Thu, 28 Mar 2019 14:44:10 +1100
Subject: [PATCH 012/137] Build breaking recursion tests for cache hashing

---
 eth2/utils/ssz/src/cached_tree_hash/tests.rs | 136 ++++++++++++++++++-
 1 file changed, 134 insertions(+), 2 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index 6c7567250..8b235b3b9 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -68,6 +68,67 @@ impl CachedTreeHash for Inner {
     }
 }
 
+#[derive(Clone)]
+pub struct Outer {
+    pub a: u64,
+    pub b: Inner,
+    pub c: u64,
+}
+
+impl CachedTreeHash for Outer {
+    type Item = Self;
+
+    fn build_cache_bytes(&self) -> Vec<u8> {
+        let mut leaves = vec![];
+
+        leaves.append(&mut self.a.build_cache_bytes());
+        leaves.append(&mut self.b.build_cache_bytes());
+        leaves.append(&mut self.c.build_cache_bytes());
+
+        merkleize(leaves)
+    }
+
+    fn num_bytes(&self) -> usize {
+        let mut bytes = 0;
+        bytes += self.a.num_bytes();
+        bytes += self.b.num_bytes();
+        bytes
+    }
+
+    fn cached_hash_tree_root(
+        &self,
+        other: &Self,
+        cache: &mut TreeHashCache,
+        chunk: usize,
+    ) -> Option<usize> {
+        let mut num_leaves: usize = 0;
+        num_leaves += num_unsanitized_leaves(self.a.num_bytes());
+        num_leaves += num_unsanitized_leaves(self.b.num_bytes());
+        num_leaves += num_unsanitized_leaves(self.c.num_bytes());
+
+        let num_nodes = num_nodes(num_leaves);
+        let num_internal_nodes = num_nodes - num_leaves;
+
+        // Skip past the internal nodes and update any changed leaf nodes.
+        {
+            let chunk = chunk + num_internal_nodes;
+            let chunk = self.a.cached_hash_tree_root(&other.a, cache, chunk)?;
+            let chunk = self.b.cached_hash_tree_root(&other.b, cache, chunk)?;
+            let _chunk = self.c.cached_hash_tree_root(&other.c, cache, chunk)?;
+        }
+
+        // Iterate backwards through the internal nodes, rehashing any node where it's children
+        // have changed.
+        for chunk in (chunk..chunk + num_internal_nodes).into_iter().rev() {
+            if cache.children_modified(chunk)? {
+                cache.modify_chunk(chunk, &cache.hash_children(chunk)?)?;
+            }
+        }
+
+        Some(chunk + num_nodes)
+    }
+}
+
 fn join(many: Vec<Vec<u8>>) -> Vec<u8> {
     let mut all = vec![];
     for one in many {
@@ -76,6 +137,73 @@ fn join(many: Vec<Vec<u8>>) -> Vec<u8> {
     all
 }
 
+#[test]
+fn partial_modification_to_outer() {
+    let inner = Inner {
+        a: 1,
+        b: 2,
+        c: 3,
+        d: 4,
+    };
+
+    let original_outer = Outer {
+        a: 0,
+        b: inner.clone(),
+        c: 5,
+    };
+
+    // Build the initial cache.
+    let original_cache = original_outer.build_cache_bytes();
+
+    // Modify outer
+    let modified_outer = Outer {
+        c: 42,
+        ..original_outer.clone()
+    };
+
+    // Perform a differential hash
+    let mut cache_struct = TreeHashCache::from_bytes(original_cache.clone()).unwrap();
+    modified_outer.cached_hash_tree_root(&original_outer, &mut cache_struct, 0);
+    let modified_cache: Vec<u8> = cache_struct.into();
+
+    // Generate reference data.
+    let mut data = vec![];
+    data.append(&mut int_to_bytes32(0));
+    data.append(&mut inner.build_cache_bytes());
+    data.append(&mut int_to_bytes32(42));
+    let merkle = merkleize(data);
+
+    assert_eq!(merkle, modified_cache);
+}
+
+#[test]
+fn outer_builds() {
+    let inner = Inner {
+        a: 1,
+        b: 2,
+        c: 3,
+        d: 4,
+    };
+
+    let outer = Outer {
+        a: 0,
+        b: inner.clone(),
+        c: 5,
+    };
+
+    // Build the function output.
+    let cache = outer.build_cache_bytes();
+
+    // Generate reference data.
+    let mut data = vec![];
+    data.append(&mut int_to_bytes32(0));
+    data.append(&mut inner.build_cache_bytes());
+    data.append(&mut int_to_bytes32(5));
+    let merkle = merkleize(data);
+
+    assert_eq!(merkle, cache);
+}
+
 #[test]
 fn partial_modification_u64_vec() {
     let n: u64 = 50;
@@ -155,7 +283,11 @@ fn merkleize_odd() {
         int_to_bytes32(5),
     ]);
 
-    merkleize(sanitise_bytes(data));
+    let merkle = merkleize(sanitise_bytes(data));
+
+    let expected_len = num_nodes(8) * BYTES_PER_CHUNK;
+
+    assert_eq!(merkle.len(), expected_len);
 }
 
 fn generic_test(index: usize) {
@@ -221,7 +353,7 @@ fn cached_hash_on_inner() {
 }
 
 #[test]
-fn build_cache_matches_merkelize() {
+fn inner_builds() {
     let data1 = int_to_bytes32(1);
     let data2 = int_to_bytes32(2);
     let data3 = int_to_bytes32(3);

From 49639c40ee26ec31dccf03d9ec2dfa3d6dc92d2d Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Thu, 28 Mar 2019 19:01:31 +1100
Subject: [PATCH 013/137] Implement failing cache hash test

---
 eth2/utils/ssz/src/cached_tree_hash.rs       | 111 +++++++++++-
 eth2/utils/ssz/src/cached_tree_hash/impls.rs |   6 +
 eth2/utils/ssz/src/cached_tree_hash/tests.rs | 181 ++++++++++++++-----
 3 files changed, 248 insertions(+), 50 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index caafaa2cf..75598f0b2 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -1,4 +1,5 @@
 use hashing::hash;
+use std::iter::Iterator;
 
 mod impls;
 mod tests;
@@ -16,6 +17,8 @@ pub trait CachedTreeHash {
     /// prefixes.
     fn num_bytes(&self) -> usize;
 
+    fn num_child_nodes(&self) -> usize;
+
     fn cached_hash_tree_root(
         &self,
         other: &Self::Item,
@@ -81,15 +84,24 @@ impl TreeHashCache {
         self.chunk_modified.get(chunk).cloned()
     }
 
-    pub fn children_modified(&self, parent_chunk: usize) -> Option<bool> {
-        let children = children(parent_chunk);
-
-        Some(self.changed(children.0)? | self.changed(children.1)?)
+    pub fn either_modified(&self, children: (&usize, &usize)) -> Option<bool> {
+        dbg!(&self.chunk_modified.len());
+        dbg!(&self.cache.len() / BYTES_PER_CHUNK);
+        Some(self.changed(*children.0)? | self.changed(*children.1)?)
     }
 
-    pub fn hash_children(&self, parent_chunk: usize) -> Option<Vec<u8>> {
+    /*
+    pub fn children_modified(&self, parent_chunk: usize, child_offsets: &[usize]) -> Option<bool> {
         let children = children(parent_chunk);
 
+        let a = *child_offsets.get(children.0)?;
+        let b = *child_offsets.get(children.1)?;
+
+        Some(self.changed(a)? | self.changed(b)?)
+    }
+    */
+
+    pub fn hash_children(&self, children: (&usize, &usize)) -> Option<Vec<u8>> {
         let start = children.0 * BYTES_PER_CHUNK;
         let end = start + BYTES_PER_CHUNK * 2;
 
@@ -97,6 +109,30 @@ impl TreeHashCache {
     }
 }
 
+/*
+pub struct LocalCache {
+    offsets: Vec<usize>,
+}
+
+impl LocalCache {
+
+}
+
+pub struct OffsetBTree {
+    offsets: Vec<usize>,
+}
+
+impl From<Vec<usize>> for OffsetBTree {
+    fn from(offsets: Vec<usize>) -> Self {
+        Self { offsets }
+    }
+}
+
+impl OffsetBTree {
+    fn
+}
+*/
+
 fn children(parent: usize) -> (usize, usize) {
     ((2 * parent + 1), (2 * parent + 2))
 }
@@ -105,6 +141,71 @@ fn num_nodes(num_leaves: usize) -> usize {
     2 * num_leaves - 1
 }
 
+pub struct OffsetHandler {
+    num_internal_nodes: usize,
+    num_leaf_nodes: usize,
+    next_node: usize,
+    offsets: Vec<usize>,
+}
+
+impl OffsetHandler {
+    fn from_lengths(offset: usize, mut lengths: Vec<usize>) -> Self {
+        // Extend it to the next power-of-two, if it is not already.
+        let num_leaf_nodes = if lengths.len().is_power_of_two() {
+            lengths.len()
+        } else {
+            let num_leaf_nodes = lengths.len().next_power_of_two();
+            lengths.resize(num_leaf_nodes, 1);
+            num_leaf_nodes
+        };
+
+        let num_nodes = num_nodes(num_leaf_nodes);
+        let num_internal_nodes = num_nodes - num_leaf_nodes;
+
+        let mut offsets = Vec::with_capacity(num_nodes);
+        offsets.append(&mut (offset..offset + num_internal_nodes).collect());
+
+        let mut next_node = num_internal_nodes + offset;
+        for i in 0..num_leaf_nodes {
+            offsets.push(next_node);
+            next_node += lengths[i];
+        }
+
+        Self {
+            num_internal_nodes,
+            num_leaf_nodes,
+            offsets,
+            next_node,
+        }
+    }
+
+    pub fn total_nodes(&self) -> usize {
+        self.num_internal_nodes + self.num_leaf_nodes
+    }
+
+    pub fn first_leaf_node(&self) -> Option<usize> {
+        self.offsets.get(self.num_internal_nodes).cloned()
+    }
+
+    pub fn next_node(&self) -> usize {
+        self.next_node
+    }
+
+    pub fn iter_internal_nodes<'a>(
+        &'a self,
+    ) -> impl DoubleEndedIterator<Item = (&'a usize, (&'a usize, &'a usize))> {
+        let internal_nodes = &self.offsets[0..self.num_internal_nodes];
+
+        internal_nodes.iter().enumerate().map(move |(i, parent)| {
+            let children = children(i);
+            (
+                parent,
+                (&self.offsets[children.0], &self.offsets[children.1]),
+            )
+        })
+    }
+}
+
 /// Split `values` into a power-of-two, identical-length chunks (padding with `0`) and merkleize
 /// them, returning the entire merkle tree.
 ///
diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index 1c2bf342e..6fb2d8938 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -12,6 +12,10 @@ impl CachedTreeHash for u64 {
         8
     }
 
+    fn num_child_nodes(&self) -> usize {
+        0
+    }
+
     fn cached_hash_tree_root(
         &self,
         other: &Self,
@@ -27,6 +31,7 @@ impl CachedTreeHash for u64 {
     }
 }
 
+/*
 impl<T> CachedTreeHash for Vec<T>
 where
     T: CachedTreeHash + Encodable,
@@ -96,3 +101,4 @@ where
         Some(chunk + num_nodes)
     }
 }
+*/
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index 8b235b3b9..791bc17b6 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -13,58 +13,87 @@ impl CachedTreeHash for Inner {
     type Item = Self;
 
     fn build_cache_bytes(&self) -> Vec<u8> {
+        let cache_a = self.a.build_cache_bytes();
+        let cache_b = self.b.build_cache_bytes();
+        let cache_c = self.c.build_cache_bytes();
+        let cache_d = self.d.build_cache_bytes();
+
         let mut leaves = vec![];
+        leaves.extend_from_slice(&cache_a[0..32].to_vec());
+        leaves.extend_from_slice(&cache_b[0..32].to_vec());
+        leaves.extend_from_slice(&cache_c[0..32].to_vec());
+        leaves.extend_from_slice(&cache_d[0..32].to_vec());
 
-        leaves.append(&mut self.a.build_cache_bytes());
-        leaves.append(&mut self.b.build_cache_bytes());
-        leaves.append(&mut self.c.build_cache_bytes());
-        leaves.append(&mut self.d.build_cache_bytes());
+        let mut merkle = merkleize(leaves);
 
-        merkleize(leaves)
+        let num_leaves = 4;
+        let num_nodes = num_nodes(num_leaves);
+        let num_internal_nodes = num_nodes - num_leaves;
+
+        let mut next_hash = num_internal_nodes * HASHSIZE;
+        merkle.splice(next_hash..next_hash + HASHSIZE, cache_a);
+        next_hash += HASHSIZE;
+        merkle.splice(next_hash..next_hash + HASHSIZE, cache_b);
+        next_hash += HASHSIZE;
+        merkle.splice(next_hash..next_hash + HASHSIZE, cache_c);
+        next_hash += HASHSIZE;
+        merkle.splice(next_hash..next_hash + HASHSIZE, cache_d);
+
+        merkle
     }
 
     fn num_bytes(&self) -> usize {
         let mut bytes = 0;
+
         bytes += self.a.num_bytes();
         bytes += self.b.num_bytes();
         bytes += self.c.num_bytes();
         bytes += self.d.num_bytes();
+
         bytes
     }
 
+    fn num_child_nodes(&self) -> usize {
+        let mut children = 0;
+        let leaves = 4;
+
+        children += self.a.num_child_nodes();
+        children += self.b.num_child_nodes();
+        children += self.c.num_child_nodes();
+        children += self.d.num_child_nodes();
+
+        num_nodes(leaves) + children - 1
+    }
+
     fn cached_hash_tree_root(
         &self,
         other: &Self,
         cache: &mut TreeHashCache,
         chunk: usize,
     ) -> Option<usize> {
-        let mut num_leaves: usize = 0;
-        num_leaves += num_unsanitized_leaves(self.a.num_bytes());
-        num_leaves += num_unsanitized_leaves(self.b.num_bytes());
-        num_leaves += num_unsanitized_leaves(self.c.num_bytes());
-        num_leaves += num_unsanitized_leaves(self.d.num_bytes());
-
-        let num_nodes = num_nodes(num_leaves);
-        let num_internal_nodes = num_nodes - num_leaves;
+        let mut offsets = vec![];
+        offsets.push(self.a.num_child_nodes() + 1);
+        offsets.push(self.b.num_child_nodes() + 1);
+        offsets.push(self.c.num_child_nodes() + 1);
+        offsets.push(self.d.num_child_nodes() + 1);
+        let offset_handler = OffsetHandler::from_lengths(chunk, offsets);
 
         // Skip past the internal nodes and update any changed leaf nodes.
         {
-            let chunk = chunk + num_internal_nodes;
+            let chunk = offset_handler.first_leaf_node()?;
             let chunk = self.a.cached_hash_tree_root(&other.a, cache, chunk)?;
             let chunk = self.b.cached_hash_tree_root(&other.b, cache, chunk)?;
             let chunk = self.c.cached_hash_tree_root(&other.c, cache, chunk)?;
             let _chunk = self.d.cached_hash_tree_root(&other.d, cache, chunk)?;
         }
 
-        // Iterate backwards through the internal nodes, rehashing any node where it's children
-        // have changed.
-        for chunk in (chunk..chunk + num_internal_nodes).into_iter().rev() {
-            if cache.children_modified(chunk)? {
-                cache.modify_chunk(chunk, &cache.hash_children(chunk)?)?;
+        for (&parent, children) in offset_handler.iter_internal_nodes().rev() {
+            if cache.either_modified(children)? {
+                cache.modify_chunk(parent, &cache.hash_children(children)?)?;
             }
         }
 
-        Some(chunk + num_nodes)
+        Some(offset_handler.next_node())
     }
 }
 
@@ -79,53 +108,79 @@ impl CachedTreeHash for Outer {
     type Item = Self;
 
     fn build_cache_bytes(&self) -> Vec<u8> {
+        let cache_a = self.a.build_cache_bytes();
+        let cache_b = self.b.build_cache_bytes();
+        let cache_c = self.c.build_cache_bytes();
+
         let mut leaves = vec![];
+        leaves.extend_from_slice(&cache_a[0..32].to_vec());
+        leaves.extend_from_slice(&cache_b[0..32].to_vec());
+        leaves.extend_from_slice(&cache_c[0..32].to_vec());
 
-        leaves.append(&mut self.a.build_cache_bytes());
-        leaves.append(&mut self.b.build_cache_bytes());
-        leaves.append(&mut self.c.build_cache_bytes());
+        let mut merkle = merkleize(leaves);
 
-        merkleize(leaves)
+        let num_leaves = 4;
+        let num_nodes = num_nodes(num_leaves);
+        let num_internal_nodes = num_nodes - num_leaves;
+
+        let mut next_hash = num_internal_nodes * HASHSIZE;
+        merkle.splice(next_hash..next_hash + HASHSIZE, cache_a);
+        next_hash += (self.a.num_child_nodes() + 1) * HASHSIZE;
+        merkle.splice(next_hash..next_hash + HASHSIZE, cache_b);
+        next_hash += (self.b.num_child_nodes() + 1) * HASHSIZE;
+        merkle.splice(next_hash..next_hash + HASHSIZE, cache_c);
+
+        merkle
     }
 
     fn num_bytes(&self) -> usize {
         let mut bytes = 0;
         bytes += self.a.num_bytes();
         bytes += self.b.num_bytes();
+        bytes += self.c.num_bytes();
         bytes
     }
 
+    fn num_child_nodes(&self) -> usize {
+        let mut children = 0;
+        let leaves = 3;
+
+        children += self.a.num_child_nodes();
+        children += self.b.num_child_nodes();
+        children += self.c.num_child_nodes();
+
+        num_nodes(leaves) + children - 1
+    }
+
     fn cached_hash_tree_root(
         &self,
         other: &Self,
         cache: &mut TreeHashCache,
         chunk: usize,
     ) -> Option<usize> {
-        let mut num_leaves: usize = 0;
-        num_leaves += num_unsanitized_leaves(self.a.num_bytes());
-        num_leaves += num_unsanitized_leaves(self.b.num_bytes());
-        num_leaves += num_unsanitized_leaves(self.c.num_bytes());
-
-        let num_nodes = num_nodes(num_leaves);
-        let num_internal_nodes = num_nodes - num_leaves;
+        let mut offsets = vec![];
+        offsets.push(self.a.num_child_nodes() + 1);
+        offsets.push(self.b.num_child_nodes() + 1);
+        offsets.push(self.c.num_child_nodes() + 1);
+        let offset_handler = OffsetHandler::from_lengths(chunk, offsets);
 
         // Skip past the internal nodes and update any changed leaf nodes.
         {
-            let chunk = chunk + num_internal_nodes;
+            let chunk = offset_handler.first_leaf_node()?;
             let chunk = self.a.cached_hash_tree_root(&other.a, cache, chunk)?;
             let chunk = self.b.cached_hash_tree_root(&other.b, cache, chunk)?;
             let _chunk = self.c.cached_hash_tree_root(&other.c, cache, chunk)?;
         }
 
-        // Iterate backwards through the internal nodes, rehashing any node where it's children
-        // have changed.
-        for chunk in (chunk..chunk + num_internal_nodes).into_iter().rev() {
-            if cache.children_modified(chunk)? {
-                cache.modify_chunk(chunk, &cache.hash_children(chunk)?)?;
+        for (&parent, children) in offset_handler.iter_internal_nodes().rev() {
+            if cache.either_modified(children)? {
+                dbg!(parent);
+                dbg!(children);
+                cache.modify_chunk(parent, &cache.hash_children(children)?)?;
             }
         }
 
-        Some(chunk + num_nodes)
+        Some(offset_handler.next_node())
     }
 }
 
@@ -163,15 +218,30 @@ fn partial_modification_to_outer() {
 
     // Perform a differential hash
     let mut cache_struct = TreeHashCache::from_bytes(original_cache.clone()).unwrap();
-    modified_outer.cached_hash_tree_root(&original_outer, &mut cache_struct, 0);
+
+    modified_outer
+        .cached_hash_tree_root(&original_outer, &mut cache_struct, 0)
+        .unwrap();
+
     let modified_cache: Vec<u8> = cache_struct.into();
 
     // Generate reference data.
     let mut data = vec![];
     data.append(&mut int_to_bytes32(0));
-    data.append(&mut inner.build_cache_bytes());
-    data.append(&mut int_to_bytes32(42));
-    let merkle = merkleize(data);
+    let inner_bytes = inner.build_cache_bytes();
+    data.append(&mut int_to_bytes32(5));
+
+    let leaves = vec![
+        int_to_bytes32(0),
+        inner_bytes[0..32].to_vec(),
+        int_to_bytes32(5),
+        vec![0; 32], // padding
+    ];
+    let mut merkle = merkleize(join(leaves));
+    merkle.splice(4 * 32..5 * 32, inner_bytes);
+
+    assert_eq!(merkle.len() / HASHSIZE, 13);
+    assert_eq!(modified_cache.len() / HASHSIZE, 13);
 
     assert_eq!(merkle, modified_cache);
 }
@@ -197,13 +267,33 @@ fn outer_builds() {
     // Generate reference data.
     let mut data = vec![];
     data.append(&mut int_to_bytes32(0));
-    data.append(&mut inner.build_cache_bytes());
+    let inner_bytes = inner.build_cache_bytes();
     data.append(&mut int_to_bytes32(5));
-    let merkle = merkleize(data);
 
-    assert_eq!(merkle, cache);
+    let leaves = vec![
+        int_to_bytes32(0),
+        inner_bytes[0..32].to_vec(),
+        int_to_bytes32(5),
+        vec![0; 32], // padding
+    ];
+    let mut merkle = merkleize(join(leaves));
+    merkle.splice(4 * 32..5 * 32, inner_bytes);
+
+    assert_eq!(merkle.len() / HASHSIZE, 13);
+    assert_eq!(cache.len() / HASHSIZE, 13);
+
+    for (i, chunk) in cache.chunks(HASHSIZE).enumerate() {
+        assert_eq!(
+            merkle[i * HASHSIZE..(i + 1) * HASHSIZE],
+            *chunk,
+            "failed on {}",
+            i
+        );
+    }
+    // assert_eq!(merkle, cache);
 }
 
+/*
 #[test]
 fn partial_modification_u64_vec() {
     let n: u64 = 50;
@@ -272,6 +362,7 @@ fn vec_of_u64_builds() {
 
     assert_eq!(expected, cache);
 }
+*/
 
 #[test]
 fn merkleize_odd() {

From 2dcf1c857c893319464a5004240f22f9bf250582 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Thu, 28 Mar 2019 23:21:24 +1100
Subject: [PATCH 014/137] Fix failing cache hashing test

---
 eth2/utils/ssz/src/cached_tree_hash.rs       |  2 --
 eth2/utils/ssz/src/cached_tree_hash/tests.rs | 14 ++------------
 2 files changed, 2 insertions(+), 14 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index 75598f0b2..bf2e4b389 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -85,8 +85,6 @@ impl TreeHashCache {
     }
 
     pub fn either_modified(&self, children: (&usize, &usize)) -> Option<bool> {
-        dbg!(&self.chunk_modified.len());
-        dbg!(&self.cache.len() / BYTES_PER_CHUNK);
         Some(self.changed(*children.0)? | self.changed(*children.1)?)
     }
 
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index 791bc17b6..13e8ef556 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -174,8 +174,6 @@ impl CachedTreeHash for Outer {
 
         for (&parent, children) in offset_handler.iter_internal_nodes().rev() {
             if cache.either_modified(children)? {
-                dbg!(parent);
-                dbg!(children);
                 cache.modify_chunk(parent, &cache.hash_children(children)?)?;
             }
         }
@@ -234,7 +232,7 @@ fn partial_modification_to_outer() {
     let leaves = vec![
         int_to_bytes32(0),
         inner_bytes[0..32].to_vec(),
-        int_to_bytes32(5),
+        int_to_bytes32(42),
         vec![0; 32], // padding
     ];
     let mut merkle = merkleize(join(leaves));
@@ -282,15 +280,7 @@ fn outer_builds() {
     assert_eq!(merkle.len() / HASHSIZE, 13);
     assert_eq!(cache.len() / HASHSIZE, 13);
 
-    for (i, chunk) in cache.chunks(HASHSIZE).enumerate() {
-        assert_eq!(
-            merkle[i * HASHSIZE..(i + 1) * HASHSIZE],
-            *chunk,
-            "failed on {}",
-            i
-        );
-    }
-    // assert_eq!(merkle, cache);
+    assert_eq!(merkle, cache);
 }
 
 /*

From 40bfd5a6c7423ea220222113255ba28c172829b4 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Thu, 28 Mar 2019 23:58:27 +1100
Subject: [PATCH 015/137] Add offset manager method to cache hash trait

---
 eth2/utils/ssz/src/cached_tree_hash.rs       |  32 +++++
 eth2/utils/ssz/src/cached_tree_hash/impls.rs |   4 +
 eth2/utils/ssz/src/cached_tree_hash/tests.rs | 136 ++++++++++++++-----
 3 files changed, 137 insertions(+), 35 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index bf2e4b389..ce90afd33 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -1,5 +1,8 @@
 use hashing::hash;
+use std::iter::IntoIterator;
 use std::iter::Iterator;
+use std::ops::Range;
+use std::vec::Splice;
 
 mod impls;
 mod tests;
@@ -17,6 +20,8 @@ pub trait CachedTreeHash {
     /// prefixes.
     fn num_bytes(&self) -> usize;
 
+    fn offset_handler(&self, initial_offset: usize) -> Option<OffsetHandler>;
+
     fn num_child_nodes(&self) -> usize;
 
     fn cached_hash_tree_root(
@@ -50,6 +55,27 @@ impl TreeHashCache {
         })
     }
 
+    pub fn single_chunk_splice<I>(&mut self, chunk: usize, replace_with: I) -> Splice<I::IntoIter>
+    where
+        I: IntoIterator<Item = u8>,
+    {
+        self.chunk_splice(chunk..chunk + 1, replace_with)
+    }
+
+    pub fn chunk_splice<I>(
+        &mut self,
+        chunk_range: Range<usize>,
+        replace_with: I,
+    ) -> Splice<I::IntoIter>
+    where
+        I: IntoIterator<Item = u8>,
+    {
+        let byte_start = chunk_range.start * BYTES_PER_CHUNK;
+        let byte_end = chunk_range.end * BYTES_PER_CHUNK;
+
+        self.cache.splice(byte_start..byte_end, replace_with)
+    }
+
     pub fn maybe_update_chunk(&mut self, chunk: usize, to: &[u8]) -> Option<()> {
         let start = chunk * BYTES_PER_CHUNK;
         let end = start + BYTES_PER_CHUNK;
@@ -202,6 +228,12 @@ impl OffsetHandler {
             )
         })
     }
+
+    pub fn iter_leaf_nodes<'a>(&'a self) -> impl DoubleEndedIterator<Item = &'a usize> {
+        let leaf_nodes = &self.offsets[self.num_internal_nodes..];
+
+        leaf_nodes.iter()
+    }
 }
 
 /// Split `values` into a power-of-two, identical-length chunks (padding with `0`) and merkleize
diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index 6fb2d8938..a4d1c7d1a 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -12,6 +12,10 @@ impl CachedTreeHash for u64 {
         8
     }
 
+    fn offset_handler(&self, _initial_offset: usize) -> Option<OffsetHandler> {
+        None
+    }
+
     fn num_child_nodes(&self) -> usize {
         0
     }
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index 13e8ef556..9db0a5906 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -24,22 +24,19 @@ impl CachedTreeHash for Inner {
         leaves.extend_from_slice(&cache_c[0..32].to_vec());
         leaves.extend_from_slice(&cache_d[0..32].to_vec());
 
-        let mut merkle = merkleize(leaves);
+        // TODO: fix unwrap
+        let mut cache = TreeHashCache::from_bytes(merkleize(leaves)).unwrap();
 
-        let num_leaves = 4;
-        let num_nodes = num_nodes(num_leaves);
-        let num_internal_nodes = num_nodes - num_leaves;
+        // TODO: fix unwrap
+        let offset_handler = self.offset_handler(0).unwrap();
+        let mut iter = offset_handler.iter_leaf_nodes();
 
-        let mut next_hash = num_internal_nodes * HASHSIZE;
-        merkle.splice(next_hash..next_hash + HASHSIZE, cache_a);
-        next_hash += HASHSIZE;
-        merkle.splice(next_hash..next_hash + HASHSIZE, cache_b);
-        next_hash += HASHSIZE;
-        merkle.splice(next_hash..next_hash + HASHSIZE, cache_c);
-        next_hash += HASHSIZE;
-        merkle.splice(next_hash..next_hash + HASHSIZE, cache_d);
+        cache.single_chunk_splice(*iter.next().unwrap(), cache_a);
+        cache.single_chunk_splice(*iter.next().unwrap(), cache_b);
+        cache.single_chunk_splice(*iter.next().unwrap(), cache_c);
+        cache.single_chunk_splice(*iter.next().unwrap(), cache_d);
 
-        merkle
+        cache.into()
     }
 
     fn num_bytes(&self) -> usize {
@@ -53,6 +50,17 @@ impl CachedTreeHash for Inner {
         bytes
     }
 
+    fn offset_handler(&self, initial_offset: usize) -> Option<OffsetHandler> {
+        let mut offsets = vec![];
+
+        offsets.push(self.a.num_child_nodes() + 1);
+        offsets.push(self.b.num_child_nodes() + 1);
+        offsets.push(self.c.num_child_nodes() + 1);
+        offsets.push(self.d.num_child_nodes() + 1);
+
+        Some(OffsetHandler::from_lengths(initial_offset, offsets))
+    }
+
     fn num_child_nodes(&self) -> usize {
         let mut children = 0;
         let leaves = 4;
@@ -71,12 +79,7 @@ impl CachedTreeHash for Inner {
         cache: &mut TreeHashCache,
         chunk: usize,
     ) -> Option<usize> {
-        let mut offsets = vec![];
-        offsets.push(self.a.num_child_nodes() + 1);
-        offsets.push(self.b.num_child_nodes() + 1);
-        offsets.push(self.c.num_child_nodes() + 1);
-        offsets.push(self.d.num_child_nodes() + 1);
-        let offset_handler = OffsetHandler::from_lengths(chunk, offsets);
+        let offset_handler = self.offset_handler(chunk)?;
 
         // Skip past the internal nodes and update any changed leaf nodes.
         {
@@ -117,20 +120,18 @@ impl CachedTreeHash for Outer {
         leaves.extend_from_slice(&cache_b[0..32].to_vec());
         leaves.extend_from_slice(&cache_c[0..32].to_vec());
 
-        let mut merkle = merkleize(leaves);
+        // TODO: fix unwrap
+        let mut cache = TreeHashCache::from_bytes(merkleize(leaves)).unwrap();
 
-        let num_leaves = 4;
-        let num_nodes = num_nodes(num_leaves);
-        let num_internal_nodes = num_nodes - num_leaves;
+        // TODO: fix unwrap
+        let offset_handler = self.offset_handler(0).unwrap();
+        let mut iter = offset_handler.iter_leaf_nodes();
 
-        let mut next_hash = num_internal_nodes * HASHSIZE;
-        merkle.splice(next_hash..next_hash + HASHSIZE, cache_a);
-        next_hash += (self.a.num_child_nodes() + 1) * HASHSIZE;
-        merkle.splice(next_hash..next_hash + HASHSIZE, cache_b);
-        next_hash += (self.b.num_child_nodes() + 1) * HASHSIZE;
-        merkle.splice(next_hash..next_hash + HASHSIZE, cache_c);
+        cache.single_chunk_splice(*iter.next().unwrap(), cache_a);
+        cache.single_chunk_splice(*iter.next().unwrap(), cache_b);
+        cache.single_chunk_splice(*iter.next().unwrap(), cache_c);
 
-        merkle
+        cache.into()
     }
 
     fn num_bytes(&self) -> usize {
@@ -152,17 +153,23 @@ impl CachedTreeHash for Outer {
         num_nodes(leaves) + children - 1
     }
 
+    fn offset_handler(&self, initial_offset: usize) -> Option<OffsetHandler> {
+        let mut offsets = vec![];
+
+        offsets.push(self.a.num_child_nodes() + 1);
+        offsets.push(self.b.num_child_nodes() + 1);
+        offsets.push(self.c.num_child_nodes() + 1);
+
+        Some(OffsetHandler::from_lengths(initial_offset, offsets))
+    }
+
     fn cached_hash_tree_root(
         &self,
         other: &Self,
         cache: &mut TreeHashCache,
         chunk: usize,
     ) -> Option<usize> {
-        let mut offsets = vec![];
-        offsets.push(self.a.num_child_nodes() + 1);
-        offsets.push(self.b.num_child_nodes() + 1);
-        offsets.push(self.c.num_child_nodes() + 1);
-        let offset_handler = OffsetHandler::from_lengths(chunk, offsets);
+        let offset_handler = self.offset_handler(chunk)?;
 
         // Skip past the internal nodes and update any changed leaf nodes.
         {
@@ -190,6 +197,65 @@ fn join(many: Vec<Vec<u8>>) -> Vec<u8> {
     all
 }
 
+#[test]
+fn partial_modification_to_inner_struct() {
+    let original_inner = Inner {
+        a: 1,
+        b: 2,
+        c: 3,
+        d: 4,
+    };
+
+    let original_outer = Outer {
+        a: 0,
+        b: original_inner.clone(),
+        c: 5,
+    };
+
+    let modified_inner = Inner {
+        a: 42,
+        ..original_inner.clone()
+    };
+
+    // Build the initial cache.
+    let original_cache = original_outer.build_cache_bytes();
+
+    // Modify outer
+    let modified_outer = Outer {
+        b: modified_inner.clone(),
+        ..original_outer.clone()
+    };
+
+    // Perform a differential hash
+    let mut cache_struct = TreeHashCache::from_bytes(original_cache.clone()).unwrap();
+
+    modified_outer
+        .cached_hash_tree_root(&original_outer, &mut cache_struct, 0)
+        .unwrap();
+
+    let modified_cache: Vec<u8> = cache_struct.into();
+
+    // Generate reference data.
+    let mut data = vec![];
+    data.append(&mut int_to_bytes32(0));
+    let inner_bytes = modified_inner.build_cache_bytes();
+    data.append(&mut int_to_bytes32(5));
+
+    let leaves = vec![
+        int_to_bytes32(0),
+        inner_bytes[0..32].to_vec(),
+        int_to_bytes32(5),
+        vec![0; 32], // padding
+    ];
+    let mut merkle = merkleize(join(leaves));
+    merkle.splice(4 * 32..5 * 32, inner_bytes);
+
+    assert_eq!(merkle.len() / HASHSIZE, 13);
+    assert_eq!(modified_cache.len() / HASHSIZE, 13);
+
+    assert_eq!(merkle, modified_cache);
+}
+
 #[test]
 fn partial_modification_to_outer() {
     let inner = Inner {

From 7b05c506df36317e07aaa751e556526b0719cbdd Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Fri, 29 Mar 2019 00:47:42 +1100
Subject: [PATCH 016/137] Add new build method for cached hashes

---
 eth2/utils/ssz/src/cached_tree_hash.rs       | 33 +++++++++++++++
 eth2/utils/ssz/src/cached_tree_hash/tests.rs | 44 ++++++--------------
 2 files changed, 46 insertions(+), 31 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index ce90afd33..be3fe98de 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -44,6 +44,39 @@ impl Into<Vec<u8>> for TreeHashCache {
 }
 
 impl TreeHashCache {
+    pub fn new(mut leaves_and_subtrees: Vec<u8>, offset_handler: OffsetHandler) -> Option<Self> {
+        if leaves_and_subtrees.len() % BYTES_PER_CHUNK != 0 {
+            return None;
+        }
+
+        // Allocate enough bytes to store the internal nodes and the leaves and subtrees, then fill
+        // all the to-be-built internal nodes with zeros and append the leaves and subtrees.
+        let internal_node_bytes = offset_handler.num_internal_nodes * BYTES_PER_CHUNK;
+        let mut cache = Vec::with_capacity(internal_node_bytes + leaves_and_subtrees.len());
+        cache.resize(internal_node_bytes, 0);
+        cache.append(&mut leaves_and_subtrees);
+
+        // Concat all the leaves into one big byte array, ready for `merkleize`.
+        let mut leaves = vec![];
+        for leaf_chunk in offset_handler.iter_leaf_nodes() {
+            let start = leaf_chunk * BYTES_PER_CHUNK;
+            let end = start + BYTES_PER_CHUNK;
+
+            leaves.extend_from_slice(cache.get(start..end)?);
+        }
+
+        // Merkleize the leaves, then split the leaf nodes off them. Then, replace all-zeros
+        // internal nodes created earlier with the internal nodes generated by `merkleize`.
+        let mut merkleized = merkleize(leaves);
+        merkleized.split_off(internal_node_bytes);
+        cache.splice(0..internal_node_bytes, merkleized);
+
+        Some(Self {
+            chunk_modified: vec![false; cache.len() / BYTES_PER_CHUNK],
+            cache,
+        })
+    }
+
     pub fn from_bytes(bytes: Vec<u8>) -> Option<Self> {
         if bytes.len() % BYTES_PER_CHUNK > 0 {
             return None;
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index 9db0a5906..9cb012c79 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -13,28 +13,18 @@ impl CachedTreeHash for Inner {
     type Item = Self;
 
     fn build_cache_bytes(&self) -> Vec<u8> {
-        let cache_a = self.a.build_cache_bytes();
-        let cache_b = self.b.build_cache_bytes();
-        let cache_c = self.c.build_cache_bytes();
-        let cache_d = self.d.build_cache_bytes();
+        let mut leaves_and_subtrees = vec![];
 
-        let mut leaves = vec![];
-        leaves.extend_from_slice(&cache_a[0..32].to_vec());
-        leaves.extend_from_slice(&cache_b[0..32].to_vec());
-        leaves.extend_from_slice(&cache_c[0..32].to_vec());
-        leaves.extend_from_slice(&cache_d[0..32].to_vec());
-
-        // TODO: fix unwrap
-        let mut cache = TreeHashCache::from_bytes(merkleize(leaves)).unwrap();
+        leaves_and_subtrees.append(&mut self.a.build_cache_bytes());
+        leaves_and_subtrees.append(&mut self.b.build_cache_bytes());
+        leaves_and_subtrees.append(&mut self.c.build_cache_bytes());
+        leaves_and_subtrees.append(&mut self.d.build_cache_bytes());
 
         // TODO: fix unwrap
         let offset_handler = self.offset_handler(0).unwrap();
-        let mut iter = offset_handler.iter_leaf_nodes();
 
-        cache.single_chunk_splice(*iter.next().unwrap(), cache_a);
-        cache.single_chunk_splice(*iter.next().unwrap(), cache_b);
-        cache.single_chunk_splice(*iter.next().unwrap(), cache_c);
-        cache.single_chunk_splice(*iter.next().unwrap(), cache_d);
+        // TODO: fix unwrap
+        let cache = TreeHashCache::new(leaves_and_subtrees, offset_handler).unwrap();
 
         cache.into()
     }
@@ -111,25 +101,17 @@ impl CachedTreeHash for Outer {
     type Item = Self;
 
     fn build_cache_bytes(&self) -> Vec<u8> {
-        let cache_a = self.a.build_cache_bytes();
-        let cache_b = self.b.build_cache_bytes();
-        let cache_c = self.c.build_cache_bytes();
+        let mut leaves_and_subtrees = vec![];
 
-        let mut leaves = vec![];
-        leaves.extend_from_slice(&cache_a[0..32].to_vec());
-        leaves.extend_from_slice(&cache_b[0..32].to_vec());
-        leaves.extend_from_slice(&cache_c[0..32].to_vec());
-
-        // TODO: fix unwrap
-        let mut cache = TreeHashCache::from_bytes(merkleize(leaves)).unwrap();
+        leaves_and_subtrees.append(&mut self.a.build_cache_bytes());
+        leaves_and_subtrees.append(&mut self.b.build_cache_bytes());
+        leaves_and_subtrees.append(&mut self.c.build_cache_bytes());
 
         // TODO: fix unwrap
         let offset_handler = self.offset_handler(0).unwrap();
-        let mut iter = offset_handler.iter_leaf_nodes();
 
-        cache.single_chunk_splice(*iter.next().unwrap(), cache_a);
-        cache.single_chunk_splice(*iter.next().unwrap(), cache_b);
-        cache.single_chunk_splice(*iter.next().unwrap(), cache_c);
+        // TODO: fix unwrap
+        let cache = TreeHashCache::new(leaves_and_subtrees, offset_handler).unwrap();
 
         cache.into()
     }

From 267c978abb2b1bc42513866cff54aa818f648a8a Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Fri, 29 Mar 2019 02:36:34 +1100
Subject: [PATCH 017/137] Tidy cache hash API

---
 eth2/utils/ssz/src/cached_tree_hash.rs       | 137 ++++++++++---------
 eth2/utils/ssz/src/cached_tree_hash/impls.rs |  12 +-
 eth2/utils/ssz/src/cached_tree_hash/tests.rs |  79 +++++------
 3 files changed, 119 insertions(+), 109 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index be3fe98de..3b900e503 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -11,16 +11,27 @@ const BYTES_PER_CHUNK: usize = 32;
 const HASHSIZE: usize = 32;
 const MERKLE_HASH_CHUNCK: usize = 2 * BYTES_PER_CHUNK;
 
+#[derive(Debug, PartialEq, Clone)]
+pub enum Error {
+    LeavesAndSubtreesIncomplete(usize),
+    ShouldNotProduceOffsetHandler,
+    NoFirstNode,
+    BytesAreNotEvenChunks(usize),
+    NoModifiedFieldForChunk(usize),
+    NoBytesForChunk(usize),
+    NoChildrenForHashing((usize, usize)),
+}
+
 pub trait CachedTreeHash {
     type Item: CachedTreeHash;
 
-    fn build_cache_bytes(&self) -> Vec<u8>;
+    fn build_cache(&self) -> Result<TreeHashCache, Error>;
 
     /// Return the number of bytes when this element is encoded as raw SSZ _without_ length
     /// prefixes.
     fn num_bytes(&self) -> usize;
 
-    fn offset_handler(&self, initial_offset: usize) -> Option<OffsetHandler>;
+    fn offset_handler(&self, initial_offset: usize) -> Result<OffsetHandler, Error>;
 
     fn num_child_nodes(&self) -> usize;
 
@@ -29,9 +40,10 @@ pub trait CachedTreeHash {
         other: &Self::Item,
         cache: &mut TreeHashCache,
         chunk: usize,
-    ) -> Option<usize>;
+    ) -> Result<usize, Error>;
 }
 
+#[derive(Debug, PartialEq, Clone)]
 pub struct TreeHashCache {
     cache: Vec<u8>,
     chunk_modified: Vec<bool>,
@@ -44,11 +56,17 @@ impl Into<Vec<u8>> for TreeHashCache {
 }
 
 impl TreeHashCache {
-    pub fn new(mut leaves_and_subtrees: Vec<u8>, offset_handler: OffsetHandler) -> Option<Self> {
-        if leaves_and_subtrees.len() % BYTES_PER_CHUNK != 0 {
-            return None;
-        }
+    pub fn new<T>(item: &T) -> Result<Self, Error>
+    where
+        T: CachedTreeHash,
+    {
+        item.build_cache()
+    }
 
+    pub fn from_leaves_and_subtrees(
+        mut leaves_and_subtrees: Vec<u8>,
+        offset_handler: OffsetHandler,
+    ) -> Result<Self, Error> {
         // Allocate enough bytes to store the internal nodes and the leaves and subtrees, then fill
         // all the to-be-built internal nodes with zeros and append the leaves and subtrees.
         let internal_node_bytes = offset_handler.num_internal_nodes * BYTES_PER_CHUNK;
@@ -56,13 +74,22 @@ impl TreeHashCache {
         cache.resize(internal_node_bytes, 0);
         cache.append(&mut leaves_and_subtrees);
 
+        dbg!(cache.len() / BYTES_PER_CHUNK);
+
         // Concat all the leaves into one big byte array, ready for `merkleize`.
         let mut leaves = vec![];
         for leaf_chunk in offset_handler.iter_leaf_nodes() {
             let start = leaf_chunk * BYTES_PER_CHUNK;
             let end = start + BYTES_PER_CHUNK;
 
-            leaves.extend_from_slice(cache.get(start..end)?);
+            dbg!(end);
+            dbg!(cache.len());
+
+            leaves.extend_from_slice(
+                cache
+                    .get(start..end)
+                    .ok_or_else(|| Error::LeavesAndSubtreesIncomplete(*leaf_chunk))?,
+            );
         }
 
         // Merkleize the leaves, then split the leaf nodes off them. Then, replace all-zeros
@@ -71,18 +98,18 @@ impl TreeHashCache {
         merkleized.split_off(internal_node_bytes);
         cache.splice(0..internal_node_bytes, merkleized);
 
-        Some(Self {
+        Ok(Self {
             chunk_modified: vec![false; cache.len() / BYTES_PER_CHUNK],
             cache,
         })
     }
 
-    pub fn from_bytes(bytes: Vec<u8>) -> Option<Self> {
+    pub fn from_bytes(bytes: Vec<u8>) -> Result<Self, Error> {
         if bytes.len() % BYTES_PER_CHUNK > 0 {
-            return None;
+            return Err(Error::BytesAreNotEvenChunks(bytes.len()));
         }
 
-        Some(Self {
+        Ok(Self {
             chunk_modified: vec![false; bytes.len() / BYTES_PER_CHUNK],
             cache: bytes,
         })
@@ -121,15 +148,18 @@ impl TreeHashCache {
         Some(())
     }
 
-    pub fn modify_chunk(&mut self, chunk: usize, to: &[u8]) -> Option<()> {
+    pub fn modify_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> {
         let start = chunk * BYTES_PER_CHUNK;
         let end = start + BYTES_PER_CHUNK;
 
-        self.cache.get_mut(start..end)?.copy_from_slice(to);
+        self.cache
+            .get_mut(start..end)
+            .ok_or_else(|| Error::NoBytesForChunk(chunk))?
+            .copy_from_slice(to);
 
         self.chunk_modified[chunk] = true;
 
-        Some(())
+        Ok(())
     }
 
     pub fn chunk_equals(&mut self, chunk: usize, other: &[u8]) -> Option<bool> {
@@ -139,57 +169,30 @@ impl TreeHashCache {
         Some(self.cache.get(start..end)? == other)
     }
 
-    pub fn changed(&self, chunk: usize) -> Option<bool> {
-        self.chunk_modified.get(chunk).cloned()
+    pub fn changed(&self, chunk: usize) -> Result<bool, Error> {
+        self.chunk_modified
+            .get(chunk)
+            .cloned()
+            .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))
     }
 
-    pub fn either_modified(&self, children: (&usize, &usize)) -> Option<bool> {
-        Some(self.changed(*children.0)? | self.changed(*children.1)?)
+    pub fn either_modified(&self, children: (&usize, &usize)) -> Result<bool, Error> {
+        Ok(self.changed(*children.0)? | self.changed(*children.1)?)
     }
 
-    /*
-    pub fn children_modified(&self, parent_chunk: usize, child_offsets: &[usize]) -> Option<bool> {
-        let children = children(parent_chunk);
-
-        let a = *child_offsets.get(children.0)?;
-        let b = *child_offsets.get(children.1)?;
-
-        Some(self.changed(a)? | self.changed(b)?)
-    }
-    */
-
-    pub fn hash_children(&self, children: (&usize, &usize)) -> Option<Vec<u8>> {
+    pub fn hash_children(&self, children: (&usize, &usize)) -> Result<Vec<u8>, Error> {
         let start = children.0 * BYTES_PER_CHUNK;
         let end = start + BYTES_PER_CHUNK * 2;
 
-        Some(hash(&self.cache.get(start..end)?))
+        let children = &self
+            .cache
+            .get(start..end)
+            .ok_or_else(|| Error::NoChildrenForHashing((*children.0, *children.1)))?;
+
+        Ok(hash(children))
     }
 }
 
-/*
-pub struct LocalCache {
-    offsets: Vec<usize>,
-}
-
-impl LocalCache {
-
-}
-
-pub struct OffsetBTree {
-    offsets: Vec<usize>,
-}
-
-impl From<Vec<usize>> for OffsetBTree {
-    fn from(offsets: Vec<usize>) -> Self {
-        Self { offsets }
-    }
-}
-
-impl OffsetBTree {
-    fn
-}
-*/
-
 fn children(parent: usize) -> (usize, usize) {
     ((2 * parent + 1), (2 * parent + 2))
 }
@@ -206,7 +209,7 @@ pub struct OffsetHandler {
 }
 
 impl OffsetHandler {
-    fn from_lengths(offset: usize, mut lengths: Vec<usize>) -> Self {
+    fn from_lengths(offset: usize, mut lengths: Vec<usize>) -> Result<Self, Error> {
         // Extend it to the next power-of-two, if it is not already.
         let num_leaf_nodes = if lengths.len().is_power_of_two() {
             lengths.len()
@@ -228,20 +231,23 @@ impl OffsetHandler {
             next_node += lengths[i];
         }
 
-        Self {
+        Ok(Self {
             num_internal_nodes,
             num_leaf_nodes,
             offsets,
             next_node,
-        }
+        })
     }
 
     pub fn total_nodes(&self) -> usize {
         self.num_internal_nodes + self.num_leaf_nodes
     }
 
-    pub fn first_leaf_node(&self) -> Option<usize> {
-        self.offsets.get(self.num_internal_nodes).cloned()
+    pub fn first_leaf_node(&self) -> Result<usize, Error> {
+        self.offsets
+            .get(self.num_internal_nodes)
+            .cloned()
+            .ok_or_else(|| Error::NoFirstNode)
     }
 
     pub fn next_node(&self) -> usize {
@@ -314,6 +320,15 @@ pub fn sanitise_bytes(mut bytes: Vec<u8>) -> Vec<u8> {
     bytes
 }
 
+fn pad_for_leaf_count(num_leaves: usize, bytes: &mut Vec<u8>) {
+    let required_leaves = num_leaves.next_power_of_two();
+
+    bytes.resize(
+        bytes.len() + (required_leaves - num_leaves) * BYTES_PER_CHUNK,
+        0,
+    );
+}
+
 fn last_leaf_needs_padding(num_bytes: usize) -> bool {
     num_bytes % HASHSIZE != 0
 }
diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index a4d1c7d1a..9c0a8ec6d 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -4,16 +4,16 @@ use crate::{ssz_encode, Encodable};
 impl CachedTreeHash for u64 {
     type Item = Self;
 
-    fn build_cache_bytes(&self) -> Vec<u8> {
-        merkleize(ssz_encode(self))
+    fn build_cache(&self) -> Result<TreeHashCache, Error> {
+        TreeHashCache::from_bytes(merkleize(ssz_encode(self)))
     }
 
     fn num_bytes(&self) -> usize {
         8
     }
 
-    fn offset_handler(&self, _initial_offset: usize) -> Option<OffsetHandler> {
-        None
+    fn offset_handler(&self, _initial_offset: usize) -> Result<OffsetHandler, Error> {
+        Err(Error::ShouldNotProduceOffsetHandler)
     }
 
     fn num_child_nodes(&self) -> usize {
@@ -25,13 +25,13 @@ impl CachedTreeHash for u64 {
         other: &Self,
         cache: &mut TreeHashCache,
         chunk: usize,
-    ) -> Option<usize> {
+    ) -> Result<usize, Error> {
         if self != other {
             let leaf = merkleize(ssz_encode(self));
             cache.modify_chunk(chunk, &leaf)?;
         }
 
-        Some(chunk + 1)
+        Ok(chunk + 1)
     }
 }
 
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index 9cb012c79..a6be7f9ae 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -1,5 +1,5 @@
 use super::*;
-use int_to_bytes::{int_to_bytes32, int_to_bytes8};
+use int_to_bytes::int_to_bytes32;
 
 #[derive(Clone)]
 pub struct Inner {
@@ -12,21 +12,19 @@ pub struct Inner {
 impl CachedTreeHash for Inner {
     type Item = Self;
 
-    fn build_cache_bytes(&self) -> Vec<u8> {
+    fn build_cache(&self) -> Result<TreeHashCache, Error> {
+        let offset_handler = self.offset_handler(0)?;
+
         let mut leaves_and_subtrees = vec![];
 
-        leaves_and_subtrees.append(&mut self.a.build_cache_bytes());
-        leaves_and_subtrees.append(&mut self.b.build_cache_bytes());
-        leaves_and_subtrees.append(&mut self.c.build_cache_bytes());
-        leaves_and_subtrees.append(&mut self.d.build_cache_bytes());
+        leaves_and_subtrees.append(&mut self.a.build_cache()?.into());
+        leaves_and_subtrees.append(&mut self.b.build_cache()?.into());
+        leaves_and_subtrees.append(&mut self.c.build_cache()?.into());
+        leaves_and_subtrees.append(&mut self.d.build_cache()?.into());
 
-        // TODO: fix unwrap
-        let offset_handler = self.offset_handler(0).unwrap();
+        pad_for_leaf_count(offset_handler.num_leaf_nodes, &mut leaves_and_subtrees);
 
-        // TODO: fix unwrap
-        let cache = TreeHashCache::new(leaves_and_subtrees, offset_handler).unwrap();
-
-        cache.into()
+        TreeHashCache::from_leaves_and_subtrees(leaves_and_subtrees, self.offset_handler(0)?)
     }
 
     fn num_bytes(&self) -> usize {
@@ -40,7 +38,7 @@ impl CachedTreeHash for Inner {
         bytes
     }
 
-    fn offset_handler(&self, initial_offset: usize) -> Option<OffsetHandler> {
+    fn offset_handler(&self, initial_offset: usize) -> Result<OffsetHandler, Error> {
         let mut offsets = vec![];
 
         offsets.push(self.a.num_child_nodes() + 1);
@@ -48,7 +46,7 @@ impl CachedTreeHash for Inner {
         offsets.push(self.c.num_child_nodes() + 1);
         offsets.push(self.d.num_child_nodes() + 1);
 
-        Some(OffsetHandler::from_lengths(initial_offset, offsets))
+        OffsetHandler::from_lengths(initial_offset, offsets)
     }
 
     fn num_child_nodes(&self) -> usize {
@@ -68,7 +66,7 @@ impl CachedTreeHash for Inner {
         other: &Self,
         cache: &mut TreeHashCache,
         chunk: usize,
-    ) -> Option<usize> {
+    ) -> Result<usize, Error> {
         let offset_handler = self.offset_handler(chunk)?;
 
         // Skip past the internal nodes and update any changed leaf nodes.
@@ -86,7 +84,7 @@ impl CachedTreeHash for Inner {
             }
         }
 
-        Some(offset_handler.next_node())
+        Ok(offset_handler.next_node())
     }
 }
 
@@ -100,20 +98,18 @@ pub struct Outer {
 impl CachedTreeHash for Outer {
     type Item = Self;
 
-    fn build_cache_bytes(&self) -> Vec<u8> {
+    fn build_cache(&self) -> Result<TreeHashCache, Error> {
+        let offset_handler = self.offset_handler(0)?;
+
         let mut leaves_and_subtrees = vec![];
 
-        leaves_and_subtrees.append(&mut self.a.build_cache_bytes());
-        leaves_and_subtrees.append(&mut self.b.build_cache_bytes());
-        leaves_and_subtrees.append(&mut self.c.build_cache_bytes());
+        leaves_and_subtrees.append(&mut self.a.build_cache()?.into());
+        leaves_and_subtrees.append(&mut self.b.build_cache()?.into());
+        leaves_and_subtrees.append(&mut self.c.build_cache()?.into());
 
-        // TODO: fix unwrap
-        let offset_handler = self.offset_handler(0).unwrap();
+        pad_for_leaf_count(offset_handler.num_leaf_nodes, &mut leaves_and_subtrees);
 
-        // TODO: fix unwrap
-        let cache = TreeHashCache::new(leaves_and_subtrees, offset_handler).unwrap();
-
-        cache.into()
+        TreeHashCache::from_leaves_and_subtrees(leaves_and_subtrees, self.offset_handler(0)?)
     }
 
     fn num_bytes(&self) -> usize {
@@ -135,14 +131,14 @@ impl CachedTreeHash for Outer {
         num_nodes(leaves) + children - 1
     }
 
-    fn offset_handler(&self, initial_offset: usize) -> Option<OffsetHandler> {
+    fn offset_handler(&self, initial_offset: usize) -> Result<OffsetHandler, Error> {
         let mut offsets = vec![];
 
         offsets.push(self.a.num_child_nodes() + 1);
         offsets.push(self.b.num_child_nodes() + 1);
         offsets.push(self.c.num_child_nodes() + 1);
 
-        Some(OffsetHandler::from_lengths(initial_offset, offsets))
+        OffsetHandler::from_lengths(initial_offset, offsets)
     }
 
     fn cached_hash_tree_root(
@@ -150,7 +146,7 @@ impl CachedTreeHash for Outer {
         other: &Self,
         cache: &mut TreeHashCache,
         chunk: usize,
-    ) -> Option<usize> {
+    ) -> Result<usize, Error> {
         let offset_handler = self.offset_handler(chunk)?;
 
         // Skip past the internal nodes and update any changed leaf nodes.
@@ -167,7 +163,7 @@ impl CachedTreeHash for Outer {
             }
         }
 
-        Some(offset_handler.next_node())
+        Ok(offset_handler.next_node())
     }
 }
 
@@ -199,17 +195,16 @@ fn partial_modification_to_inner_struct() {
         ..original_inner.clone()
     };
 
-    // Build the initial cache.
-    let original_cache = original_outer.build_cache_bytes();
-
     // Modify outer
     let modified_outer = Outer {
         b: modified_inner.clone(),
         ..original_outer.clone()
     };
 
+    println!("AAAAAAAAA");
     // Perform a differential hash
-    let mut cache_struct = TreeHashCache::from_bytes(original_cache.clone()).unwrap();
+    let mut cache_struct = TreeHashCache::new(&original_outer).unwrap();
+    println!("BBBBBBBBBB");
 
     modified_outer
         .cached_hash_tree_root(&original_outer, &mut cache_struct, 0)
@@ -220,7 +215,7 @@ fn partial_modification_to_inner_struct() {
     // Generate reference data.
     let mut data = vec![];
     data.append(&mut int_to_bytes32(0));
-    let inner_bytes = modified_inner.build_cache_bytes();
+    let inner_bytes: Vec<u8> = TreeHashCache::new(&modified_inner).unwrap().into();
     data.append(&mut int_to_bytes32(5));
 
     let leaves = vec![
@@ -254,7 +249,7 @@ fn partial_modification_to_outer() {
     };
 
     // Build the initial cache.
-    let original_cache = original_outer.build_cache_bytes();
+    // let original_cache = original_outer.build_cache_bytes();
 
     // Modify outer
     let modified_outer = Outer {
@@ -263,7 +258,7 @@ fn partial_modification_to_outer() {
     };
 
     // Perform a differential hash
-    let mut cache_struct = TreeHashCache::from_bytes(original_cache.clone()).unwrap();
+    let mut cache_struct = TreeHashCache::new(&original_outer).unwrap();
 
     modified_outer
         .cached_hash_tree_root(&original_outer, &mut cache_struct, 0)
@@ -274,7 +269,7 @@ fn partial_modification_to_outer() {
     // Generate reference data.
     let mut data = vec![];
     data.append(&mut int_to_bytes32(0));
-    let inner_bytes = inner.build_cache_bytes();
+    let inner_bytes: Vec<u8> = TreeHashCache::new(&inner).unwrap().into();
     data.append(&mut int_to_bytes32(5));
 
     let leaves = vec![
@@ -308,12 +303,12 @@ fn outer_builds() {
     };
 
     // Build the function output.
-    let cache = outer.build_cache_bytes();
+    let cache: Vec<u8> = TreeHashCache::new(&outer).unwrap().into();
 
     // Generate reference data.
     let mut data = vec![];
     data.append(&mut int_to_bytes32(0));
-    let inner_bytes = inner.build_cache_bytes();
+    let inner_bytes: Vec<u8> = inner.build_cache().unwrap().into();
     data.append(&mut int_to_bytes32(5));
 
     let leaves = vec![
@@ -427,7 +422,7 @@ fn generic_test(index: usize) {
         d: 4,
     };
 
-    let cache = inner.build_cache_bytes();
+    let cache: Vec<u8> = TreeHashCache::new(&inner).unwrap().into();
 
     let changed_inner = match index {
         0 => Inner {
@@ -498,7 +493,7 @@ fn inner_builds() {
         d: 4,
     };
 
-    let cache = inner.build_cache_bytes();
+    let cache: Vec<u8> = TreeHashCache::new(&inner).unwrap().into();
 
     assert_eq!(expected, cache);
 }

From e0104e61997d6ca868829502c8775bae07952fef Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Fri, 29 Mar 2019 13:04:01 +1100
Subject: [PATCH 018/137] Move offset_handler construction into self

---
 eth2/utils/ssz/src/cached_tree_hash.rs       | 17 ++++++--
 eth2/utils/ssz/src/cached_tree_hash/impls.rs |  6 +--
 eth2/utils/ssz/src/cached_tree_hash/tests.rs | 44 ++++++++------------
 3 files changed, 35 insertions(+), 32 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index 3b900e503..83b516ac7 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -25,13 +25,13 @@ pub enum Error {
 pub trait CachedTreeHash {
     type Item: CachedTreeHash;
 
-    fn build_cache(&self) -> Result<TreeHashCache, Error>;
+    fn leaves_and_subtrees(&self) -> Vec<u8>;
 
     /// Return the number of bytes when this element is encoded as raw SSZ _without_ length
     /// prefixes.
     fn num_bytes(&self) -> usize;
 
-    fn offset_handler(&self, initial_offset: usize) -> Result<OffsetHandler, Error>;
+    fn offsets(&self) -> Result<Vec<usize>, Error>;
 
     fn num_child_nodes(&self) -> usize;
 
@@ -60,13 +60,17 @@ impl TreeHashCache {
     where
         T: CachedTreeHash,
     {
-        item.build_cache()
+        Self::from_leaves_and_subtrees(item.leaves_and_subtrees(), OffsetHandler::new(item, 0)?)
     }
 
     pub fn from_leaves_and_subtrees(
         mut leaves_and_subtrees: Vec<u8>,
         offset_handler: OffsetHandler,
     ) -> Result<Self, Error> {
+        // Pad the leaves with zeros if the number of immediate leaf-nodes (without recursing into
+        // sub-trees) is not an even power-of-two.
+        pad_for_leaf_count(offset_handler.num_leaf_nodes, &mut leaves_and_subtrees);
+
         // Allocate enough bytes to store the internal nodes and the leaves and subtrees, then fill
         // all the to-be-built internal nodes with zeros and append the leaves and subtrees.
         let internal_node_bytes = offset_handler.num_internal_nodes * BYTES_PER_CHUNK;
@@ -209,6 +213,13 @@ pub struct OffsetHandler {
 }
 
 impl OffsetHandler {
+    pub fn new<T>(item: &T, initial_offset: usize) -> Result<Self, Error>
+    where
+        T: CachedTreeHash,
+    {
+        Self::from_lengths(initial_offset, item.offsets()?)
+    }
+
     fn from_lengths(offset: usize, mut lengths: Vec<usize>) -> Result<Self, Error> {
         // Extend it to the next power-of-two, if it is not already.
         let num_leaf_nodes = if lengths.len().is_power_of_two() {
diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index 9c0a8ec6d..54a690c6d 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -4,15 +4,15 @@ use crate::{ssz_encode, Encodable};
 impl CachedTreeHash for u64 {
     type Item = Self;
 
-    fn build_cache(&self) -> Result<TreeHashCache, Error> {
-        TreeHashCache::from_bytes(merkleize(ssz_encode(self)))
+    fn leaves_and_subtrees(&self) -> Vec<u8> {
+        merkleize(ssz_encode(self))
     }
 
     fn num_bytes(&self) -> usize {
         8
     }
 
-    fn offset_handler(&self, _initial_offset: usize) -> Result<OffsetHandler, Error> {
+    fn offsets(&self) -> Result<Vec<usize>, Error> {
         Err(Error::ShouldNotProduceOffsetHandler)
     }
 
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index a6be7f9ae..f6c52ef8f 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -12,19 +12,15 @@ pub struct Inner {
 impl CachedTreeHash for Inner {
     type Item = Self;
 
-    fn build_cache(&self) -> Result<TreeHashCache, Error> {
-        let offset_handler = self.offset_handler(0)?;
-
+    fn leaves_and_subtrees(&self) -> Vec<u8> {
         let mut leaves_and_subtrees = vec![];
 
-        leaves_and_subtrees.append(&mut self.a.build_cache()?.into());
-        leaves_and_subtrees.append(&mut self.b.build_cache()?.into());
-        leaves_and_subtrees.append(&mut self.c.build_cache()?.into());
-        leaves_and_subtrees.append(&mut self.d.build_cache()?.into());
+        leaves_and_subtrees.append(&mut self.a.leaves_and_subtrees());
+        leaves_and_subtrees.append(&mut self.b.leaves_and_subtrees());
+        leaves_and_subtrees.append(&mut self.c.leaves_and_subtrees());
+        leaves_and_subtrees.append(&mut self.d.leaves_and_subtrees());
 
-        pad_for_leaf_count(offset_handler.num_leaf_nodes, &mut leaves_and_subtrees);
-
-        TreeHashCache::from_leaves_and_subtrees(leaves_and_subtrees, self.offset_handler(0)?)
+        leaves_and_subtrees
     }
 
     fn num_bytes(&self) -> usize {
@@ -38,7 +34,7 @@ impl CachedTreeHash for Inner {
         bytes
     }
 
-    fn offset_handler(&self, initial_offset: usize) -> Result<OffsetHandler, Error> {
+    fn offsets(&self) -> Result<Vec<usize>, Error> {
         let mut offsets = vec![];
 
         offsets.push(self.a.num_child_nodes() + 1);
@@ -46,7 +42,7 @@ impl CachedTreeHash for Inner {
         offsets.push(self.c.num_child_nodes() + 1);
         offsets.push(self.d.num_child_nodes() + 1);
 
-        OffsetHandler::from_lengths(initial_offset, offsets)
+        Ok(offsets)
     }
 
     fn num_child_nodes(&self) -> usize {
@@ -67,7 +63,7 @@ impl CachedTreeHash for Inner {
         cache: &mut TreeHashCache,
         chunk: usize,
     ) -> Result<usize, Error> {
-        let offset_handler = self.offset_handler(chunk)?;
+        let offset_handler = OffsetHandler::new(self, chunk)?;
 
         // Skip past the internal nodes and update any changed leaf nodes.
         {
@@ -98,18 +94,14 @@ pub struct Outer {
 impl CachedTreeHash for Outer {
     type Item = Self;
 
-    fn build_cache(&self) -> Result<TreeHashCache, Error> {
-        let offset_handler = self.offset_handler(0)?;
-
+    fn leaves_and_subtrees(&self) -> Vec<u8> {
         let mut leaves_and_subtrees = vec![];
 
-        leaves_and_subtrees.append(&mut self.a.build_cache()?.into());
-        leaves_and_subtrees.append(&mut self.b.build_cache()?.into());
-        leaves_and_subtrees.append(&mut self.c.build_cache()?.into());
+        leaves_and_subtrees.append(&mut self.a.leaves_and_subtrees());
+        leaves_and_subtrees.append(&mut self.b.leaves_and_subtrees());
+        leaves_and_subtrees.append(&mut self.c.leaves_and_subtrees());
 
-        pad_for_leaf_count(offset_handler.num_leaf_nodes, &mut leaves_and_subtrees);
-
-        TreeHashCache::from_leaves_and_subtrees(leaves_and_subtrees, self.offset_handler(0)?)
+        leaves_and_subtrees
     }
 
     fn num_bytes(&self) -> usize {
@@ -131,14 +123,14 @@ impl CachedTreeHash for Outer {
         num_nodes(leaves) + children - 1
     }
 
-    fn offset_handler(&self, initial_offset: usize) -> Result<OffsetHandler, Error> {
+    fn offsets(&self) -> Result<Vec<usize>, Error> {
         let mut offsets = vec![];
 
         offsets.push(self.a.num_child_nodes() + 1);
         offsets.push(self.b.num_child_nodes() + 1);
         offsets.push(self.c.num_child_nodes() + 1);
 
-        OffsetHandler::from_lengths(initial_offset, offsets)
+        Ok(offsets)
     }
 
     fn cached_hash_tree_root(
@@ -147,7 +139,7 @@ impl CachedTreeHash for Outer {
         cache: &mut TreeHashCache,
         chunk: usize,
     ) -> Result<usize, Error> {
-        let offset_handler = self.offset_handler(chunk)?;
+        let offset_handler = OffsetHandler::new(self, chunk)?;
 
         // Skip past the internal nodes and update any changed leaf nodes.
         {
@@ -308,7 +300,7 @@ fn outer_builds() {
     // Generate reference data.
     let mut data = vec![];
     data.append(&mut int_to_bytes32(0));
-    let inner_bytes: Vec<u8> = inner.build_cache().unwrap().into();
+    let inner_bytes: Vec<u8> = TreeHashCache::new(&inner).unwrap().into();
     data.append(&mut int_to_bytes32(5));
 
     let leaves = vec![

From fc17d5fea4c46ae35ab11dc9c6b1ae7a93dfaccb Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Fri, 29 Mar 2019 14:37:27 +1100
Subject: [PATCH 019/137] Fix failing tree hash tests

---
 eth2/utils/ssz/src/cached_tree_hash.rs       | 72 ++++++++++++--------
 eth2/utils/ssz/src/cached_tree_hash/impls.rs |  4 +-
 eth2/utils/ssz/src/cached_tree_hash/tests.rs | 38 ++++++-----
 3 files changed, 68 insertions(+), 46 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index 83b516ac7..510185b40 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -13,9 +13,9 @@ const MERKLE_HASH_CHUNCK: usize = 2 * BYTES_PER_CHUNK;
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum Error {
-    LeavesAndSubtreesIncomplete(usize),
     ShouldNotProduceOffsetHandler,
     NoFirstNode,
+    NoBytesForRoot,
     BytesAreNotEvenChunks(usize),
     NoModifiedFieldForChunk(usize),
     NoBytesForChunk(usize),
@@ -25,7 +25,7 @@ pub enum Error {
 pub trait CachedTreeHash {
     type Item: CachedTreeHash;
 
-    fn leaves_and_subtrees(&self) -> Vec<u8>;
+    fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error>;
 
     /// Return the number of bytes when this element is encoded as raw SSZ _without_ length
     /// prefixes.
@@ -60,42 +60,44 @@ impl TreeHashCache {
     where
         T: CachedTreeHash,
     {
-        Self::from_leaves_and_subtrees(item.leaves_and_subtrees(), OffsetHandler::new(item, 0)?)
+        item.build_tree_hash_cache()
     }
 
-    pub fn from_leaves_and_subtrees(
-        mut leaves_and_subtrees: Vec<u8>,
-        offset_handler: OffsetHandler,
-    ) -> Result<Self, Error> {
-        // Pad the leaves with zeros if the number of immediate leaf-nodes (without recursing into
-        // sub-trees) is not an even power-of-two.
-        pad_for_leaf_count(offset_handler.num_leaf_nodes, &mut leaves_and_subtrees);
+    pub fn from_leaves_and_subtrees<T>(
+        item: &T,
+        leaves_and_subtrees: Vec<Self>,
+    ) -> Result<Self, Error>
+    where
+        T: CachedTreeHash,
+    {
+        let offset_handler = OffsetHandler::new(item, 0)?;
+
+        // Note how many leaves were provided. If is not a power-of-two, we'll need to pad it out
+        // later.
+        let num_provided_leaf_nodes = leaves_and_subtrees.len();
 
         // Allocate enough bytes to store the internal nodes and the leaves and subtrees, then fill
         // all the to-be-built internal nodes with zeros and append the leaves and subtrees.
         let internal_node_bytes = offset_handler.num_internal_nodes * BYTES_PER_CHUNK;
-        let mut cache = Vec::with_capacity(internal_node_bytes + leaves_and_subtrees.len());
+        let leaves_and_subtrees_bytes = leaves_and_subtrees
+            .iter()
+            .fold(0, |acc, t| acc + t.bytes_len());
+        let mut cache = Vec::with_capacity(leaves_and_subtrees_bytes + internal_node_bytes);
         cache.resize(internal_node_bytes, 0);
-        cache.append(&mut leaves_and_subtrees);
 
-        dbg!(cache.len() / BYTES_PER_CHUNK);
+        // Allocate enough bytes to store all the leaves.
+        let mut leaves = Vec::with_capacity(offset_handler.num_leaf_nodes * HASHSIZE);
 
-        // Concat all the leaves into one big byte array, ready for `merkleize`.
-        let mut leaves = vec![];
-        for leaf_chunk in offset_handler.iter_leaf_nodes() {
-            let start = leaf_chunk * BYTES_PER_CHUNK;
-            let end = start + BYTES_PER_CHUNK;
-
-            dbg!(end);
-            dbg!(cache.len());
-
-            leaves.extend_from_slice(
-                cache
-                    .get(start..end)
-                    .ok_or_else(|| Error::LeavesAndSubtreesIncomplete(*leaf_chunk))?,
-            );
+        // Iterate through all of the leaves/subtrees, adding their root as a leaf node and then
+        // concatenating their merkle trees.
+        for t in leaves_and_subtrees {
+            leaves.append(&mut t.root()?);
+            cache.append(&mut t.into_merkle_tree());
         }
 
+        // Pad the leaves to an even power-of-two, using zeros.
+        pad_for_leaf_count(num_provided_leaf_nodes, &mut cache);
+
         // Merkleize the leaves, then split the leaf nodes off them. Then, replace all-zeros
         // internal nodes created earlier with the internal nodes generated by `merkleize`.
         let mut merkleized = merkleize(leaves);
@@ -108,6 +110,17 @@ impl TreeHashCache {
         })
     }
 
+    pub fn bytes_len(&self) -> usize {
+        self.cache.len()
+    }
+
+    pub fn root(&self) -> Result<Vec<u8>, Error> {
+        self.cache
+            .get(0..HASHSIZE)
+            .ok_or_else(|| Error::NoBytesForRoot)
+            .and_then(|slice| Ok(slice.to_vec()))
+    }
+
     pub fn from_bytes(bytes: Vec<u8>) -> Result<Self, Error> {
         if bytes.len() % BYTES_PER_CHUNK > 0 {
             return Err(Error::BytesAreNotEvenChunks(bytes.len()));
@@ -195,6 +208,10 @@ impl TreeHashCache {
 
         Ok(hash(children))
     }
+
+    pub fn into_merkle_tree(self) -> Vec<u8> {
+        self.cache
+    }
 }
 
 fn children(parent: usize) -> (usize, usize) {
@@ -205,6 +222,7 @@ fn num_nodes(num_leaves: usize) -> usize {
     2 * num_leaves - 1
 }
 
+#[derive(Debug)]
 pub struct OffsetHandler {
     num_internal_nodes: usize,
     num_leaf_nodes: usize,
diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index 54a690c6d..012a4a8be 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -4,8 +4,8 @@ use crate::{ssz_encode, Encodable};
 impl CachedTreeHash for u64 {
     type Item = Self;
 
-    fn leaves_and_subtrees(&self) -> Vec<u8> {
-        merkleize(ssz_encode(self))
+    fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
+        Ok(TreeHashCache::from_bytes(merkleize(ssz_encode(self)))?)
     }
 
     fn num_bytes(&self) -> usize {
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index f6c52ef8f..0593b2bae 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -12,15 +12,18 @@ pub struct Inner {
 impl CachedTreeHash for Inner {
     type Item = Self;
 
-    fn leaves_and_subtrees(&self) -> Vec<u8> {
-        let mut leaves_and_subtrees = vec![];
+    fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
+        let tree = TreeHashCache::from_leaves_and_subtrees(
+            self,
+            vec![
+                self.a.build_tree_hash_cache()?,
+                self.b.build_tree_hash_cache()?,
+                self.c.build_tree_hash_cache()?,
+                self.d.build_tree_hash_cache()?,
+            ],
+        )?;
 
-        leaves_and_subtrees.append(&mut self.a.leaves_and_subtrees());
-        leaves_and_subtrees.append(&mut self.b.leaves_and_subtrees());
-        leaves_and_subtrees.append(&mut self.c.leaves_and_subtrees());
-        leaves_and_subtrees.append(&mut self.d.leaves_and_subtrees());
-
-        leaves_and_subtrees
+        Ok(tree)
     }
 
     fn num_bytes(&self) -> usize {
@@ -94,14 +97,17 @@ pub struct Outer {
 impl CachedTreeHash for Outer {
     type Item = Self;
 
-    fn leaves_and_subtrees(&self) -> Vec<u8> {
-        let mut leaves_and_subtrees = vec![];
+    fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
+        let tree = TreeHashCache::from_leaves_and_subtrees(
+            self,
+            vec![
+                self.a.build_tree_hash_cache()?,
+                self.b.build_tree_hash_cache()?,
+                self.c.build_tree_hash_cache()?,
+            ],
+        )?;
 
-        leaves_and_subtrees.append(&mut self.a.leaves_and_subtrees());
-        leaves_and_subtrees.append(&mut self.b.leaves_and_subtrees());
-        leaves_and_subtrees.append(&mut self.c.leaves_and_subtrees());
-
-        leaves_and_subtrees
+        Ok(tree)
     }
 
     fn num_bytes(&self) -> usize {
@@ -193,10 +199,8 @@ fn partial_modification_to_inner_struct() {
         ..original_outer.clone()
     };
 
-    println!("AAAAAAAAA");
     // Perform a differential hash
     let mut cache_struct = TreeHashCache::new(&original_outer).unwrap();
-    println!("BBBBBBBBBB");
 
     modified_outer
         .cached_hash_tree_root(&original_outer, &mut cache_struct, 0)

From 56fe15625bf349b84e7c40b84c9fcb13e8112ec8 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 10 Apr 2019 15:47:42 +1000
Subject: [PATCH 020/137] Allow for building cached vec

---
 eth2/utils/ssz/src/cached_tree_hash.rs       | 12 +++
 eth2/utils/ssz/src/cached_tree_hash/impls.rs | 74 +++++++++++++++---
 eth2/utils/ssz/src/cached_tree_hash/tests.rs | 80 +++++++++++++++++++-
 3 files changed, 153 insertions(+), 13 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index 510185b40..ba55fbf1b 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -22,9 +22,18 @@ pub enum Error {
     NoChildrenForHashing((usize, usize)),
 }
 
+#[derive(Debug, PartialEq, Clone)]
+pub enum ItemType {
+    Basic,
+    List,
+    Composite,
+}
+
 pub trait CachedTreeHash {
     type Item: CachedTreeHash;
 
+    fn item_type() -> ItemType;
+
     fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error>;
 
     /// Return the number of bytes when this element is encoded as raw SSZ _without_ length
@@ -35,6 +44,8 @@ pub trait CachedTreeHash {
 
     fn num_child_nodes(&self) -> usize;
 
+    fn packed_encoding(&self) -> Vec<u8>;
+
     fn cached_hash_tree_root(
         &self,
         other: &Self::Item,
@@ -101,6 +112,7 @@ impl TreeHashCache {
         // Merkleize the leaves, then split the leaf nodes off them. Then, replace all-zeros
         // internal nodes created earlier with the internal nodes generated by `merkleize`.
         let mut merkleized = merkleize(leaves);
+        dbg!(&merkleized);
         merkleized.split_off(internal_node_bytes);
         cache.splice(0..internal_node_bytes, merkleized);
 
diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index 012a4a8be..e088d481d 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -4,6 +4,10 @@ use crate::{ssz_encode, Encodable};
 impl CachedTreeHash for u64 {
     type Item = Self;
 
+    fn item_type() -> ItemType {
+        ItemType::Basic
+    }
+
     fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
         Ok(TreeHashCache::from_bytes(merkleize(ssz_encode(self)))?)
     }
@@ -20,6 +24,10 @@ impl CachedTreeHash for u64 {
         0
     }
 
+    fn packed_encoding(&self) -> Vec<u8> {
+        ssz_encode(self)
+    }
+
     fn cached_hash_tree_root(
         &self,
         other: &Self,
@@ -35,38 +43,73 @@ impl CachedTreeHash for u64 {
     }
 }
 
-/*
 impl<T> CachedTreeHash for Vec<T>
 where
-    T: CachedTreeHash + Encodable,
+    T: CachedTreeHash,
 {
     type Item = Self;
 
-    fn build_cache_bytes(&self) -> Vec<u8> {
-        let num_packed_bytes = self.num_bytes();
-        let num_leaves = num_sanitized_leaves(num_packed_bytes);
+    fn item_type() -> ItemType {
+        ItemType::List
+    }
 
-        let mut packed = Vec::with_capacity(num_leaves * HASHSIZE);
+    fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
+        match T::item_type() {
+            ItemType::Basic => {
+                let num_packed_bytes = self.num_bytes();
+                let num_leaves = num_sanitized_leaves(num_packed_bytes);
+
+                let mut packed = Vec::with_capacity(num_leaves * HASHSIZE);
+
+                for item in self {
+                    packed.append(&mut item.packed_encoding());
+                }
+
+                let packed = sanitise_bytes(packed);
+
+                TreeHashCache::from_bytes(merkleize(packed))
+            }
+            ItemType::Composite | ItemType::List => {
+                let subtrees = self
+                    .iter()
+                    .map(|item| TreeHashCache::new(item))
+                    .collect::<Result<Vec<TreeHashCache>, _>>()?;
+
+                TreeHashCache::from_leaves_and_subtrees(self, subtrees)
+            }
+        }
+    }
+
+    fn offsets(&self) -> Result<Vec<usize>, Error> {
+        let mut offsets = vec![];
 
         for item in self {
-            packed.append(&mut ssz_encode(item));
+            offsets.push(item.offsets()?.iter().sum())
         }
 
-        let packed = sanitise_bytes(packed);
+        Ok(offsets)
+    }
 
-        merkleize(packed)
+    fn num_child_nodes(&self) -> usize {
+        // TODO
+        42
     }
 
     fn num_bytes(&self) -> usize {
         self.iter().fold(0, |acc, item| acc + item.num_bytes())
     }
 
+    fn packed_encoding(&self) -> Vec<u8> {
+        panic!("List should never be packed")
+    }
+
     fn cached_hash_tree_root(
         &self,
         other: &Self::Item,
         cache: &mut TreeHashCache,
         chunk: usize,
-    ) -> Option<usize> {
+    ) -> Result<usize, Error> {
+        /*
         let num_packed_bytes = self.num_bytes();
         let num_leaves = num_sanitized_leaves(num_packed_bytes);
 
@@ -103,6 +146,17 @@ where
         }
 
         Some(chunk + num_nodes)
+        */
+        // TODO
+        Ok(42)
     }
 }
+
+/*
+fn get_packed_leaves<T>(vec: Vec<T>) -> Vec<u8>
+where
+    T: Encodable,
+{
+    //
+}
 */
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index 0593b2bae..8124a8dd8 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -1,5 +1,6 @@
 use super::*;
-use int_to_bytes::int_to_bytes32;
+use crate::Encodable;
+use int_to_bytes::{int_to_bytes32, int_to_bytes8};
 
 #[derive(Clone)]
 pub struct Inner {
@@ -12,6 +13,10 @@ pub struct Inner {
 impl CachedTreeHash for Inner {
     type Item = Self;
 
+    fn item_type() -> ItemType {
+        ItemType::Composite
+    }
+
     fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
         let tree = TreeHashCache::from_leaves_and_subtrees(
             self,
@@ -60,6 +65,10 @@ impl CachedTreeHash for Inner {
         num_nodes(leaves) + children - 1
     }
 
+    fn packed_encoding(&self) -> Vec<u8> {
+        panic!("Struct should never be packed")
+    }
+
     fn cached_hash_tree_root(
         &self,
         other: &Self,
@@ -97,6 +106,10 @@ pub struct Outer {
 impl CachedTreeHash for Outer {
     type Item = Self;
 
+    fn item_type() -> ItemType {
+        ItemType::Composite
+    }
+
     fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
         let tree = TreeHashCache::from_leaves_and_subtrees(
             self,
@@ -139,6 +152,10 @@ impl CachedTreeHash for Outer {
         Ok(offsets)
     }
 
+    fn packed_encoding(&self) -> Vec<u8> {
+        panic!("Struct should never be packed")
+    }
+
     fn cached_hash_tree_root(
         &self,
         other: &Self,
@@ -371,6 +388,64 @@ fn large_vec_of_u64_builds() {
 
     assert_eq!(expected, cache);
 }
+*/
+
+#[test]
+fn vec_of_inner_builds() {
+    let numbers: Vec<u64> = (0..12).collect();
+
+    let mut leaves = vec![];
+    let mut full_bytes = vec![];
+
+    for n in numbers.chunks(4) {
+        let mut merkle = merkleize(join(vec![
+            int_to_bytes32(n[0]),
+            int_to_bytes32(n[1]),
+            int_to_bytes32(n[2]),
+            int_to_bytes32(n[3]),
+        ]));
+        leaves.append(&mut merkle[0..HASHSIZE].to_vec());
+        full_bytes.append(&mut merkle);
+    }
+
+    let mut expected = merkleize(leaves);
+    expected.splice(3 * HASHSIZE.., full_bytes);
+    expected.append(&mut vec![0; HASHSIZE]);
+
+    let my_vec = vec![
+        Inner {
+            a: 0,
+            b: 1,
+            c: 2,
+            d: 3,
+        },
+        Inner {
+            a: 4,
+            b: 5,
+            c: 6,
+            d: 7,
+        },
+        Inner {
+            a: 8,
+            b: 9,
+            c: 10,
+            d: 11,
+        },
+    ];
+
+    let cache: Vec<u8> = TreeHashCache::new(&my_vec).unwrap().into();
+
+    assert_trees_eq(&expected, &cache);
+}
+
+/// Provides detailed assertions when comparing merkle trees.
+fn assert_trees_eq(a: &[u8], b: &[u8]) {
+    assert_eq!(a.len(), b.len(), "Byte lens different");
+    for i in 0..a.len() / HASHSIZE {
+        let range = i * HASHSIZE..(i + 1) * HASHSIZE;
+        assert_eq!(a[range.clone()], b[range], "Chunk {} different", i);
+    }
+}
 
 #[test]
 fn vec_of_u64_builds() {
@@ -387,11 +462,10 @@ fn vec_of_u64_builds() {
 
     let my_vec = vec![1, 2, 3, 4, 5];
 
-    let cache = my_vec.build_cache_bytes();
+    let cache: Vec<u8> = TreeHashCache::new(&my_vec).unwrap().into();
 
     assert_eq!(expected, cache);
 }
-*/
 
 #[test]
 fn merkleize_odd() {

From e5783d43a9c3bb8d389737b9ee3b676be70062b6 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 10 Apr 2019 16:59:14 +1000
Subject: [PATCH 021/137] First passing vec modified cache test

---
 eth2/utils/ssz/src/cached_tree_hash.rs       |  29 ++++-
 eth2/utils/ssz/src/cached_tree_hash/impls.rs | 110 +++++++++----------
 eth2/utils/ssz/src/cached_tree_hash/tests.rs |  18 ++-
 3 files changed, 90 insertions(+), 67 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index ba55fbf1b..e7f2114e4 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -46,6 +46,8 @@ pub trait CachedTreeHash {
 
     fn packed_encoding(&self) -> Vec<u8>;
 
+    fn packing_factor() -> usize;
+
     fn cached_hash_tree_root(
         &self,
         other: &Self::Item,
@@ -165,16 +167,19 @@ impl TreeHashCache {
         self.cache.splice(byte_start..byte_end, replace_with)
     }
 
-    pub fn maybe_update_chunk(&mut self, chunk: usize, to: &[u8]) -> Option<()> {
+    pub fn maybe_update_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> {
         let start = chunk * BYTES_PER_CHUNK;
         let end = start + BYTES_PER_CHUNK;
 
         if !self.chunk_equals(chunk, to)? {
-            self.cache.get_mut(start..end)?.copy_from_slice(to);
+            self.cache
+                .get_mut(start..end)
+                .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))?
+                .copy_from_slice(to);
             self.chunk_modified[chunk] = true;
         }
 
-        Some(())
+        Ok(())
     }
 
     pub fn modify_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> {
@@ -191,11 +196,25 @@ impl TreeHashCache {
         Ok(())
     }
 
-    pub fn chunk_equals(&mut self, chunk: usize, other: &[u8]) -> Option<bool> {
+    pub fn chunk_equals(&mut self, chunk: usize, other: &[u8]) -> Result<bool, Error> {
         let start = chunk * BYTES_PER_CHUNK;
         let end = start + BYTES_PER_CHUNK;
 
-        Some(self.cache.get(start..end)? == other)
+        Ok(self
+            .cache
+            .get(start..end)
+            .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))?
+            == other)
+    }
+
+    pub fn set_changed(&mut self, chunk: usize, to: bool) -> Result<(), Error> {
+        if chunk < self.chunk_modified.len() {
+            self.chunk_modified[chunk] = to;
+
+            Ok(())
+        } else {
+            Err(Error::NoModifiedFieldForChunk(chunk))
+        }
     }
 
     pub fn changed(&self, chunk: usize) -> Result<bool, Error> {
diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index e088d481d..621c5d02b 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -28,6 +28,10 @@ impl CachedTreeHash for u64 {
         ssz_encode(self)
     }
 
+    fn packing_factor() -> usize {
+        32 / 8
+    }
+
     fn cached_hash_tree_root(
         &self,
         other: &Self,
@@ -55,20 +59,7 @@ where
 
     fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
         match T::item_type() {
-            ItemType::Basic => {
-                let num_packed_bytes = self.num_bytes();
-                let num_leaves = num_sanitized_leaves(num_packed_bytes);
-
-                let mut packed = Vec::with_capacity(num_leaves * HASHSIZE);
-
-                for item in self {
-                    packed.append(&mut item.packed_encoding());
-                }
-
-                let packed = sanitise_bytes(packed);
-
-                TreeHashCache::from_bytes(merkleize(packed))
-            }
+            ItemType::Basic => TreeHashCache::from_bytes(merkleize(get_packed_leaves(self))),
             ItemType::Composite | ItemType::List => {
                 let subtrees = self
                     .iter()
@@ -81,11 +72,18 @@ where
     }
 
     fn offsets(&self) -> Result<Vec<usize>, Error> {
-        let mut offsets = vec![];
+        let offsets = match T::item_type() {
+            ItemType::Basic => vec![1; self.len() / T::packing_factor()],
+            ItemType::Composite | ItemType::List => {
+                let mut offsets = vec![];
 
-        for item in self {
-            offsets.push(item.offsets()?.iter().sum())
-        }
+                for item in self {
+                    offsets.push(item.offsets()?.iter().sum())
+                }
+
+                offsets
+            }
+        };
 
         Ok(offsets)
     }
@@ -103,60 +101,58 @@ where
         panic!("List should never be packed")
     }
 
+    fn packing_factor() -> usize {
+        1
+    }
+
     fn cached_hash_tree_root(
         &self,
         other: &Self::Item,
         cache: &mut TreeHashCache,
         chunk: usize,
     ) -> Result<usize, Error> {
-        /*
-        let num_packed_bytes = self.num_bytes();
-        let num_leaves = num_sanitized_leaves(num_packed_bytes);
+        let offset_handler = OffsetHandler::new(self, chunk)?;
 
-        if num_leaves != num_sanitized_leaves(other.num_bytes()) {
-            panic!("Need to handle a change in leaf count");
+        match T::item_type() {
+            ItemType::Basic => {
+                let leaves = get_packed_leaves(self);
+
+                for (i, chunk) in offset_handler.iter_leaf_nodes().enumerate() {
+                    if let Some(latest) = leaves.get(i * HASHSIZE..(i + 1) * HASHSIZE) {
+                        if !cache.chunk_equals(*chunk, latest)? {
+                            dbg!(chunk);
+                            cache.set_changed(*chunk, true)?;
+                        }
+                    }
+                }
+                let first_leaf_chunk = offset_handler.first_leaf_node()?;
+                cache.chunk_splice(first_leaf_chunk..offset_handler.next_node, leaves);
+            }
+            _ => panic!("not implemented"),
         }
 
-        let mut packed = Vec::with_capacity(num_leaves * HASHSIZE);
-
-        // TODO: try and avoid fully encoding the whole list
-        for item in self {
-            packed.append(&mut ssz_encode(item));
-        }
-
-        let packed = sanitise_bytes(packed);
-
-        let num_nodes = num_nodes(num_leaves);
-        let num_internal_nodes = num_nodes - num_leaves;
-
-        {
-            let mut chunk = chunk + num_internal_nodes;
-            for new_chunk_bytes in packed.chunks(HASHSIZE) {
-                cache.maybe_update_chunk(chunk, new_chunk_bytes)?;
-                chunk += 1;
+        for (&parent, children) in offset_handler.iter_internal_nodes().rev() {
+            if cache.either_modified(children)? {
+                cache.modify_chunk(parent, &cache.hash_children(children)?)?;
             }
         }
 
-        // Iterate backwards through the internal nodes, rehashing any node where it's children
-        // have changed.
-        for chunk in (chunk..chunk + num_internal_nodes).into_iter().rev() {
-            if cache.children_modified(chunk)? {
-                cache.modify_chunk(chunk, &cache.hash_children(chunk)?)?;
-            }
-        }
-
-        Some(chunk + num_nodes)
-        */
-        // TODO
-        Ok(42)
+        Ok(offset_handler.next_node())
     }
 }
 
-/*
-fn get_packed_leaves<T>(vec: Vec<T>) -> Vec<u8>
+fn get_packed_leaves<T>(vec: &Vec<T>) -> Vec<u8>
 where
-    T: Encodable,
+    T: CachedTreeHash,
 {
-    //
+    let num_packed_bytes = vec.num_bytes();
+    let num_leaves = num_sanitized_leaves(num_packed_bytes);
+
+    let mut packed = Vec::with_capacity(num_leaves * HASHSIZE);
+
+    for item in vec {
+        packed.append(&mut item.packed_encoding());
+    }
+
+    sanitise_bytes(packed)
 }
-*/
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index 8124a8dd8..e65c87bbd 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -69,6 +69,10 @@ impl CachedTreeHash for Inner {
         panic!("Struct should never be packed")
     }
 
+    fn packing_factor() -> usize {
+        1
+    }
+
     fn cached_hash_tree_root(
         &self,
         other: &Self,
@@ -156,6 +160,10 @@ impl CachedTreeHash for Outer {
         panic!("Struct should never be packed")
     }
 
+    fn packing_factor() -> usize {
+        1
+    }
+
     fn cached_hash_tree_root(
         &self,
         other: &Self,
@@ -339,7 +347,6 @@ fn outer_builds() {
     assert_eq!(merkle, cache);
 }
 
-/*
 #[test]
 fn partial_modification_u64_vec() {
     let n: u64 = 50;
@@ -347,7 +354,7 @@ fn partial_modification_u64_vec() {
     let original_vec: Vec<u64> = (0..n).collect();
 
     // Generate initial cache.
-    let original_cache = original_vec.build_cache_bytes();
+    let original_cache: Vec<u8> = TreeHashCache::new(&original_vec).unwrap().into();
 
     // Modify the vec
     let mut modified_vec = original_vec.clone();
@@ -355,7 +362,9 @@ fn partial_modification_u64_vec() {
 
     // Perform a differential hash
     let mut cache_struct = TreeHashCache::from_bytes(original_cache.clone()).unwrap();
-    modified_vec.cached_hash_tree_root(&original_vec, &mut cache_struct, 0);
+    modified_vec
+        .cached_hash_tree_root(&original_vec, &mut cache_struct, 0)
+        .unwrap();
     let modified_cache: Vec<u8> = cache_struct.into();
 
     // Generate reference data.
@@ -376,7 +385,7 @@ fn large_vec_of_u64_builds() {
     let my_vec: Vec<u64> = (0..n).collect();
 
     // Generate function output.
-    let cache = my_vec.build_cache_bytes();
+    let cache: Vec<u8> = TreeHashCache::new(&my_vec).unwrap().into();
 
     // Generate reference data.
     let mut data = vec![];
@@ -388,7 +397,6 @@ fn large_vec_of_u64_builds() {
 
     assert_eq!(expected, cache);
 }
-*/
 
 #[test]
 fn vec_of_inner_builds() {

From 0c0eebd7740fe9e021e83a799696fbdffdb93c4b Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Thu, 11 Apr 2019 12:57:36 +1000
Subject: [PATCH 022/137] Add progress on variable list hashing

---
 eth2/utils/ssz/src/cached_tree_hash.rs       | 30 +++----
 eth2/utils/ssz/src/cached_tree_hash/impls.rs | 45 +++++++----
 eth2/utils/ssz/src/cached_tree_hash/tests.rs | 84 +++++++++++++++++---
 3 files changed, 115 insertions(+), 44 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index e7f2114e4..0889718a2 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -1,4 +1,5 @@
 use hashing::hash;
+use std::fmt::Debug;
 use std::iter::IntoIterator;
 use std::iter::Iterator;
 use std::ops::Range;
@@ -29,9 +30,8 @@ pub enum ItemType {
     Composite,
 }
 
-pub trait CachedTreeHash {
-    type Item: CachedTreeHash;
-
+// TODO: remove debug requirement.
+pub trait CachedTreeHash<Item>: Debug {
     fn item_type() -> ItemType;
 
     fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error>;
@@ -50,7 +50,7 @@ pub trait CachedTreeHash {
 
     fn cached_hash_tree_root(
         &self,
-        other: &Self::Item,
+        other: &Item,
         cache: &mut TreeHashCache,
         chunk: usize,
     ) -> Result<usize, Error>;
@@ -71,7 +71,7 @@ impl Into<Vec<u8>> for TreeHashCache {
 impl TreeHashCache {
     pub fn new<T>(item: &T) -> Result<Self, Error>
     where
-        T: CachedTreeHash,
+        T: CachedTreeHash<T>,
     {
         item.build_tree_hash_cache()
     }
@@ -81,7 +81,7 @@ impl TreeHashCache {
         leaves_and_subtrees: Vec<Self>,
     ) -> Result<Self, Error>
     where
-        T: CachedTreeHash,
+        T: CachedTreeHash<T>,
     {
         let offset_handler = OffsetHandler::new(item, 0)?;
 
@@ -114,7 +114,6 @@ impl TreeHashCache {
         // Merkleize the leaves, then split the leaf nodes off them. Then, replace all-zeros
         // internal nodes created earlier with the internal nodes generated by `merkleize`.
         let mut merkleized = merkleize(leaves);
-        dbg!(&merkleized);
         merkleized.split_off(internal_node_bytes);
         cache.splice(0..internal_node_bytes, merkleized);
 
@@ -207,16 +206,6 @@ impl TreeHashCache {
             == other)
     }
 
-    pub fn set_changed(&mut self, chunk: usize, to: bool) -> Result<(), Error> {
-        if chunk < self.chunk_modified.len() {
-            self.chunk_modified[chunk] = to;
-
-            Ok(())
-        } else {
-            Err(Error::NoModifiedFieldForChunk(chunk))
-        }
-    }
-
     pub fn changed(&self, chunk: usize) -> Result<bool, Error> {
         self.chunk_modified
             .get(chunk)
@@ -256,7 +245,7 @@ fn num_nodes(num_leaves: usize) -> usize {
 #[derive(Debug)]
 pub struct OffsetHandler {
     num_internal_nodes: usize,
-    num_leaf_nodes: usize,
+    pub num_leaf_nodes: usize,
     next_node: usize,
     offsets: Vec<usize>,
 }
@@ -264,7 +253,7 @@ pub struct OffsetHandler {
 impl OffsetHandler {
     pub fn new<T>(item: &T, initial_offset: usize) -> Result<Self, Error>
     where
-        T: CachedTreeHash,
+        T: CachedTreeHash<T>,
     {
         Self::from_lengths(initial_offset, item.offsets()?)
     }
@@ -314,6 +303,8 @@ impl OffsetHandler {
         self.next_node
     }
 
+    /// Returns an iterator visiting each internal node, providing the left and right child chunks
+    /// for the node.
     pub fn iter_internal_nodes<'a>(
         &'a self,
     ) -> impl DoubleEndedIterator<Item = (&'a usize, (&'a usize, &'a usize))> {
@@ -328,6 +319,7 @@ impl OffsetHandler {
         })
     }
 
+    /// Returns an iterator visiting each leaf node, providing the chunk for that node.
     pub fn iter_leaf_nodes<'a>(&'a self) -> impl DoubleEndedIterator<Item = &'a usize> {
         let leaf_nodes = &self.offsets[self.num_internal_nodes..];
 
diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index 621c5d02b..58343de3a 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -1,9 +1,7 @@
 use super::*;
 use crate::{ssz_encode, Encodable};
 
-impl CachedTreeHash for u64 {
-    type Item = Self;
-
+impl CachedTreeHash<u64> for u64 {
     fn item_type() -> ItemType {
         ItemType::Basic
     }
@@ -47,12 +45,10 @@ impl CachedTreeHash for u64 {
     }
 }
 
-impl<T> CachedTreeHash for Vec<T>
+impl<T> CachedTreeHash<Vec<T>> for Vec<T>
 where
-    T: CachedTreeHash,
+    T: CachedTreeHash<T>,
 {
-    type Item = Self;
-
     fn item_type() -> ItemType {
         ItemType::List
     }
@@ -78,7 +74,7 @@ where
                 let mut offsets = vec![];
 
                 for item in self {
-                    offsets.push(item.offsets()?.iter().sum())
+                    offsets.push(OffsetHandler::new(item, 0)?.total_nodes())
                 }
 
                 offsets
@@ -107,32 +103,51 @@ where
 
     fn cached_hash_tree_root(
         &self,
-        other: &Self::Item,
+        other: &Vec<T>,
         cache: &mut TreeHashCache,
         chunk: usize,
     ) -> Result<usize, Error> {
         let offset_handler = OffsetHandler::new(self, chunk)?;
 
+        if self.len() != other.len() {
+            panic!("variable sized lists not implemented");
+        }
+
         match T::item_type() {
             ItemType::Basic => {
                 let leaves = get_packed_leaves(self);
 
                 for (i, chunk) in offset_handler.iter_leaf_nodes().enumerate() {
                     if let Some(latest) = leaves.get(i * HASHSIZE..(i + 1) * HASHSIZE) {
-                        if !cache.chunk_equals(*chunk, latest)? {
-                            dbg!(chunk);
-                            cache.set_changed(*chunk, true)?;
-                        }
+                        cache.maybe_update_chunk(*chunk, latest)?;
                     }
                 }
                 let first_leaf_chunk = offset_handler.first_leaf_node()?;
                 cache.chunk_splice(first_leaf_chunk..offset_handler.next_node, leaves);
             }
-            _ => panic!("not implemented"),
+            ItemType::Composite | ItemType::List => {
+                let mut i = offset_handler.num_leaf_nodes;
+                for start_chunk in offset_handler.iter_leaf_nodes().rev() {
+                    i -= 1;
+                    match (other.get(i), self.get(i)) {
+                        // The item existed in the previous list and exsits in the current list.
+                        (Some(old), Some(new)) => {
+                            new.cached_hash_tree_root(old, cache, *start_chunk)?;
+                        },
+                        // The item didn't exist in the old list and doesn't exist in the new list,
+                        // nothing to do.
+                        (None, None) => {},
+                        _ => panic!("variable sized lists not implemented")
+                    };
+                }
+                // this thing
+            }
         }
 
         for (&parent, children) in offset_handler.iter_internal_nodes().rev() {
             if cache.either_modified(children)? {
+                dbg!(parent);
+                dbg!(children);
                 cache.modify_chunk(parent, &cache.hash_children(children)?)?;
             }
         }
@@ -143,7 +158,7 @@ where
 
 fn get_packed_leaves<T>(vec: &Vec<T>) -> Vec<u8>
 where
-    T: CachedTreeHash,
+    T: CachedTreeHash<T>,
 {
     let num_packed_bytes = vec.num_bytes();
     let num_leaves = num_sanitized_leaves(num_packed_bytes);
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index e65c87bbd..156e2c2e5 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -2,7 +2,7 @@ use super::*;
 use crate::Encodable;
 use int_to_bytes::{int_to_bytes32, int_to_bytes8};
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct Inner {
     pub a: u64,
     pub b: u64,
@@ -10,9 +10,7 @@ pub struct Inner {
     pub d: u64,
 }
 
-impl CachedTreeHash for Inner {
-    type Item = Self;
-
+impl CachedTreeHash<Inner> for Inner {
     fn item_type() -> ItemType {
         ItemType::Composite
     }
@@ -100,16 +98,14 @@ impl CachedTreeHash for Inner {
     }
 }
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct Outer {
     pub a: u64,
     pub b: Inner,
     pub c: u64,
 }
 
-impl CachedTreeHash for Outer {
-    type Item = Self;
-
+impl CachedTreeHash<Outer> for Outer {
     fn item_type() -> ItemType {
         ItemType::Composite
     }
@@ -398,6 +394,66 @@ fn large_vec_of_u64_builds() {
     assert_eq!(expected, cache);
 }
 
+#[test]
+fn partial_modification_of_vec_of_inner() {
+    let original_vec = vec![
+        Inner {
+            a: 0,
+            b: 1,
+            c: 2,
+            d: 3,
+        },
+        Inner {
+            a: 4,
+            b: 5,
+            c: 6,
+            d: 7,
+        },
+        Inner {
+            a: 8,
+            b: 9,
+            c: 10,
+            d: 11,
+        },
+    ];
+    let mut cache = TreeHashCache::new(&original_vec).unwrap();
+
+    let mut modified_vec = original_vec.clone();
+    modified_vec[1].a = 42;
+
+    modified_vec
+        .cached_hash_tree_root(&original_vec, &mut cache, 0)
+        .unwrap();
+    let modified_cache: Vec<u8> = cache.into();
+
+    // Build the reference vec.
+
+    let mut numbers: Vec<u64> = (0..12).collect();
+    numbers[4] = 42;
+
+    let mut leaves = vec![];
+    let mut full_bytes = vec![];
+
+    for n in numbers.chunks(4) {
+        let mut merkle = merkleize(join(vec![
+            int_to_bytes32(n[0]),
+            int_to_bytes32(n[1]),
+            int_to_bytes32(n[2]),
+            int_to_bytes32(n[3]),
+        ]));
+        leaves.append(&mut merkle[0..HASHSIZE].to_vec());
+        full_bytes.append(&mut merkle);
+    }
+
+    let mut expected = merkleize(leaves);
+    expected.splice(3 * HASHSIZE.., full_bytes);
+    expected.append(&mut vec![0; HASHSIZE]);
+
+    // Compare the cached tree to the reference tree.
+
+    assert_trees_eq(&expected, &modified_cache);
+}
+
 #[test]
 fn vec_of_inner_builds() {
     let numbers: Vec<u64> = (0..12).collect();
@@ -449,9 +505,17 @@ fn vec_of_inner_builds() {
 /// Provides detailed assertions when comparing merkle trees.
 fn assert_trees_eq(a: &[u8], b: &[u8]) {
     assert_eq!(a.len(), b.len(), "Byte lens different");
-    for i in 0..a.len() / HASHSIZE {
+    for i in (0..a.len() / HASHSIZE).rev() {
         let range = i * HASHSIZE..(i + 1) * HASHSIZE;
-        assert_eq!(a[range.clone()], b[range], "Chunk {} different", i);
+        assert_eq!(
+            a[range.clone()],
+            b[range],
+            "Chunk {}/{} different \n\n a: {:?} \n\n b: {:?}",
+            i,
+            a.len() / HASHSIZE,
+            a,
+            b,
+        );
     }
 }
 

From 0bdd61e564f62f62c85d8b3d64a3146ead38dc30 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Thu, 11 Apr 2019 17:21:57 +1000
Subject: [PATCH 023/137] Fix failing vec hashing test

---
 eth2/utils/ssz/src/cached_tree_hash.rs       | 21 ++++++++++----------
 eth2/utils/ssz/src/cached_tree_hash/impls.rs |  7 +++----
 2 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index 0889718a2..9960d1f6a 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -195,15 +195,18 @@ impl TreeHashCache {
         Ok(())
     }
 
-    pub fn chunk_equals(&mut self, chunk: usize, other: &[u8]) -> Result<bool, Error> {
+    pub fn get_chunk(&self, chunk: usize) -> Result<&[u8], Error> {
         let start = chunk * BYTES_PER_CHUNK;
         let end = start + BYTES_PER_CHUNK;
 
         Ok(self
             .cache
             .get(start..end)
-            .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))?
-            == other)
+            .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))?)
+    }
+
+    pub fn chunk_equals(&mut self, chunk: usize, other: &[u8]) -> Result<bool, Error> {
+        Ok(self.get_chunk(chunk)? == other)
     }
 
     pub fn changed(&self, chunk: usize) -> Result<bool, Error> {
@@ -218,15 +221,11 @@ impl TreeHashCache {
     }
 
     pub fn hash_children(&self, children: (&usize, &usize)) -> Result<Vec<u8>, Error> {
-        let start = children.0 * BYTES_PER_CHUNK;
-        let end = start + BYTES_PER_CHUNK * 2;
+        let mut child_bytes = Vec::with_capacity(BYTES_PER_CHUNK * 2);
+        child_bytes.append(&mut self.get_chunk(*children.0)?.to_vec());
+        child_bytes.append(&mut self.get_chunk(*children.1)?.to_vec());
 
-        let children = &self
-            .cache
-            .get(start..end)
-            .ok_or_else(|| Error::NoChildrenForHashing((*children.0, *children.1)))?;
-
-        Ok(hash(children))
+        Ok(hash(&child_bytes))
     }
 
     pub fn into_merkle_tree(self) -> Vec<u8> {
diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index 58343de3a..c6cd05cd9 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -133,14 +133,13 @@ where
                         // The item existed in the previous list and exsits in the current list.
                         (Some(old), Some(new)) => {
                             new.cached_hash_tree_root(old, cache, *start_chunk)?;
-                        },
+                        }
                         // The item didn't exist in the old list and doesn't exist in the new list,
                         // nothing to do.
-                        (None, None) => {},
-                        _ => panic!("variable sized lists not implemented")
+                        (None, None) => {}
+                        _ => panic!("variable sized lists not implemented"),
                     };
                 }
-                // this thing
             }
         }
 

From 55ee8e20aefb0498f90383592fcca0a7b4fec7ea Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Thu, 11 Apr 2019 17:40:11 +1000
Subject: [PATCH 024/137] Add more passing tests for vec hash caching

---
 eth2/utils/ssz/src/cached_tree_hash/impls.rs |  4 +-
 eth2/utils/ssz/src/cached_tree_hash/tests.rs | 71 ++++++++++++++++----
 2 files changed, 59 insertions(+), 16 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index c6cd05cd9..01e9e3130 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -109,8 +109,8 @@ where
     ) -> Result<usize, Error> {
         let offset_handler = OffsetHandler::new(self, chunk)?;
 
-        if self.len() != other.len() {
-            panic!("variable sized lists not implemented");
+        if self.len().next_power_of_two() != other.len().next_power_of_two() {
+            panic!("not implemented: vary between power-of-two boundary");
         }
 
         match T::item_type() {
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index 156e2c2e5..62f387321 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -343,29 +343,27 @@ fn outer_builds() {
     assert_eq!(merkle, cache);
 }
 
-#[test]
-fn partial_modification_u64_vec() {
-    let n: u64 = 50;
-
-    let original_vec: Vec<u64> = (0..n).collect();
-
+/// Generic test that covers:
+///
+/// 1. Produce a new cache from `original`.
+/// 2. Do a differential hash between `original` and `modified`.
+/// 3. Test that the cache generated matches the one we generate manually.
+///
+/// In effect it ensures that we can do a differential hash between two `Vec<u64>`.
+fn test_u64_vec_modifications(original: Vec<u64>, modified: Vec<u64>) {
     // Generate initial cache.
-    let original_cache: Vec<u8> = TreeHashCache::new(&original_vec).unwrap().into();
-
-    // Modify the vec
-    let mut modified_vec = original_vec.clone();
-    modified_vec[n as usize - 1] = 42;
+    let original_cache: Vec<u8> = TreeHashCache::new(&original).unwrap().into();
 
     // Perform a differential hash
     let mut cache_struct = TreeHashCache::from_bytes(original_cache.clone()).unwrap();
-    modified_vec
-        .cached_hash_tree_root(&original_vec, &mut cache_struct, 0)
+    modified
+        .cached_hash_tree_root(&original, &mut cache_struct, 0)
         .unwrap();
     let modified_cache: Vec<u8> = cache_struct.into();
 
     // Generate reference data.
     let mut data = vec![];
-    for i in &modified_vec {
+    for i in &modified {
         data.append(&mut int_to_bytes8(*i));
     }
     let data = sanitise_bytes(data);
@@ -374,6 +372,51 @@ fn partial_modification_u64_vec() {
     assert_eq!(expected, modified_cache);
 }
 
+#[test]
+fn partial_modification_u64_vec() {
+    let n: u64 = 2_u64.pow(5);
+
+    let original_vec: Vec<u64> = (0..n).collect();
+
+    let mut modified_vec = original_vec.clone();
+    modified_vec[n as usize - 1] = 42;
+
+    test_u64_vec_modifications(original_vec, modified_vec);
+}
+
+#[test]
+fn shortened_u64_vec_len_within_pow_2_boundary() {
+    let n: u64 = 2_u64.pow(5) - 1;
+
+    let original_vec: Vec<u64> = (0..n).collect();
+
+    let mut modified_vec = original_vec.clone();
+    modified_vec.pop();
+
+    test_u64_vec_modifications(original_vec, modified_vec);
+}
+
+#[test]
+fn extended_u64_vec_len_within_pow_2_boundary() {
+    let n: u64 = 2_u64.pow(5) - 2;
+
+    let original_vec: Vec<u64> = (0..n).collect();
+
+    let mut modified_vec = original_vec.clone();
+    modified_vec.push(42);
+
+    test_u64_vec_modifications(original_vec, modified_vec);
+}
+
+#[test]
+fn extended_u64_vec_len_outside_pow_2_boundary() {
+    let original_vec: Vec<u64> = (0..2_u64.pow(5)).collect();
+
+    let modified_vec: Vec<u64> = (0..2_u64.pow(6)).collect();
+
+    test_u64_vec_modifications(original_vec, modified_vec);
+}
+
 #[test]
 fn large_vec_of_u64_builds() {
     let n: u64 = 50;

From 48cf75e394eafe8afac52b835d50a71d3ee6f96c Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Fri, 12 Apr 2019 15:05:26 +1000
Subject: [PATCH 025/137] Add failing test for extending struct list

---
 eth2/utils/ssz/src/cached_tree_hash.rs       |   5 +
 eth2/utils/ssz/src/cached_tree_hash/impls.rs |  20 ++-
 eth2/utils/ssz/src/cached_tree_hash/tests.rs | 145 +++++++++++++++----
 3 files changed, 140 insertions(+), 30 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index 9960d1f6a..6e84233fc 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -163,6 +163,11 @@ impl TreeHashCache {
         let byte_start = chunk_range.start * BYTES_PER_CHUNK;
         let byte_end = chunk_range.end * BYTES_PER_CHUNK;
 
+        // Update the `chunk_modified` vec, marking all spliced-in nodes as changed.
+        self.chunk_modified.splice(
+            chunk_range.clone(),
+            vec![true; chunk_range.end - chunk_range.start],
+        );
         self.cache.splice(byte_start..byte_end, replace_with)
     }
 
diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index 01e9e3130..37a3678c2 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -127,12 +127,28 @@ where
             }
             ItemType::Composite | ItemType::List => {
                 let mut i = offset_handler.num_leaf_nodes;
-                for start_chunk in offset_handler.iter_leaf_nodes().rev() {
+                for &start_chunk in offset_handler.iter_leaf_nodes().rev() {
                     i -= 1;
                     match (other.get(i), self.get(i)) {
                         // The item existed in the previous list and exsits in the current list.
                         (Some(old), Some(new)) => {
-                            new.cached_hash_tree_root(old, cache, *start_chunk)?;
+                            new.cached_hash_tree_root(old, cache, start_chunk)?;
+                        }
+                        // The item existed in the previous list but does not exist in this list.
+                        //
+                        // I.e., the list has been shortened.
+                        (Some(old), None) => {
+                            // Splice out the entire tree of the removed node, replacing it with a
+                            // single padding node.
+                            let end_chunk = OffsetHandler::new(old, start_chunk)?.next_node();
+                            cache.chunk_splice(start_chunk..end_chunk, vec![0; HASHSIZE]);
+                        }
+                        // The item existed in the previous list but does exist in this list.
+                        //
+                        // I.e., the list has been lengthened.
+                        (None, Some(new)) => {
+                            let bytes: Vec<u8> = TreeHashCache::new(new)?.into();
+                            cache.chunk_splice(start_chunk..start_chunk + 1, bytes);
                         }
                         // The item didn't exist in the old list and doesn't exist in the new list,
                         // nothing to do.
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index 62f387321..4110e29a1 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -408,6 +408,7 @@ fn extended_u64_vec_len_within_pow_2_boundary() {
     test_u64_vec_modifications(original_vec, modified_vec);
 }
 
+/*
 #[test]
 fn extended_u64_vec_len_outside_pow_2_boundary() {
     let original_vec: Vec<u64> = (0..2_u64.pow(5)).collect();
@@ -416,6 +417,7 @@ fn extended_u64_vec_len_outside_pow_2_boundary() {
 
     test_u64_vec_modifications(original_vec, modified_vec);
 }
+*/
 
 #[test]
 fn large_vec_of_u64_builds() {
@@ -437,9 +439,51 @@ fn large_vec_of_u64_builds() {
     assert_eq!(expected, cache);
 }
 
+/// Generic test that covers:
+///
+/// 1. Produce a new cache from `original`.
+/// 2. Do a differential hash between `original` and `modified`.
+/// 3. Test that the cache generated matches the one we generate manually.
+///
+/// The `reference` vec is used to build the tree hash cache manually. `Inner` is just 4x `u64`, so
+/// you can represent 2x `Inner` with a `reference` vec of len 8.
+///
+/// In effect it ensures that we can do a differential hash between two `Vec<Inner>`.
+fn test_inner_vec_modifications(original: Vec<Inner>, modified: Vec<Inner>, reference: Vec<u64>) {
+    let mut cache = TreeHashCache::new(&original).unwrap();
+
+    modified
+        .cached_hash_tree_root(&original, &mut cache, 0)
+        .unwrap();
+    let modified_cache: Vec<u8> = cache.into();
+
+    // Build the reference vec.
+
+    let mut leaves = vec![];
+    let mut full_bytes = vec![];
+
+    for n in reference.chunks(4) {
+        let mut merkle = merkleize(join(vec![
+            int_to_bytes32(n[0]),
+            int_to_bytes32(n[1]),
+            int_to_bytes32(n[2]),
+            int_to_bytes32(n[3]),
+        ]));
+        leaves.append(&mut merkle[0..HASHSIZE].to_vec());
+        full_bytes.append(&mut merkle);
+    }
+
+    let mut expected = merkleize(leaves);
+    expected.splice(3 * HASHSIZE.., full_bytes);
+    expected.append(&mut vec![0; HASHSIZE]);
+
+    // Compare the cached tree to the reference tree.
+    assert_trees_eq(&expected, &modified_cache);
+}
+
 #[test]
 fn partial_modification_of_vec_of_inner() {
-    let original_vec = vec![
+    let original = vec![
         Inner {
             a: 0,
             b: 1,
@@ -459,42 +503,87 @@ fn partial_modification_of_vec_of_inner() {
             d: 11,
         },
     ];
-    let mut cache = TreeHashCache::new(&original_vec).unwrap();
 
-    let mut modified_vec = original_vec.clone();
-    modified_vec[1].a = 42;
+    let mut modified = original.clone();
+    modified[1].a = 42;
 
-    modified_vec
-        .cached_hash_tree_root(&original_vec, &mut cache, 0)
-        .unwrap();
-    let modified_cache: Vec<u8> = cache.into();
+    let mut reference_vec: Vec<u64> = (0..12).collect();
+    reference_vec[4] = 42;
 
-    // Build the reference vec.
+    test_inner_vec_modifications(original, modified, reference_vec);
+}
 
-    let mut numbers: Vec<u64> = (0..12).collect();
-    numbers[4] = 42;
+#[test]
+fn shortened_vec_of_inner_within_power_of_two_boundary() {
+    let original = vec![
+        Inner {
+            a: 0,
+            b: 1,
+            c: 2,
+            d: 3,
+        },
+        Inner {
+            a: 4,
+            b: 5,
+            c: 6,
+            d: 7,
+        },
+        Inner {
+            a: 8,
+            b: 9,
+            c: 10,
+            d: 11,
+        },
+        Inner {
+            a: 12,
+            b: 13,
+            c: 14,
+            d: 15,
+        },
+    ];
 
-    let mut leaves = vec![];
-    let mut full_bytes = vec![];
+    let mut modified = original.clone();
+    modified.pop(); // remove the last element from the list.
 
-    for n in numbers.chunks(4) {
-        let mut merkle = merkleize(join(vec![
-            int_to_bytes32(n[0]),
-            int_to_bytes32(n[1]),
-            int_to_bytes32(n[2]),
-            int_to_bytes32(n[3]),
-        ]));
-        leaves.append(&mut merkle[0..HASHSIZE].to_vec());
-        full_bytes.append(&mut merkle);
-    }
+    let reference_vec: Vec<u64> = (0..12).collect();
 
-    let mut expected = merkleize(leaves);
-    expected.splice(3 * HASHSIZE.., full_bytes);
-    expected.append(&mut vec![0; HASHSIZE]);
+    test_inner_vec_modifications(original, modified, reference_vec);
+}
 
-    // Compare the cached tree to the reference tree.
+#[test]
+fn lengthened_vec_of_inner_within_power_of_two_boundary() {
+    let original = vec![
+        Inner {
+            a: 0,
+            b: 1,
+            c: 2,
+            d: 3,
+        },
+        Inner {
+            a: 4,
+            b: 5,
+            c: 6,
+            d: 7,
+        },
+        Inner {
+            a: 8,
+            b: 9,
+            c: 10,
+            d: 11,
+        },
+    ];
 
-    assert_trees_eq(&expected, &modified_cache);
+    let mut modified = original.clone();
+    modified.push(Inner {
+        a: 12,
+        b: 13,
+        c: 14,
+        d: 15,
+    });
+
+    let reference_vec: Vec<u64> = (0..16).collect();
+
+    test_inner_vec_modifications(original, modified, reference_vec);
 }
 
 #[test]

From d79616fee67a51954cbf2835a1f381aff638aa9a Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Fri, 12 Apr 2019 16:52:11 +1000
Subject: [PATCH 026/137] Fix failing struct vec vectors

---
 eth2/utils/ssz/src/cached_tree_hash.rs       | 20 +++++---------------
 eth2/utils/ssz/src/cached_tree_hash/impls.rs |  1 -
 eth2/utils/ssz/src/cached_tree_hash/tests.rs |  7 ++++++-
 3 files changed, 11 insertions(+), 17 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index 6e84233fc..d6ff884ef 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -145,30 +145,20 @@ impl TreeHashCache {
         })
     }
 
-    pub fn single_chunk_splice<I>(&mut self, chunk: usize, replace_with: I) -> Splice<I::IntoIter>
-    where
-        I: IntoIterator<Item = u8>,
-    {
-        self.chunk_splice(chunk..chunk + 1, replace_with)
+    pub fn single_chunk_splice<I>(&mut self, chunk: usize, replace_with: Vec<u8>) {
+        self.chunk_splice(chunk..chunk + 1, replace_with);
     }
 
-    pub fn chunk_splice<I>(
-        &mut self,
-        chunk_range: Range<usize>,
-        replace_with: I,
-    ) -> Splice<I::IntoIter>
-    where
-        I: IntoIterator<Item = u8>,
-    {
+    pub fn chunk_splice(&mut self, chunk_range: Range<usize>, replace_with: Vec<u8>) {
         let byte_start = chunk_range.start * BYTES_PER_CHUNK;
         let byte_end = chunk_range.end * BYTES_PER_CHUNK;
 
         // Update the `chunk_modified` vec, marking all spliced-in nodes as changed.
         self.chunk_modified.splice(
             chunk_range.clone(),
-            vec![true; chunk_range.end - chunk_range.start],
+            vec![true; replace_with.len() / HASHSIZE],
         );
-        self.cache.splice(byte_start..byte_end, replace_with)
+        self.cache.splice(byte_start..byte_end, replace_with);
     }
 
     pub fn maybe_update_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> {
diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index 37a3678c2..2d0ab5059 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -153,7 +153,6 @@ where
                         // The item didn't exist in the old list and doesn't exist in the new list,
                         // nothing to do.
                         (None, None) => {}
-                        _ => panic!("variable sized lists not implemented"),
                     };
                 }
             }
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index 4110e29a1..d48ed9eb8 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -473,9 +473,14 @@ fn test_inner_vec_modifications(original: Vec<Inner>, modified: Vec<Inner>, refe
         full_bytes.append(&mut merkle);
     }
 
+    let num_leaves = leaves.len() / HASHSIZE;
+
     let mut expected = merkleize(leaves);
     expected.splice(3 * HASHSIZE.., full_bytes);
-    expected.append(&mut vec![0; HASHSIZE]);
+
+    for _ in num_leaves..num_leaves.next_power_of_two() {
+        expected.append(&mut vec![0; HASHSIZE]);
+    }
 
     // Compare the cached tree to the reference tree.
     assert_trees_eq(&expected, &modified_cache);

From a124042e30eec4d6dd78168314c2b29255a2736e Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Sat, 13 Apr 2019 09:11:19 +1000
Subject: [PATCH 027/137] Start implementing grow merkle fn

---
 eth2/utils/ssz/src/cached_tree_hash/impls.rs | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index 2d0ab5059..14eab3180 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -170,6 +170,23 @@ where
     }
 }
 
+/// New vec is bigger than old vec.
+fn grow_merkle_cache(cache: Vec<u8>, to: usize) -> Vec<u8> {
+    let new = Vec::with_capacity(to * HASHSIZE);
+
+    let i = cache.len() / HASHSIZE;
+    let j = to;
+
+    assert_eq!(i.next_power_of_two(), i);
+    assert_eq!(j.next_power_of_two(), j);
+
+    while i > 0 {
+        
+    }
+
+    new
+}
+
 fn get_packed_leaves<T>(vec: &Vec<T>) -> Vec<u8>
 where
     T: CachedTreeHash<T>,

From 75177837d0717045fb4c9271ca2ee96989f47ab3 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Sat, 13 Apr 2019 09:42:43 +1000
Subject: [PATCH 028/137] Add first pass of grow cache algo

---
 eth2/utils/ssz/src/cached_tree_hash.rs        |  1 +
 eth2/utils/ssz/src/cached_tree_hash/impls.rs  | 17 ----------
 eth2/utils/ssz/src/cached_tree_hash/resize.rs | 33 +++++++++++++++++++
 3 files changed, 34 insertions(+), 17 deletions(-)
 create mode 100644 eth2/utils/ssz/src/cached_tree_hash/resize.rs

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index d6ff884ef..0588ab772 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -6,6 +6,7 @@ use std::ops::Range;
 use std::vec::Splice;
 
 mod impls;
+mod resize;
 mod tests;
 
 const BYTES_PER_CHUNK: usize = 32;
diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index 14eab3180..2d0ab5059 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -170,23 +170,6 @@ where
     }
 }
 
-/// New vec is bigger than old vec.
-fn grow_merkle_cache(cache: Vec<u8>, to: usize) -> Vec<u8> {
-    let new = Vec::with_capacity(to * HASHSIZE);
-
-    let i = cache.len() / HASHSIZE;
-    let j = to;
-
-    assert_eq!(i.next_power_of_two(), i);
-    assert_eq!(j.next_power_of_two(), j);
-
-    while i > 0 {
-        
-    }
-
-    new
-}
-
 fn get_packed_leaves<T>(vec: &Vec<T>) -> Vec<u8>
 where
     T: CachedTreeHash<T>,
diff --git a/eth2/utils/ssz/src/cached_tree_hash/resize.rs b/eth2/utils/ssz/src/cached_tree_hash/resize.rs
new file mode 100644
index 000000000..d41453e9a
--- /dev/null
+++ b/eth2/utils/ssz/src/cached_tree_hash/resize.rs
@@ -0,0 +1,33 @@
+use super::*;
+
+/// New vec is bigger than old vec.
+fn grow_merkle_cache(old_bytes: &[u8], old_flags: &[bool], to: usize) -> Option<Vec<u8>> {
+    let mut bytes = Vec::with_capacity(to * HASHSIZE);
+    let mut flags = Vec::with_capacity(to);
+
+    let from = old_bytes.len() / HASHSIZE;
+    let to = to;
+
+    let distance = (from.leading_zeros() - to.leading_zeros()) as usize;
+
+    let leading_zero_chunks = 1 >> distance;
+
+    bytes.resize(leading_zero_chunks * HASHSIZE, 0);
+    flags.resize(leading_zero_chunks, true); // all new chunks are modified by default.
+
+    for i in 0..to.leading_zeros() as usize {
+        let new_slice = bytes.get_mut(1 >> i + distance..1 >> i + distance + 1)?;
+        let old_slice = old_bytes.get(1 >> i..1 >> i + 1)?;
+        new_slice.copy_from_slice(old_slice);
+    }
+
+    Some(bytes)
+}
+
+#[cfg(test)]
+mod test {
+    #[test]
+    fn can_grow() {
+        // TODO
+    }
+}

From 0b186f772fdfe432bdcc707482539a1199aae346 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Sat, 13 Apr 2019 12:12:56 +1000
Subject: [PATCH 029/137] Refactor resize functions for clarity

---
 eth2/utils/ssz/src/cached_tree_hash/resize.rs | 105 +++++++++++++++---
 1 file changed, 92 insertions(+), 13 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash/resize.rs b/eth2/utils/ssz/src/cached_tree_hash/resize.rs
index d41453e9a..a7bad0b04 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/resize.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/resize.rs
@@ -1,33 +1,112 @@
 use super::*;
 
 /// New vec is bigger than old vec.
-fn grow_merkle_cache(old_bytes: &[u8], old_flags: &[bool], to: usize) -> Option<Vec<u8>> {
-    let mut bytes = Vec::with_capacity(to * HASHSIZE);
-    let mut flags = Vec::with_capacity(to);
+fn grow_merkle_cache(
+    old_bytes: &[u8],
+    old_flags: &[bool],
+    from_height: usize,
+    to_height: usize,
+) -> Option<Vec<u8>> {
+    let to_nodes = (1 << to_height.next_power_of_two()) - 1;
 
-    let from = old_bytes.len() / HASHSIZE;
-    let to = to;
+    // Determine the size of our new tree. It is not just a simple `1 << to_height` as there can be
+    // an arbitrary number of bytes in `old_bytes` leaves.
+    let new_byte_count = {
+        let additional_from_nodes = old_bytes.len() / HASHSIZE - ((1 << from_height) - 1);
+        ((1 << to_height + additional_from_nodes) - 1) * HASHSIZE
+    };
+    dbg!(new_byte_count / 32);
 
-    let distance = (from.leading_zeros() - to.leading_zeros()) as usize;
+    let mut bytes = vec![0; new_byte_count];
+    let mut flags = vec![true; to_nodes];
 
-    let leading_zero_chunks = 1 >> distance;
+    let leaf_level = from_height - 1;
 
-    bytes.resize(leading_zero_chunks * HASHSIZE, 0);
-    flags.resize(leading_zero_chunks, true); // all new chunks are modified by default.
+    // Loop through all internal levels of the tree (skipping the final, leaves level).
+    for i in 0..from_height - 1 as usize {
+        dbg!(i);
+        dbg!(bytes.len());
+        // If we're on the leaf slice, grab the first byte and all the of the bytes after that.
+        // This is required because we can have an arbitrary number of bytes at the leaf level
+        // (e.g., the case where there are subtrees as leaves).
+        //
+        // If we're not on a leaf level, the number of nodes is fixed and known.
+        let old_slice = if i == leaf_level {
+            old_bytes.get(first_byte_at_height(i)..)
+        } else {
+            old_bytes.get(byte_range_at_height(i))
+        }?;
+
+        dbg!(byte_range_at_height(i + to_height - from_height));
+
+        let new_slice = bytes
+            .get_mut(byte_range_at_height(i + to_height - from_height))?
+            .get_mut(0..old_slice.len())?;
 
-    for i in 0..to.leading_zeros() as usize {
-        let new_slice = bytes.get_mut(1 >> i + distance..1 >> i + distance + 1)?;
-        let old_slice = old_bytes.get(1 >> i..1 >> i + 1)?;
         new_slice.copy_from_slice(old_slice);
     }
 
     Some(bytes)
 }
 
+fn byte_range_at_height(h: usize) -> Range<usize> {
+    first_byte_at_height(h)..last_node_at_height(h) * HASHSIZE
+}
+
+fn first_byte_at_height(h: usize) -> usize {
+    first_node_at_height(h) * HASHSIZE
+}
+
+fn first_node_at_height(h: usize) -> usize {
+    (1 << h) - 1
+}
+
+fn last_node_at_height(h: usize) -> usize {
+    (1 << (h + 1)) - 1
+}
+
 #[cfg(test)]
 mod test {
+    use super::*;
+
     #[test]
     fn can_grow() {
-        // TODO
+        let from: usize = 7;
+        let to: usize = 15;
+
+        let old_bytes = vec![42; from * HASHSIZE];
+        let old_flags = vec![false; from];
+
+        let new = grow_merkle_cache(
+            &old_bytes,
+            &old_flags,
+            (from + 1).trailing_zeros() as usize,
+            (to + 1).trailing_zeros() as usize,
+        )
+        .unwrap();
+
+        println!("{:?}", new);
+        let mut expected = vec![];
+        // First level
+        expected.append(&mut vec![0; 32]);
+        // Second level
+        expected.append(&mut vec![42; 32]);
+        expected.append(&mut vec![0; 32]);
+        // Third level
+        expected.append(&mut vec![42; 32]);
+        expected.append(&mut vec![42; 32]);
+        expected.append(&mut vec![0; 32]);
+        expected.append(&mut vec![0; 32]);
+        // Fourth level
+        expected.append(&mut vec![0; 32]);
+        expected.append(&mut vec![0; 32]);
+        expected.append(&mut vec![0; 32]);
+        expected.append(&mut vec![0; 32]);
+        expected.append(&mut vec![0; 32]);
+        expected.append(&mut vec![0; 32]);
+        expected.append(&mut vec![0; 32]);
+        expected.append(&mut vec![0; 32]);
+
+        assert_eq!(expected, new);
     }
 }

From 0420607ff130fc8f4b61458267479fb6ad16980b Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Sat, 13 Apr 2019 13:02:41 +1000
Subject: [PATCH 030/137] Tidy, remove debug prints

---
 eth2/utils/ssz/src/cached_tree_hash/resize.rs | 25 ++++++++++++++++---
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash/resize.rs b/eth2/utils/ssz/src/cached_tree_hash/resize.rs
index a7bad0b04..21b729c9e 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/resize.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/resize.rs
@@ -24,8 +24,6 @@ fn grow_merkle_cache(
 
     // Loop through all internal levels of the tree (skipping the final, leaves level).
     for i in 0..from_height - 1 as usize {
-        dbg!(i);
-        dbg!(bytes.len());
         // If we're on the leaf slice, grab the first byte and all the of the bytes after that.
         // This is required because we can have an arbitrary number of bytes at the leaf level
         // (e.g., the case where there are subtrees as leaves).
@@ -37,8 +35,6 @@ fn grow_merkle_cache(
             old_bytes.get(byte_range_at_height(i))
         }?;
 
-        dbg!(byte_range_at_height(i + to_height - from_height));
-
         let new_slice = bytes
             .get_mut(byte_range_at_height(i + to_height - from_height))?
             .get_mut(0..old_slice.len())?;
@@ -49,6 +45,27 @@ fn grow_merkle_cache(
     Some(bytes)
 }
 
+/*
+fn copy_bytes(
+    from_range: Range<usize>,
+    to_range: Range<usize>,
+    from: &[u8],
+    to: &mut Vec<u8>,
+) -> Option<()> {
+    let from_slice = from.get(node_range_to_byte_range(from_range));
+
+    let to_slice = to
+        .get_mut(byte_range_at_height(i + to_height - from_height))?
+        .get_mut(0..old_slice.len())?;
+
+    Ok(())
+}
+*/
+
+fn node_range_to_byte_range(node_range: Range<usize>) -> Range<usize> {
+    node_range.start * HASHSIZE..node_range.end * HASHSIZE
+}
+
 fn byte_range_at_height(h: usize) -> Range<usize> {
     first_byte_at_height(h)..last_node_at_height(h) * HASHSIZE
 }

From 42d6a39832d867bbaab4103e447b740e5ba3d49e Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Sat, 13 Apr 2019 13:18:18 +1000
Subject: [PATCH 031/137] Refactor TreeHashCache splice method

---
 eth2/utils/ssz/src/cached_tree_hash.rs       | 43 ++++++++++++++------
 eth2/utils/ssz/src/cached_tree_hash/impls.rs | 25 +++++++++---
 eth2/utils/ssz/src/cached_tree_hash/tests.rs |  4 +-
 3 files changed, 52 insertions(+), 20 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index 0588ab772..b676ececc 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -124,6 +124,17 @@ impl TreeHashCache {
         })
     }
 
+    pub fn from_bytes(bytes: Vec<u8>, initial_modified_state: bool) -> Result<Self, Error> {
+        if bytes.len() % BYTES_PER_CHUNK > 0 {
+            return Err(Error::BytesAreNotEvenChunks(bytes.len()));
+        }
+
+        Ok(Self {
+            chunk_modified: vec![initial_modified_state; bytes.len() / BYTES_PER_CHUNK],
+            cache: bytes,
+        })
+    }
+
     pub fn bytes_len(&self) -> usize {
         self.cache.len()
     }
@@ -135,22 +146,19 @@ impl TreeHashCache {
             .and_then(|slice| Ok(slice.to_vec()))
     }
 
-    pub fn from_bytes(bytes: Vec<u8>) -> Result<Self, Error> {
-        if bytes.len() % BYTES_PER_CHUNK > 0 {
-            return Err(Error::BytesAreNotEvenChunks(bytes.len()));
-        }
+    pub fn splice(&mut self, chunk_range: Range<usize>, replace_with: Self) {
+        let (bytes, bools) = replace_with.into_components();
 
-        Ok(Self {
-            chunk_modified: vec![false; bytes.len() / BYTES_PER_CHUNK],
-            cache: bytes,
-        })
+        // Update the `chunk_modified` vec, marking all spliced-in nodes as changed.
+        self.chunk_modified.splice(
+            chunk_range.clone(),
+            bools,
+        );
+        self.cache.splice(node_range_to_byte_range(chunk_range), bytes);
     }
 
-    pub fn single_chunk_splice<I>(&mut self, chunk: usize, replace_with: Vec<u8>) {
-        self.chunk_splice(chunk..chunk + 1, replace_with);
-    }
-
-    pub fn chunk_splice(&mut self, chunk_range: Range<usize>, replace_with: Vec<u8>) {
+    /*
+    pub fn byte_splice(&mut self, chunk_range: Range<usize>, replace_with: Vec<u8>) {
         let byte_start = chunk_range.start * BYTES_PER_CHUNK;
         let byte_end = chunk_range.end * BYTES_PER_CHUNK;
 
@@ -161,6 +169,7 @@ impl TreeHashCache {
         );
         self.cache.splice(byte_start..byte_end, replace_with);
     }
+    */
 
     pub fn maybe_update_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> {
         let start = chunk * BYTES_PER_CHUNK;
@@ -227,6 +236,10 @@ impl TreeHashCache {
     pub fn into_merkle_tree(self) -> Vec<u8> {
         self.cache
     }
+
+    pub fn into_components(self) -> (Vec<u8>, Vec<bool>) {
+        (self.cache, self.chunk_modified)
+    }
 }
 
 fn children(parent: usize) -> (usize, usize) {
@@ -237,6 +250,10 @@ fn num_nodes(num_leaves: usize) -> usize {
     2 * num_leaves - 1
 }
 
+fn node_range_to_byte_range(node_range: Range<usize>) -> Range<usize> {
+    node_range.start * HASHSIZE..node_range.end * HASHSIZE
+}
+
 #[derive(Debug)]
 pub struct OffsetHandler {
     num_internal_nodes: usize,
diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index 2d0ab5059..f598de79a 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -7,7 +7,10 @@ impl CachedTreeHash<u64> for u64 {
     }
 
     fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
-        Ok(TreeHashCache::from_bytes(merkleize(ssz_encode(self)))?)
+        Ok(TreeHashCache::from_bytes(
+            merkleize(ssz_encode(self)),
+            false,
+        )?)
     }
 
     fn num_bytes(&self) -> usize {
@@ -55,7 +58,7 @@ where
 
     fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
         match T::item_type() {
-            ItemType::Basic => TreeHashCache::from_bytes(merkleize(get_packed_leaves(self))),
+            ItemType::Basic => TreeHashCache::from_bytes(merkleize(get_packed_leaves(self)), false),
             ItemType::Composite | ItemType::List => {
                 let subtrees = self
                     .iter()
@@ -123,7 +126,11 @@ where
                     }
                 }
                 let first_leaf_chunk = offset_handler.first_leaf_node()?;
-                cache.chunk_splice(first_leaf_chunk..offset_handler.next_node, leaves);
+
+                cache.splice(
+                    first_leaf_chunk..offset_handler.next_node,
+                    TreeHashCache::from_bytes(leaves, true)?,
+                );
             }
             ItemType::Composite | ItemType::List => {
                 let mut i = offset_handler.num_leaf_nodes;
@@ -141,14 +148,22 @@ where
                             // Splice out the entire tree of the removed node, replacing it with a
                             // single padding node.
                             let end_chunk = OffsetHandler::new(old, start_chunk)?.next_node();
-                            cache.chunk_splice(start_chunk..end_chunk, vec![0; HASHSIZE]);
+
+                            cache.splice(
+                                start_chunk..end_chunk,
+                                TreeHashCache::from_bytes(vec![0; HASHSIZE], true)?,
+                            );
                         }
                         // The item existed in the previous list but does exist in this list.
                         //
                         // I.e., the list has been lengthened.
                         (None, Some(new)) => {
                             let bytes: Vec<u8> = TreeHashCache::new(new)?.into();
-                            cache.chunk_splice(start_chunk..start_chunk + 1, bytes);
+
+                            cache.splice(
+                                start_chunk..start_chunk + 1,
+                                TreeHashCache::from_bytes(bytes, true)?,
+                            );
                         }
                         // The item didn't exist in the old list and doesn't exist in the new list,
                         // nothing to do.
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index d48ed9eb8..d784a0889 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -355,7 +355,7 @@ fn test_u64_vec_modifications(original: Vec<u64>, modified: Vec<u64>) {
     let original_cache: Vec<u8> = TreeHashCache::new(&original).unwrap().into();
 
     // Perform a differential hash
-    let mut cache_struct = TreeHashCache::from_bytes(original_cache.clone()).unwrap();
+    let mut cache_struct = TreeHashCache::from_bytes(original_cache.clone(), false).unwrap();
     modified
         .cached_hash_tree_root(&original, &mut cache_struct, 0)
         .unwrap();
@@ -723,7 +723,7 @@ fn generic_test(index: usize) {
         _ => panic!("bad index"),
     };
 
-    let mut cache_struct = TreeHashCache::from_bytes(cache.clone()).unwrap();
+    let mut cache_struct = TreeHashCache::from_bytes(cache.clone(), false).unwrap();
 
     changed_inner
         .cached_hash_tree_root(&inner, &mut cache_struct, 0)

From 1ce1fce03c5bc31f120bcbf4650ea2c0acd53557 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Sat, 13 Apr 2019 17:21:50 +1000
Subject: [PATCH 032/137] Fix failing grow tree test

---
 eth2/utils/ssz/src/cached_tree_hash.rs        |  25 +--
 eth2/utils/ssz/src/cached_tree_hash/impls.rs  |   9 +-
 eth2/utils/ssz/src/cached_tree_hash/resize.rs | 174 ++++++++++++------
 eth2/utils/ssz/src/cached_tree_hash/tests.rs  |   2 +-
 4 files changed, 129 insertions(+), 81 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index b676ececc..84ef82233 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -1,9 +1,7 @@
 use hashing::hash;
 use std::fmt::Debug;
-use std::iter::IntoIterator;
 use std::iter::Iterator;
 use std::ops::Range;
-use std::vec::Splice;
 
 mod impls;
 mod resize;
@@ -21,7 +19,6 @@ pub enum Error {
     BytesAreNotEvenChunks(usize),
     NoModifiedFieldForChunk(usize),
     NoBytesForChunk(usize),
-    NoChildrenForHashing((usize, usize)),
 }
 
 #[derive(Debug, PartialEq, Clone)]
@@ -150,27 +147,11 @@ impl TreeHashCache {
         let (bytes, bools) = replace_with.into_components();
 
         // Update the `chunk_modified` vec, marking all spliced-in nodes as changed.
-        self.chunk_modified.splice(
-            chunk_range.clone(),
-            bools,
-        );
-        self.cache.splice(node_range_to_byte_range(chunk_range), bytes);
+        self.chunk_modified.splice(chunk_range.clone(), bools);
+        self.cache
+            .splice(node_range_to_byte_range(chunk_range), bytes);
     }
 
-    /*
-    pub fn byte_splice(&mut self, chunk_range: Range<usize>, replace_with: Vec<u8>) {
-        let byte_start = chunk_range.start * BYTES_PER_CHUNK;
-        let byte_end = chunk_range.end * BYTES_PER_CHUNK;
-
-        // Update the `chunk_modified` vec, marking all spliced-in nodes as changed.
-        self.chunk_modified.splice(
-            chunk_range.clone(),
-            vec![true; replace_with.len() / HASHSIZE],
-        );
-        self.cache.splice(byte_start..byte_end, replace_with);
-    }
-    */
-
     pub fn maybe_update_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> {
         let start = chunk * BYTES_PER_CHUNK;
         let end = start + BYTES_PER_CHUNK;
diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index f598de79a..dca00b6ba 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -1,5 +1,6 @@
+use super::resize::grow_merkle_cache;
 use super::*;
-use crate::{ssz_encode, Encodable};
+use crate::ssz_encode;
 
 impl CachedTreeHash<u64> for u64 {
     fn item_type() -> ItemType {
@@ -112,8 +113,10 @@ where
     ) -> Result<usize, Error> {
         let offset_handler = OffsetHandler::new(self, chunk)?;
 
-        if self.len().next_power_of_two() != other.len().next_power_of_two() {
-            panic!("not implemented: vary between power-of-two boundary");
+        if other.len().next_power_of_two() > self.len().next_power_of_two() {
+            //
+        } else if other.len().next_power_of_two() < self.len().next_power_of_two() {
+            panic!("shrinking below power of two is not implemented")
         }
 
         match T::item_type() {
diff --git a/eth2/utils/ssz/src/cached_tree_hash/resize.rs b/eth2/utils/ssz/src/cached_tree_hash/resize.rs
index 21b729c9e..bce722a5e 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/resize.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/resize.rs
@@ -1,12 +1,12 @@
 use super::*;
 
 /// New vec is bigger than old vec.
-fn grow_merkle_cache(
+pub fn grow_merkle_cache(
     old_bytes: &[u8],
     old_flags: &[bool],
     from_height: usize,
     to_height: usize,
-) -> Option<Vec<u8>> {
+) -> Option<(Vec<u8>, Vec<bool>)> {
     let to_nodes = (1 << to_height.next_power_of_two()) - 1;
 
     // Determine the size of our new tree. It is not just a simple `1 << to_height` as there can be
@@ -15,7 +15,6 @@ fn grow_merkle_cache(
         let additional_from_nodes = old_bytes.len() / HASHSIZE - ((1 << from_height) - 1);
         ((1 << to_height + additional_from_nodes) - 1) * HASHSIZE
     };
-    dbg!(new_byte_count / 32);
 
     let mut bytes = vec![0; new_byte_count];
     let mut flags = vec![true; to_nodes];
@@ -23,53 +22,45 @@ fn grow_merkle_cache(
     let leaf_level = from_height - 1;
 
     // Loop through all internal levels of the tree (skipping the final, leaves level).
-    for i in 0..from_height - 1 as usize {
+    for i in 0..from_height as usize {
         // If we're on the leaf slice, grab the first byte and all the of the bytes after that.
         // This is required because we can have an arbitrary number of bytes at the leaf level
         // (e.g., the case where there are subtrees as leaves).
         //
         // If we're not on a leaf level, the number of nodes is fixed and known.
-        let old_slice = if i == leaf_level {
-            old_bytes.get(first_byte_at_height(i)..)
+        let (byte_slice, flag_slice) = if i == leaf_level {
+            (
+                old_bytes.get(first_byte_at_height(i)..)?,
+                old_flags.get(first_node_at_height(i)..)?,
+            )
         } else {
-            old_bytes.get(byte_range_at_height(i))
-        }?;
+            (
+                old_bytes.get(byte_range_at_height(i))?,
+                old_flags.get(node_range_at_height(i))?
+            )
+        };
 
-        let new_slice = bytes
+        bytes
             .get_mut(byte_range_at_height(i + to_height - from_height))?
-            .get_mut(0..old_slice.len())?;
-
-        new_slice.copy_from_slice(old_slice);
+            .get_mut(0..byte_slice.len())?
+            .copy_from_slice(byte_slice);
+        flags
+            .get_mut(node_range_at_height(i + to_height - from_height))?
+            .get_mut(0..flag_slice.len())?
+            .copy_from_slice(flag_slice);
     }
 
-    Some(bytes)
-}
-
-/*
-fn copy_bytes(
-    from_range: Range<usize>,
-    to_range: Range<usize>,
-    from: &[u8],
-    to: &mut Vec<u8>,
-) -> Option<()> {
-    let from_slice = from.get(node_range_to_byte_range(from_range));
-
-    let to_slice = to
-        .get_mut(byte_range_at_height(i + to_height - from_height))?
-        .get_mut(0..old_slice.len())?;
-
-    Ok(())
-}
-*/
-
-fn node_range_to_byte_range(node_range: Range<usize>) -> Range<usize> {
-    node_range.start * HASHSIZE..node_range.end * HASHSIZE
+    Some((bytes, flags))
 }
 
 fn byte_range_at_height(h: usize) -> Range<usize> {
     first_byte_at_height(h)..last_node_at_height(h) * HASHSIZE
 }
 
+fn node_range_at_height(h: usize) -> Range<usize> {
+    first_node_at_height(h)..last_node_at_height(h)
+}
+
 fn first_byte_at_height(h: usize) -> usize {
     first_node_at_height(h) * HASHSIZE
 }
@@ -87,14 +78,14 @@ mod test {
     use super::*;
 
     #[test]
-    fn can_grow() {
-        let from: usize = 7;
+    fn can_grow_three_levels() {
+        let from: usize = 1;
         let to: usize = 15;
 
         let old_bytes = vec![42; from * HASHSIZE];
         let old_flags = vec![false; from];
 
-        let new = grow_merkle_cache(
+        let (new_bytes, new_flags) = grow_merkle_cache(
             &old_bytes,
             &old_flags,
             (from + 1).trailing_zeros() as usize,
@@ -102,28 +93,101 @@ mod test {
         )
         .unwrap();
 
-        println!("{:?}", new);
-        let mut expected = vec![];
+        let mut expected_bytes = vec![];
+        let mut expected_flags = vec![];
         // First level
-        expected.append(&mut vec![0; 32]);
+        expected_bytes.append(&mut vec![0; 32]);
+        expected_flags.push(true);
         // Second level
-        expected.append(&mut vec![42; 32]);
-        expected.append(&mut vec![0; 32]);
+        expected_bytes.append(&mut vec![0; 32]);
+        expected_bytes.append(&mut vec![0; 32]);
+        expected_flags.push(true);
+        expected_flags.push(true);
         // Third level
-        expected.append(&mut vec![42; 32]);
-        expected.append(&mut vec![42; 32]);
-        expected.append(&mut vec![0; 32]);
-        expected.append(&mut vec![0; 32]);
+        expected_bytes.append(&mut vec![0; 32]);
+        expected_bytes.append(&mut vec![0; 32]);
+        expected_bytes.append(&mut vec![0; 32]);
+        expected_bytes.append(&mut vec![0; 32]);
+        expected_flags.push(true);
+        expected_flags.push(true);
+        expected_flags.push(true);
+        expected_flags.push(true);
         // Fourth level
-        expected.append(&mut vec![0; 32]);
-        expected.append(&mut vec![0; 32]);
-        expected.append(&mut vec![0; 32]);
-        expected.append(&mut vec![0; 32]);
-        expected.append(&mut vec![0; 32]);
-        expected.append(&mut vec![0; 32]);
-        expected.append(&mut vec![0; 32]);
-        expected.append(&mut vec![0; 32]);
+        expected_bytes.append(&mut vec![42; 32]);
+        expected_bytes.append(&mut vec![0; 32]);
+        expected_bytes.append(&mut vec![0; 32]);
+        expected_bytes.append(&mut vec![0; 32]);
+        expected_bytes.append(&mut vec![0; 32]);
+        expected_bytes.append(&mut vec![0; 32]);
+        expected_bytes.append(&mut vec![0; 32]);
+        expected_bytes.append(&mut vec![0; 32]);
+        expected_flags.push(false);
+        expected_flags.push(true);
+        expected_flags.push(true);
+        expected_flags.push(true);
+        expected_flags.push(true);
+        expected_flags.push(true);
+        expected_flags.push(true);
+        expected_flags.push(true);
 
-        assert_eq!(expected, new);
+        assert_eq!(expected_bytes, new_bytes);
+        assert_eq!(expected_flags, new_flags);
+    }
+
+    #[test]
+    fn can_grow_one_level() {
+        let from: usize = 7;
+        let to: usize = 15;
+
+        let old_bytes = vec![42; from * HASHSIZE];
+        let old_flags = vec![false; from];
+
+        let (new_bytes, new_flags) = grow_merkle_cache(
+            &old_bytes,
+            &old_flags,
+            (from + 1).trailing_zeros() as usize,
+            (to + 1).trailing_zeros() as usize,
+        )
+        .unwrap();
+
+        let mut expected_bytes = vec![];
+        let mut expected_flags = vec![];
+        // First level
+        expected_bytes.append(&mut vec![0; 32]);
+        expected_flags.push(true);
+        // Second level
+        expected_bytes.append(&mut vec![42; 32]);
+        expected_bytes.append(&mut vec![0; 32]);
+        expected_flags.push(false);
+        expected_flags.push(true);
+        // Third level
+        expected_bytes.append(&mut vec![42; 32]);
+        expected_bytes.append(&mut vec![42; 32]);
+        expected_bytes.append(&mut vec![0; 32]);
+        expected_bytes.append(&mut vec![0; 32]);
+        expected_flags.push(false);
+        expected_flags.push(false);
+        expected_flags.push(true);
+        expected_flags.push(true);
+        // Fourth level
+        expected_bytes.append(&mut vec![42; 32]);
+        expected_bytes.append(&mut vec![42; 32]);
+        expected_bytes.append(&mut vec![42; 32]);
+        expected_bytes.append(&mut vec![42; 32]);
+        expected_bytes.append(&mut vec![0; 32]);
+        expected_bytes.append(&mut vec![0; 32]);
+        expected_bytes.append(&mut vec![0; 32]);
+        expected_bytes.append(&mut vec![0; 32]);
+        expected_flags.push(false);
+        expected_flags.push(false);
+        expected_flags.push(false);
+        expected_flags.push(false);
+        expected_flags.push(true);
+        expected_flags.push(true);
+        expected_flags.push(true);
+        expected_flags.push(true);
+
+        assert_eq!(expected_bytes, new_bytes);
+        assert_eq!(expected_flags, new_flags);
     }
 }
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index d784a0889..9b8e81bc9 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -1,5 +1,5 @@
+#![cfg(test)]
 use super::*;
-use crate::Encodable;
 use int_to_bytes::{int_to_bytes32, int_to_bytes8};
 
 #[derive(Clone, Debug)]

From e038bd18b59033712746de8927dc23f8ce4d2430 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Sun, 14 Apr 2019 10:34:54 +1000
Subject: [PATCH 033/137] Add failing test for grow merkle tree

---
 eth2/utils/ssz/src/cached_tree_hash.rs       | 24 ++++++++++++++++++--
 eth2/utils/ssz/src/cached_tree_hash/impls.rs | 24 +++++++++++++++++---
 eth2/utils/ssz/src/cached_tree_hash/tests.rs |  2 --
 3 files changed, 43 insertions(+), 7 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index 84ef82233..42faf1211 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -16,6 +16,8 @@ pub enum Error {
     ShouldNotProduceOffsetHandler,
     NoFirstNode,
     NoBytesForRoot,
+    UnableToObtainSlices,
+    UnableToGrowMerkleTree,
     BytesAreNotEvenChunks(usize),
     NoModifiedFieldForChunk(usize),
     NoBytesForChunk(usize),
@@ -74,6 +76,13 @@ impl TreeHashCache {
         item.build_tree_hash_cache()
     }
 
+    pub fn from_elems(cache: Vec<u8>, chunk_modified: Vec<bool>) -> Self {
+        Self {
+            cache,
+            chunk_modified,
+        }
+    }
+
     pub fn from_leaves_and_subtrees<T>(
         item: &T,
         leaves_and_subtrees: Vec<Self>,
@@ -149,7 +158,7 @@ impl TreeHashCache {
         // Update the `chunk_modified` vec, marking all spliced-in nodes as changed.
         self.chunk_modified.splice(chunk_range.clone(), bools);
         self.cache
-            .splice(node_range_to_byte_range(chunk_range), bytes);
+            .splice(node_range_to_byte_range(&chunk_range), bytes);
     }
 
     pub fn maybe_update_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> {
@@ -167,6 +176,13 @@ impl TreeHashCache {
         Ok(())
     }
 
+    pub fn slices(&self, chunk_range: Range<usize>) -> Option<(&[u8], &[bool])> {
+        Some((
+            self.cache.get(node_range_to_byte_range(&chunk_range))?,
+            self.chunk_modified.get(chunk_range)?,
+        ))
+    }
+
     pub fn modify_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> {
         let start = chunk * BYTES_PER_CHUNK;
         let end = start + BYTES_PER_CHUNK;
@@ -231,7 +247,7 @@ fn num_nodes(num_leaves: usize) -> usize {
     2 * num_leaves - 1
 }
 
-fn node_range_to_byte_range(node_range: Range<usize>) -> Range<usize> {
+fn node_range_to_byte_range(node_range: &Range<usize>) -> Range<usize> {
     node_range.start * HASHSIZE..node_range.end * HASHSIZE
 }
 
@@ -281,6 +297,10 @@ impl OffsetHandler {
         })
     }
 
+    pub fn node_range(&self) -> Result<Range<usize>, Error> {
+        Ok(*self.offsets.first().ok_or_else(|| Error::NoFirstNode)?..self.next_node())
+    }
+
     pub fn total_nodes(&self) -> usize {
         self.num_internal_nodes + self.num_leaf_nodes
     }
diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index dca00b6ba..2010aeb0d 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -113,9 +113,27 @@ where
     ) -> Result<usize, Error> {
         let offset_handler = OffsetHandler::new(self, chunk)?;
 
-        if other.len().next_power_of_two() > self.len().next_power_of_two() {
-            //
-        } else if other.len().next_power_of_two() < self.len().next_power_of_two() {
+        if self.len().next_power_of_two() > other.len().next_power_of_two() {
+            // Get slices of the exsiting tree from the cache.
+            let (old_bytes, old_flags) = cache
+                .slices(offset_handler.node_range()?)
+                .ok_or_else(|| Error::UnableToObtainSlices)?;
+
+            // From the existing slices build new, expanded Vecs.
+            let (new_bytes, new_flags) = grow_merkle_cache(
+                old_bytes,
+                old_flags,
+                other.len().next_power_of_two().leading_zeros() as usize,
+                self.len().next_power_of_two().leading_zeros() as usize,
+            ).ok_or_else(|| Error::UnableToGrowMerkleTree)?;
+
+            // Create a `TreeHashCache` from the raw elements.
+            let expanded_cache = TreeHashCache::from_elems(new_bytes, new_flags);
+
+            // Splice the newly created `TreeHashCache` over the existing, smaller elements.
+            cache.splice(offset_handler.node_range()?, expanded_cache);
+        //
+        } else if self.len().next_power_of_two() < other.len().next_power_of_two() {
             panic!("shrinking below power of two is not implemented")
         }
 
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index 9b8e81bc9..c402fd15b 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -408,7 +408,6 @@ fn extended_u64_vec_len_within_pow_2_boundary() {
     test_u64_vec_modifications(original_vec, modified_vec);
 }
 
-/*
 #[test]
 fn extended_u64_vec_len_outside_pow_2_boundary() {
     let original_vec: Vec<u64> = (0..2_u64.pow(5)).collect();
@@ -417,7 +416,6 @@ fn extended_u64_vec_len_outside_pow_2_boundary() {
 
     test_u64_vec_modifications(original_vec, modified_vec);
 }
-*/
 
 #[test]
 fn large_vec_of_u64_builds() {

From 737e6b9a866beaf1974249644c3474b7168f2901 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Sun, 14 Apr 2019 13:54:04 +1000
Subject: [PATCH 034/137] Fix failing tree hash test

---
 eth2/utils/ssz/src/cached_tree_hash.rs        | 10 ++++-
 eth2/utils/ssz/src/cached_tree_hash/impls.rs  | 14 +++++--
 eth2/utils/ssz/src/cached_tree_hash/resize.rs | 39 ++++++++++---------
 3 files changed, 39 insertions(+), 24 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index 42faf1211..0e6bdf986 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -255,6 +255,7 @@ fn node_range_to_byte_range(node_range: &Range<usize>) -> Range<usize> {
 pub struct OffsetHandler {
     num_internal_nodes: usize,
     pub num_leaf_nodes: usize,
+    first_node: usize,
     next_node: usize,
     offsets: Vec<usize>,
 }
@@ -293,12 +294,17 @@ impl OffsetHandler {
             num_internal_nodes,
             num_leaf_nodes,
             offsets,
+            first_node: offset,
             next_node,
         })
     }
 
-    pub fn node_range(&self) -> Result<Range<usize>, Error> {
-        Ok(*self.offsets.first().ok_or_else(|| Error::NoFirstNode)?..self.next_node())
+    pub fn height(&self) -> usize {
+        self.num_leaf_nodes.trailing_zeros() as usize
+    }
+
+    pub fn node_range(&self) -> Range<usize> {
+        self.first_node..self.next_node
     }
 
     pub fn total_nodes(&self) -> usize {
diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index 2010aeb0d..c55415e54 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -113,25 +113,31 @@ where
     ) -> Result<usize, Error> {
         let offset_handler = OffsetHandler::new(self, chunk)?;
 
+        // Check to see if the length of the list has changed length beyond a power-of-two
+        // boundary. In such a case we need to resize the merkle tree bytes.
         if self.len().next_power_of_two() > other.len().next_power_of_two() {
+            let old_offset_handler = OffsetHandler::new(other, chunk)?;
+
+            dbg!(old_offset_handler.node_range());
+
             // Get slices of the exsiting tree from the cache.
             let (old_bytes, old_flags) = cache
-                .slices(offset_handler.node_range()?)
+                .slices(old_offset_handler.node_range())
                 .ok_or_else(|| Error::UnableToObtainSlices)?;
 
             // From the existing slices build new, expanded Vecs.
             let (new_bytes, new_flags) = grow_merkle_cache(
                 old_bytes,
                 old_flags,
-                other.len().next_power_of_two().leading_zeros() as usize,
-                self.len().next_power_of_two().leading_zeros() as usize,
+                old_offset_handler.height(),
+                offset_handler.height(),
             ).ok_or_else(|| Error::UnableToGrowMerkleTree)?;
 
             // Create a `TreeHashCache` from the raw elements.
             let expanded_cache = TreeHashCache::from_elems(new_bytes, new_flags);
 
             // Splice the newly created `TreeHashCache` over the existing, smaller elements.
-            cache.splice(offset_handler.node_range()?, expanded_cache);
+            cache.splice(old_offset_handler.node_range(), expanded_cache);
         //
         } else if self.len().next_power_of_two() < other.len().next_power_of_two() {
             panic!("shrinking below power of two is not implemented")
diff --git a/eth2/utils/ssz/src/cached_tree_hash/resize.rs b/eth2/utils/ssz/src/cached_tree_hash/resize.rs
index bce722a5e..2cdce4827 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/resize.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/resize.rs
@@ -7,22 +7,20 @@ pub fn grow_merkle_cache(
     from_height: usize,
     to_height: usize,
 ) -> Option<(Vec<u8>, Vec<bool>)> {
-    let to_nodes = (1 << to_height.next_power_of_two()) - 1;
-
     // Determine the size of our new tree. It is not just a simple `1 << to_height` as there can be
-    // an arbitrary number of bytes in `old_bytes` leaves.
-    let new_byte_count = {
-        let additional_from_nodes = old_bytes.len() / HASHSIZE - ((1 << from_height) - 1);
-        ((1 << to_height + additional_from_nodes) - 1) * HASHSIZE
+    // an arbitrary number of nodes in `old_bytes` leaves if those leaves are subtrees.
+    let to_nodes = {
+        let old_nodes = old_bytes.len() / HASHSIZE;
+        let additional_nodes = old_nodes - nodes_in_tree_of_height(from_height);
+        nodes_in_tree_of_height(to_height) + additional_nodes
     };
 
-    let mut bytes = vec![0; new_byte_count];
+    let mut bytes = vec![0; to_nodes * HASHSIZE];
     let mut flags = vec![true; to_nodes];
 
-    let leaf_level = from_height - 1;
+    let leaf_level = from_height;
 
-    // Loop through all internal levels of the tree (skipping the final, leaves level).
-    for i in 0..from_height as usize {
+    for i in 0..=from_height as usize {
         // If we're on the leaf slice, grab the first byte and all the of the bytes after that.
         // This is required because we can have an arbitrary number of bytes at the leaf level
         // (e.g., the case where there are subtrees as leaves).
@@ -36,7 +34,7 @@ pub fn grow_merkle_cache(
         } else {
             (
                 old_bytes.get(byte_range_at_height(i))?,
-                old_flags.get(node_range_at_height(i))?
+                old_flags.get(node_range_at_height(i))?,
             )
         };
 
@@ -53,12 +51,17 @@ pub fn grow_merkle_cache(
     Some((bytes, flags))
 }
 
+fn nodes_in_tree_of_height(h: usize) -> usize {
+    2 * (1 << h) - 1
+}
+
 fn byte_range_at_height(h: usize) -> Range<usize> {
-    first_byte_at_height(h)..last_node_at_height(h) * HASHSIZE
+    let node_range = node_range_at_height(h);
+    node_range.start * HASHSIZE..node_range.end * HASHSIZE
 }
 
 fn node_range_at_height(h: usize) -> Range<usize> {
-    first_node_at_height(h)..last_node_at_height(h)
+    first_node_at_height(h)..last_node_at_height(h) + 1
 }
 
 fn first_byte_at_height(h: usize) -> usize {
@@ -70,7 +73,7 @@ fn first_node_at_height(h: usize) -> usize {
 }
 
 fn last_node_at_height(h: usize) -> usize {
-    (1 << (h + 1)) - 1
+    (1 << (h + 1)) - 2
 }
 
 #[cfg(test)]
@@ -88,8 +91,8 @@ mod test {
         let (new_bytes, new_flags) = grow_merkle_cache(
             &old_bytes,
             &old_flags,
-            (from + 1).trailing_zeros() as usize,
-            (to + 1).trailing_zeros() as usize,
+            (from + 1).trailing_zeros() as usize - 1,
+            (to + 1).trailing_zeros() as usize - 1,
         )
         .unwrap();
 
@@ -145,8 +148,8 @@ mod test {
         let (new_bytes, new_flags) = grow_merkle_cache(
             &old_bytes,
             &old_flags,
-            (from + 1).trailing_zeros() as usize,
-            (to + 1).trailing_zeros() as usize,
+            (from + 1).trailing_zeros() as usize - 1,
+            (to + 1).trailing_zeros() as usize - 1,
         )
         .unwrap();
 

From 582f465ffd9ff57bcad2933a47547bcace5aa0a6 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Sun, 14 Apr 2019 14:20:33 +1000
Subject: [PATCH 035/137] Add test for growing vec of structs

---
 eth2/utils/ssz/src/cached_tree_hash/resize.rs | 29 ++++++++----
 eth2/utils/ssz/src/cached_tree_hash/tests.rs  | 47 ++++++++++++++++++-
 2 files changed, 65 insertions(+), 11 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash/resize.rs b/eth2/utils/ssz/src/cached_tree_hash/resize.rs
index 2cdce4827..3c2d2c407 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/resize.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/resize.rs
@@ -26,7 +26,7 @@ pub fn grow_merkle_cache(
         // (e.g., the case where there are subtrees as leaves).
         //
         // If we're not on a leaf level, the number of nodes is fixed and known.
-        let (byte_slice, flag_slice) = if i == leaf_level {
+        let (old_byte_slice, old_flag_slice) = if i == leaf_level {
             (
                 old_bytes.get(first_byte_at_height(i)..)?,
                 old_flags.get(first_node_at_height(i)..)?,
@@ -38,14 +38,25 @@ pub fn grow_merkle_cache(
             )
         };
 
-        bytes
-            .get_mut(byte_range_at_height(i + to_height - from_height))?
-            .get_mut(0..byte_slice.len())?
-            .copy_from_slice(byte_slice);
-        flags
-            .get_mut(node_range_at_height(i + to_height - from_height))?
-            .get_mut(0..flag_slice.len())?
-            .copy_from_slice(flag_slice);
+        let new_i = i + to_height - from_height;
+        let (new_byte_slice, new_flag_slice) = if i == leaf_level {
+            (
+                bytes.get_mut(first_byte_at_height(new_i)..)?,
+                flags.get_mut(first_node_at_height(new_i)..)?,
+            )
+        } else {
+            (
+                bytes.get_mut(byte_range_at_height(new_i))?,
+                flags.get_mut(node_range_at_height(new_i))?,
+            )
+        };
+
+        new_byte_slice
+            .get_mut(0..old_byte_slice.len())?
+            .copy_from_slice(old_byte_slice);
+        new_flag_slice
+            .get_mut(0..old_flag_slice.len())?
+            .copy_from_slice(old_flag_slice);
     }
 
     Some((bytes, flags))
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index c402fd15b..fb6ed9080 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -472,9 +472,10 @@ fn test_inner_vec_modifications(original: Vec<Inner>, modified: Vec<Inner>, refe
     }
 
     let num_leaves = leaves.len() / HASHSIZE;
-
     let mut expected = merkleize(leaves);
-    expected.splice(3 * HASHSIZE.., full_bytes);
+
+    let num_internal_nodes = num_leaves.next_power_of_two() - 1;
+    expected.splice(num_internal_nodes * HASHSIZE.., full_bytes);
 
     for _ in num_leaves..num_leaves.next_power_of_two() {
         expected.append(&mut vec![0; HASHSIZE]);
@@ -589,6 +590,48 @@ fn lengthened_vec_of_inner_within_power_of_two_boundary() {
     test_inner_vec_modifications(original, modified, reference_vec);
 }
 
+#[test]
+fn lengthened_vec_of_inner_outside_power_of_two_boundary() {
+    let original = vec![
+        Inner {
+            a: 0,
+            b: 1,
+            c: 2,
+            d: 3,
+        },
+        Inner {
+            a: 4,
+            b: 5,
+            c: 6,
+            d: 7,
+        },
+        Inner {
+            a: 8,
+            b: 9,
+            c: 10,
+            d: 11,
+        },
+        Inner {
+            a: 12,
+            b: 13,
+            c: 14,
+            d: 15,
+        },
+    ];
+
+    let mut modified = original.clone();
+    modified.push(Inner {
+        a: 16,
+        b: 17,
+        c: 18,
+        d: 19,
+    });
+
+    let reference_vec: Vec<u64> = (0..20).collect();
+
+    test_inner_vec_modifications(original, modified, reference_vec);
+}
+
 #[test]
 fn vec_of_inner_builds() {
     let numbers: Vec<u64> = (0..12).collect();

From 9bc0519092e31a834719a706d0277eb022ad3678 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Sun, 14 Apr 2019 16:31:47 +1000
Subject: [PATCH 036/137] Add tree shrink fn

---
 eth2/utils/ssz/src/cached_tree_hash/resize.rs | 118 ++++++++++++++----
 1 file changed, 94 insertions(+), 24 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash/resize.rs b/eth2/utils/ssz/src/cached_tree_hash/resize.rs
index 3c2d2c407..0b492770f 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/resize.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/resize.rs
@@ -62,6 +62,52 @@ pub fn grow_merkle_cache(
     Some((bytes, flags))
 }
 
+/// New vec is smaller than old vec.
+pub fn shrink_merkle_cache(
+    from_bytes: &[u8],
+    from_flags: &[bool],
+    from_height: usize,
+    to_height: usize,
+    to_nodes: usize,
+) -> Option<(Vec<u8>, Vec<bool>)> {
+    let mut bytes = vec![0; to_nodes * HASHSIZE];
+    let mut flags = vec![true; to_nodes];
+
+    let leaf_level = to_height;
+
+    for i in 0..=leaf_level as usize {
+        let from_i = i + from_height - to_height;
+        let (from_byte_slice, from_flag_slice) = if from_i == leaf_level {
+            (
+                from_bytes.get(first_byte_at_height(from_i)..)?,
+                from_flags.get(first_node_at_height(from_i)..)?,
+            )
+        } else {
+            (
+                from_bytes.get(byte_range_at_height(from_i))?,
+                from_flags.get(node_range_at_height(from_i))?,
+            )
+        };
+
+        let (to_byte_slice, to_flag_slice) = if i == leaf_level {
+            (
+                bytes.get_mut(first_byte_at_height(i)..)?,
+                flags.get_mut(first_node_at_height(i)..)?,
+            )
+        } else {
+            (
+                bytes.get_mut(byte_range_at_height(i))?,
+                flags.get_mut(node_range_at_height(i))?,
+            )
+        };
+
+        to_byte_slice.copy_from_slice(from_byte_slice.get(0..to_byte_slice.len())?);
+        to_flag_slice.copy_from_slice(from_flag_slice.get(0..to_flag_slice.len())?);
+    }
+
+    Some((bytes, flags))
+}
+
 fn nodes_in_tree_of_height(h: usize) -> usize {
     2 * (1 << h) - 1
 }
@@ -92,18 +138,18 @@ mod test {
     use super::*;
 
     #[test]
-    fn can_grow_three_levels() {
-        let from: usize = 1;
-        let to: usize = 15;
+    fn can_grow_and_shrink_three_levels() {
+        let small: usize = 1;
+        let big: usize = 15;
 
-        let old_bytes = vec![42; from * HASHSIZE];
-        let old_flags = vec![false; from];
+        let original_bytes = vec![42; small * HASHSIZE];
+        let original_flags = vec![false; small];
 
-        let (new_bytes, new_flags) = grow_merkle_cache(
-            &old_bytes,
-            &old_flags,
-            (from + 1).trailing_zeros() as usize - 1,
-            (to + 1).trailing_zeros() as usize - 1,
+        let (grown_bytes, grown_flags) = grow_merkle_cache(
+            &original_bytes,
+            &original_flags,
+            (small + 1).trailing_zeros() as usize - 1,
+            (big + 1).trailing_zeros() as usize - 1,
         )
         .unwrap();
 
@@ -144,23 +190,35 @@ mod test {
         expected_flags.push(true);
         expected_flags.push(true);
 
-        assert_eq!(expected_bytes, new_bytes);
-        assert_eq!(expected_flags, new_flags);
+        assert_eq!(expected_bytes, grown_bytes);
+        assert_eq!(expected_flags, grown_flags);
+
+        let (shrunk_bytes, shrunk_flags) = shrink_merkle_cache(
+            &grown_bytes,
+            &grown_flags,
+            (big + 1).trailing_zeros() as usize - 1,
+            (small + 1).trailing_zeros() as usize - 1,
+            small,
+        )
+        .unwrap();
+
+        assert_eq!(original_bytes, shrunk_bytes);
+        assert_eq!(original_flags, shrunk_flags);
     }
 
     #[test]
-    fn can_grow_one_level() {
-        let from: usize = 7;
-        let to: usize = 15;
+    fn can_grow_and_shrink_one_level() {
+        let small: usize = 7;
+        let big: usize = 15;
 
-        let old_bytes = vec![42; from * HASHSIZE];
-        let old_flags = vec![false; from];
+        let original_bytes = vec![42; small * HASHSIZE];
+        let original_flags = vec![false; small];
 
-        let (new_bytes, new_flags) = grow_merkle_cache(
-            &old_bytes,
-            &old_flags,
-            (from + 1).trailing_zeros() as usize - 1,
-            (to + 1).trailing_zeros() as usize - 1,
+        let (grown_bytes, grown_flags) = grow_merkle_cache(
+            &original_bytes,
+            &original_flags,
+            (small + 1).trailing_zeros() as usize - 1,
+            (big + 1).trailing_zeros() as usize - 1,
         )
         .unwrap();
 
@@ -201,7 +259,19 @@ mod test {
         expected_flags.push(true);
         expected_flags.push(true);
 
-        assert_eq!(expected_bytes, new_bytes);
-        assert_eq!(expected_flags, new_flags);
+        assert_eq!(expected_bytes, grown_bytes);
+        assert_eq!(expected_flags, grown_flags);
+
+        let (shrunk_bytes, shrunk_flags) = shrink_merkle_cache(
+            &grown_bytes,
+            &grown_flags,
+            (big + 1).trailing_zeros() as usize - 1,
+            (small + 1).trailing_zeros() as usize - 1,
+            small,
+        )
+        .unwrap();
+
+        assert_eq!(original_bytes, shrunk_bytes);
+        assert_eq!(original_flags, shrunk_flags);
     }
 }

From da74c4ce74e4fff958434a01200aad6b25478c88 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Sun, 14 Apr 2019 16:50:00 +1000
Subject: [PATCH 037/137] Add tree shrinking for u64 vec

---
 eth2/utils/ssz/src/cached_tree_hash/impls.rs | 38 +++++++++++---------
 eth2/utils/ssz/src/cached_tree_hash/tests.rs |  9 +++++
 2 files changed, 31 insertions(+), 16 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index c55415e54..f16e6a62b 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -1,4 +1,4 @@
-use super::resize::grow_merkle_cache;
+use super::resize::{grow_merkle_cache, shrink_merkle_cache};
 use super::*;
 use crate::ssz_encode;
 
@@ -112,35 +112,41 @@ where
         chunk: usize,
     ) -> Result<usize, Error> {
         let offset_handler = OffsetHandler::new(self, chunk)?;
+        let old_offset_handler = OffsetHandler::new(other, chunk)?;
 
-        // Check to see if the length of the list has changed length beyond a power-of-two
-        // boundary. In such a case we need to resize the merkle tree bytes.
-        if self.len().next_power_of_two() > other.len().next_power_of_two() {
+        if offset_handler.num_leaf_nodes != old_offset_handler.num_leaf_nodes {
             let old_offset_handler = OffsetHandler::new(other, chunk)?;
 
-            dbg!(old_offset_handler.node_range());
-
             // Get slices of the exsiting tree from the cache.
             let (old_bytes, old_flags) = cache
                 .slices(old_offset_handler.node_range())
                 .ok_or_else(|| Error::UnableToObtainSlices)?;
 
-            // From the existing slices build new, expanded Vecs.
-            let (new_bytes, new_flags) = grow_merkle_cache(
-                old_bytes,
-                old_flags,
-                old_offset_handler.height(),
-                offset_handler.height(),
-            ).ok_or_else(|| Error::UnableToGrowMerkleTree)?;
+            let (new_bytes, new_flags) =
+                if offset_handler.num_leaf_nodes > old_offset_handler.num_leaf_nodes {
+                    grow_merkle_cache(
+                        old_bytes,
+                        old_flags,
+                        old_offset_handler.height(),
+                        offset_handler.height(),
+                    )
+                    .ok_or_else(|| Error::UnableToGrowMerkleTree)?
+                } else {
+                    shrink_merkle_cache(
+                        old_bytes,
+                        old_flags,
+                        old_offset_handler.height(),
+                        offset_handler.height(),
+                        offset_handler.total_nodes(),
+                    )
+                    .ok_or_else(|| Error::UnableToGrowMerkleTree)?
+                };
 
             // Create a `TreeHashCache` from the raw elements.
             let expanded_cache = TreeHashCache::from_elems(new_bytes, new_flags);
 
             // Splice the newly created `TreeHashCache` over the existing, smaller elements.
             cache.splice(old_offset_handler.node_range(), expanded_cache);
-        //
-        } else if self.len().next_power_of_two() < other.len().next_power_of_two() {
-            panic!("shrinking below power of two is not implemented")
         }
 
         match T::item_type() {
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index fb6ed9080..22d01ec1a 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -396,6 +396,15 @@ fn shortened_u64_vec_len_within_pow_2_boundary() {
     test_u64_vec_modifications(original_vec, modified_vec);
 }
 
+#[test]
+fn shortened_u64_vec_len_outside_pow_2_boundary() {
+    let original_vec: Vec<u64> = (0..2_u64.pow(6)).collect();
+
+    let modified_vec: Vec<u64> = (0..2_u64.pow(5)).collect();
+
+    test_u64_vec_modifications(original_vec, modified_vec);
+}
+
 #[test]
 fn extended_u64_vec_len_within_pow_2_boundary() {
     let n: u64 = 2_u64.pow(5) - 2;

From 0632a00a48f6da3cd9a3e7ac1df332e6b12aee92 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Sun, 14 Apr 2019 18:50:12 +1000
Subject: [PATCH 038/137] Fix failing test for shrinking vec of structs

---
 eth2/utils/ssz/src/cached_tree_hash.rs        |  7 ++-
 eth2/utils/ssz/src/cached_tree_hash/impls.rs  | 14 +++---
 eth2/utils/ssz/src/cached_tree_hash/resize.rs |  9 ++--
 eth2/utils/ssz/src/cached_tree_hash/tests.rs  | 43 +++++++++++++++++++
 4 files changed, 59 insertions(+), 14 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index 0e6bdf986..7a722766c 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -18,6 +18,7 @@ pub enum Error {
     NoBytesForRoot,
     UnableToObtainSlices,
     UnableToGrowMerkleTree,
+    UnableToShrinkMerkleTree,
     BytesAreNotEvenChunks(usize),
     NoModifiedFieldForChunk(usize),
     NoBytesForChunk(usize),
@@ -303,10 +304,14 @@ impl OffsetHandler {
         self.num_leaf_nodes.trailing_zeros() as usize
     }
 
-    pub fn node_range(&self) -> Range<usize> {
+    pub fn chunk_range(&self) -> Range<usize> {
         self.first_node..self.next_node
     }
 
+    pub fn total_chunks(&self) -> usize {
+        self.next_node - self.first_node
+    }
+
     pub fn total_nodes(&self) -> usize {
         self.num_internal_nodes + self.num_leaf_nodes
     }
diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index f16e6a62b..0377649cb 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -119,7 +119,7 @@ where
 
             // Get slices of the exsiting tree from the cache.
             let (old_bytes, old_flags) = cache
-                .slices(old_offset_handler.node_range())
+                .slices(old_offset_handler.chunk_range())
                 .ok_or_else(|| Error::UnableToObtainSlices)?;
 
             let (new_bytes, new_flags) =
@@ -137,16 +137,16 @@ where
                         old_flags,
                         old_offset_handler.height(),
                         offset_handler.height(),
-                        offset_handler.total_nodes(),
+                        offset_handler.total_chunks(),
                     )
-                    .ok_or_else(|| Error::UnableToGrowMerkleTree)?
+                    .ok_or_else(|| Error::UnableToShrinkMerkleTree)?
                 };
 
             // Create a `TreeHashCache` from the raw elements.
-            let expanded_cache = TreeHashCache::from_elems(new_bytes, new_flags);
+            let modified_cache = TreeHashCache::from_elems(new_bytes, new_flags);
 
-            // Splice the newly created `TreeHashCache` over the existing, smaller elements.
-            cache.splice(old_offset_handler.node_range(), expanded_cache);
+            // Splice the newly created `TreeHashCache` over the existing elements.
+            cache.splice(old_offset_handler.chunk_range(), modified_cache);
         }
 
         match T::item_type() {
@@ -208,8 +208,6 @@ where
 
         for (&parent, children) in offset_handler.iter_internal_nodes().rev() {
             if cache.either_modified(children)? {
-                dbg!(parent);
-                dbg!(children);
                 cache.modify_chunk(parent, &cache.hash_children(children)?)?;
             }
         }
diff --git a/eth2/utils/ssz/src/cached_tree_hash/resize.rs b/eth2/utils/ssz/src/cached_tree_hash/resize.rs
index 0b492770f..44b3f0ea5 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/resize.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/resize.rs
@@ -73,11 +73,10 @@ pub fn shrink_merkle_cache(
     let mut bytes = vec![0; to_nodes * HASHSIZE];
     let mut flags = vec![true; to_nodes];
 
-    let leaf_level = to_height;
-
-    for i in 0..=leaf_level as usize {
+    for i in 0..=to_height as usize {
         let from_i = i + from_height - to_height;
-        let (from_byte_slice, from_flag_slice) = if from_i == leaf_level {
+
+        let (from_byte_slice, from_flag_slice) = if from_i == from_height {
             (
                 from_bytes.get(first_byte_at_height(from_i)..)?,
                 from_flags.get(first_node_at_height(from_i)..)?,
@@ -89,7 +88,7 @@ pub fn shrink_merkle_cache(
             )
         };
 
-        let (to_byte_slice, to_flag_slice) = if i == leaf_level {
+        let (to_byte_slice, to_flag_slice) = if i == to_height {
             (
                 bytes.get_mut(first_byte_at_height(i)..)?,
                 flags.get_mut(first_node_at_height(i)..)?,
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index 22d01ec1a..f09fac419 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -563,6 +563,49 @@ fn shortened_vec_of_inner_within_power_of_two_boundary() {
     test_inner_vec_modifications(original, modified, reference_vec);
 }
 
+#[test]
+fn shortened_vec_of_inner_outside_power_of_two_boundary() {
+    let original = vec![
+        Inner {
+            a: 0,
+            b: 1,
+            c: 2,
+            d: 3,
+        },
+        Inner {
+            a: 4,
+            b: 5,
+            c: 6,
+            d: 7,
+        },
+        Inner {
+            a: 8,
+            b: 9,
+            c: 10,
+            d: 11,
+        },
+        Inner {
+            a: 12,
+            b: 13,
+            c: 14,
+            d: 15,
+        },
+        Inner {
+            a: 16,
+            b: 17,
+            c: 18,
+            d: 19,
+        },
+    ];
+
+    let mut modified = original.clone();
+    modified.pop(); // remove the last element from the list.
+
+    let reference_vec: Vec<u64> = (0..16).collect();
+
+    test_inner_vec_modifications(original, modified, reference_vec);
+}
+
 #[test]
 fn lengthened_vec_of_inner_within_power_of_two_boundary() {
     let original = vec![

From ab78a15313f3604c8d0116bdbda2ea81bb7750b7 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Sun, 14 Apr 2019 21:39:36 +1000
Subject: [PATCH 039/137] Add mix-in-len to cached tree hash

---
 eth2/utils/ssz/src/cached_tree_hash.rs       | 14 ++++++++++++++
 eth2/utils/ssz/src/cached_tree_hash/impls.rs |  6 ++++++
 eth2/utils/ssz/src/cached_tree_hash/tests.rs | 13 ++++++++++++-
 3 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index 7a722766c..8a7b07f50 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -1,4 +1,5 @@
 use hashing::hash;
+use int_to_bytes::int_to_bytes32;
 use std::fmt::Debug;
 use std::iter::Iterator;
 use std::ops::Range;
@@ -231,6 +232,15 @@ impl TreeHashCache {
         Ok(hash(&child_bytes))
     }
 
+    pub fn mix_in_length(&self, chunk: usize, length: usize) -> Result<Vec<u8>, Error> {
+        let mut bytes = Vec::with_capacity(2 * BYTES_PER_CHUNK);
+
+        bytes.append(&mut self.get_chunk(chunk)?.to_vec());
+        bytes.append(&mut int_to_bytes32(length as u64));
+
+        Ok(hash(&bytes))
+    }
+
     pub fn into_merkle_tree(self) -> Vec<u8> {
         self.cache
     }
@@ -300,6 +310,10 @@ impl OffsetHandler {
         })
     }
 
+    pub fn root(&self) -> usize {
+        self.first_node
+    }
+
     pub fn height(&self) -> usize {
         self.num_leaf_nodes.trailing_zeros() as usize
     }
diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index 0377649cb..558b4dde5 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -212,6 +212,12 @@ where
             }
         }
 
+        // If the root node or the length has changed, mix in the length of the list.
+        let root_node = offset_handler.root();
+        if cache.changed(root_node)? | (self.len() != other.len()) {
+            cache.modify_chunk(root_node, &cache.mix_in_length(root_node, self.len())?)?;
+        }
+
         Ok(offset_handler.next_node())
     }
 }
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index f09fac419..b85c16587 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -343,6 +343,13 @@ fn outer_builds() {
     assert_eq!(merkle, cache);
 }
 
+fn mix_in_length(root: &mut [u8], len: usize) {
+    let mut bytes = root.to_vec();
+    bytes.append(&mut int_to_bytes32(len as u64));
+
+    root.copy_from_slice(&hash(&bytes));
+}
+
 /// Generic test that covers:
 ///
 /// 1. Produce a new cache from `original`.
@@ -367,7 +374,9 @@ fn test_u64_vec_modifications(original: Vec<u64>, modified: Vec<u64>) {
         data.append(&mut int_to_bytes8(*i));
     }
     let data = sanitise_bytes(data);
-    let expected = merkleize(data);
+    let mut expected = merkleize(data);
+
+    mix_in_length(&mut expected[0..HASHSIZE], modified.len());
 
     assert_eq!(expected, modified_cache);
 }
@@ -490,6 +499,8 @@ fn test_inner_vec_modifications(original: Vec<Inner>, modified: Vec<Inner>, refe
         expected.append(&mut vec![0; HASHSIZE]);
     }
 
+    mix_in_length(&mut expected[0..HASHSIZE], modified.len());
+
     // Compare the cached tree to the reference tree.
     assert_trees_eq(&expected, &modified_cache);
 }

From 7132ee59c0789c623896cd8439648f2fb24a0a6e Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Mon, 15 Apr 2019 09:06:19 +1000
Subject: [PATCH 040/137] Rename OffsetHandler -> BTreeOverlay

---
 eth2/utils/ssz/src/cached_tree_hash.rs       |  8 ++++----
 eth2/utils/ssz/src/cached_tree_hash/impls.rs | 12 ++++++------
 eth2/utils/ssz/src/cached_tree_hash/tests.rs |  4 ++--
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
index 8a7b07f50..f7d18c57c 100644
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash.rs
@@ -14,7 +14,7 @@ const MERKLE_HASH_CHUNCK: usize = 2 * BYTES_PER_CHUNK;
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum Error {
-    ShouldNotProduceOffsetHandler,
+    ShouldNotProduceBTreeOverlay,
     NoFirstNode,
     NoBytesForRoot,
     UnableToObtainSlices,
@@ -92,7 +92,7 @@ impl TreeHashCache {
     where
         T: CachedTreeHash<T>,
     {
-        let offset_handler = OffsetHandler::new(item, 0)?;
+        let offset_handler = BTreeOverlay::new(item, 0)?;
 
         // Note how many leaves were provided. If is not a power-of-two, we'll need to pad it out
         // later.
@@ -263,7 +263,7 @@ fn node_range_to_byte_range(node_range: &Range<usize>) -> Range<usize> {
 }
 
 #[derive(Debug)]
-pub struct OffsetHandler {
+pub struct BTreeOverlay {
     num_internal_nodes: usize,
     pub num_leaf_nodes: usize,
     first_node: usize,
@@ -271,7 +271,7 @@ pub struct OffsetHandler {
     offsets: Vec<usize>,
 }
 
-impl OffsetHandler {
+impl BTreeOverlay {
     pub fn new<T>(item: &T, initial_offset: usize) -> Result<Self, Error>
     where
         T: CachedTreeHash<T>,
diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
index 558b4dde5..26905c667 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs
@@ -19,7 +19,7 @@ impl CachedTreeHash<u64> for u64 {
     }
 
     fn offsets(&self) -> Result<Vec<usize>, Error> {
-        Err(Error::ShouldNotProduceOffsetHandler)
+        Err(Error::ShouldNotProduceBTreeOverlay)
     }
 
     fn num_child_nodes(&self) -> usize {
@@ -78,7 +78,7 @@ where
                 let mut offsets = vec![];
 
                 for item in self {
-                    offsets.push(OffsetHandler::new(item, 0)?.total_nodes())
+                    offsets.push(BTreeOverlay::new(item, 0)?.total_nodes())
                 }
 
                 offsets
@@ -111,11 +111,11 @@ where
         cache: &mut TreeHashCache,
         chunk: usize,
     ) -> Result<usize, Error> {
-        let offset_handler = OffsetHandler::new(self, chunk)?;
-        let old_offset_handler = OffsetHandler::new(other, chunk)?;
+        let offset_handler = BTreeOverlay::new(self, chunk)?;
+        let old_offset_handler = BTreeOverlay::new(other, chunk)?;
 
         if offset_handler.num_leaf_nodes != old_offset_handler.num_leaf_nodes {
-            let old_offset_handler = OffsetHandler::new(other, chunk)?;
+            let old_offset_handler = BTreeOverlay::new(other, chunk)?;
 
             // Get slices of the exsiting tree from the cache.
             let (old_bytes, old_flags) = cache
@@ -180,7 +180,7 @@ where
                         (Some(old), None) => {
                             // Splice out the entire tree of the removed node, replacing it with a
                             // single padding node.
-                            let end_chunk = OffsetHandler::new(old, start_chunk)?.next_node();
+                            let end_chunk = BTreeOverlay::new(old, start_chunk)?.next_node();
 
                             cache.splice(
                                 start_chunk..end_chunk,
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
index b85c16587..e6e2b1754 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs
@@ -77,7 +77,7 @@ impl CachedTreeHash<Inner> for Inner {
         cache: &mut TreeHashCache,
         chunk: usize,
     ) -> Result<usize, Error> {
-        let offset_handler = OffsetHandler::new(self, chunk)?;
+        let offset_handler = BTreeOverlay::new(self, chunk)?;
 
         // Skip past the internal nodes and update any changed leaf nodes.
         {
@@ -166,7 +166,7 @@ impl CachedTreeHash<Outer> for Outer {
         cache: &mut TreeHashCache,
         chunk: usize,
     ) -> Result<usize, Error> {
-        let offset_handler = OffsetHandler::new(self, chunk)?;
+        let offset_handler = BTreeOverlay::new(self, chunk)?;
 
         // Skip past the internal nodes and update any changed leaf nodes.
         {

From 89bc15f77ed84a53d4c6c4abe707dd258011de66 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Mon, 15 Apr 2019 09:59:35 +1000
Subject: [PATCH 041/137] Move some long-running tests to release-only

---
 beacon_node/beacon_chain/test_harness/tests/chain.rs | 2 ++
 eth2/fork_choice/tests/tests.rs                      | 1 +
 eth2/state_processing/tests/tests.rs                 | 4 ++--
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/beacon_node/beacon_chain/test_harness/tests/chain.rs b/beacon_node/beacon_chain/test_harness/tests/chain.rs
index e72c3a5aa..d47de6889 100644
--- a/beacon_node/beacon_chain/test_harness/tests/chain.rs
+++ b/beacon_node/beacon_chain/test_harness/tests/chain.rs
@@ -1,3 +1,5 @@
+#![cfg(not(debug_assertions))]
+
 use env_logger::{Builder, Env};
 use log::debug;
 use test_harness::BeaconChainHarness;
diff --git a/eth2/fork_choice/tests/tests.rs b/eth2/fork_choice/tests/tests.rs
index 3ce63eeb7..fb530ac43 100644
--- a/eth2/fork_choice/tests/tests.rs
+++ b/eth2/fork_choice/tests/tests.rs
@@ -1,3 +1,4 @@
+#![cfg(not(debug_assertions))]
 // Tests the available fork-choice algorithms
 
 extern crate beacon_chain;
diff --git a/eth2/state_processing/tests/tests.rs b/eth2/state_processing/tests/tests.rs
index 1359508dc..25613f9a7 100644
--- a/eth2/state_processing/tests/tests.rs
+++ b/eth2/state_processing/tests/tests.rs
@@ -1,6 +1,7 @@
+#![cfg(not(debug_assertions))]
+
 use serde_derive::Deserialize;
 use serde_yaml;
-#[cfg(not(debug_assertions))]
 use state_processing::{
     per_block_processing, per_block_processing_without_verifying_block_signature,
     per_slot_processing,
@@ -63,7 +64,6 @@ fn test_read_yaml() {
 }
 
 #[test]
-#[cfg(not(debug_assertions))]
 fn run_state_transition_tests_small() {
     // Test sanity-check_small-config_32-vals.yaml
     let mut file = {

From 0b5c10212d7f08d42f5d7bfe73a640fc2cb431ed Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Mon, 15 Apr 2019 11:14:30 +1000
Subject: [PATCH 042/137] Move tree_hash from ssz into own crate

---
 Cargo.toml                                    |   1 +
 eth2/utils/ssz/src/cached_tree_hash.rs        | 439 ------------------
 eth2/utils/ssz/src/lib.rs                     |   1 -
 eth2/utils/tree_hash/Cargo.toml               |  11 +
 eth2/utils/tree_hash/src/btree_overlay.rs     |   0
 eth2/utils/tree_hash/src/cached_tree_hash.rs  | 193 ++++++++
 .../src}/impls.rs                             |   2 +-
 eth2/utils/tree_hash/src/lib.rs               | 249 ++++++++++
 .../src}/resize.rs                            |   0
 .../tests}/tests.rs                           |   8 +-
 10 files changed, 461 insertions(+), 443 deletions(-)
 delete mode 100644 eth2/utils/ssz/src/cached_tree_hash.rs
 create mode 100644 eth2/utils/tree_hash/Cargo.toml
 create mode 100644 eth2/utils/tree_hash/src/btree_overlay.rs
 create mode 100644 eth2/utils/tree_hash/src/cached_tree_hash.rs
 rename eth2/utils/{ssz/src/cached_tree_hash => tree_hash/src}/impls.rs (99%)
 create mode 100644 eth2/utils/tree_hash/src/lib.rs
 rename eth2/utils/{ssz/src/cached_tree_hash => tree_hash/src}/resize.rs (100%)
 rename eth2/utils/{ssz/src/cached_tree_hash => tree_hash/tests}/tests.rs (99%)

diff --git a/Cargo.toml b/Cargo.toml
index 5c9593f5a..2574d328f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -18,6 +18,7 @@ members = [
 	"eth2/utils/ssz",
 	"eth2/utils/ssz_derive",
 	"eth2/utils/swap_or_not_shuffle",
+	"eth2/utils/tree_hash",
 	"eth2/utils/fisher_yates_shuffle",
     "eth2/utils/test_random_derive",
 	"beacon_node",
diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs
deleted file mode 100644
index f7d18c57c..000000000
--- a/eth2/utils/ssz/src/cached_tree_hash.rs
+++ /dev/null
@@ -1,439 +0,0 @@
-use hashing::hash;
-use int_to_bytes::int_to_bytes32;
-use std::fmt::Debug;
-use std::iter::Iterator;
-use std::ops::Range;
-
-mod impls;
-mod resize;
-mod tests;
-
-const BYTES_PER_CHUNK: usize = 32;
-const HASHSIZE: usize = 32;
-const MERKLE_HASH_CHUNCK: usize = 2 * BYTES_PER_CHUNK;
-
-#[derive(Debug, PartialEq, Clone)]
-pub enum Error {
-    ShouldNotProduceBTreeOverlay,
-    NoFirstNode,
-    NoBytesForRoot,
-    UnableToObtainSlices,
-    UnableToGrowMerkleTree,
-    UnableToShrinkMerkleTree,
-    BytesAreNotEvenChunks(usize),
-    NoModifiedFieldForChunk(usize),
-    NoBytesForChunk(usize),
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub enum ItemType {
-    Basic,
-    List,
-    Composite,
-}
-
-// TODO: remove debug requirement.
-pub trait CachedTreeHash<Item>: Debug {
-    fn item_type() -> ItemType;
-
-    fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error>;
-
-    /// Return the number of bytes when this element is encoded as raw SSZ _without_ length
-    /// prefixes.
-    fn num_bytes(&self) -> usize;
-
-    fn offsets(&self) -> Result<Vec<usize>, Error>;
-
-    fn num_child_nodes(&self) -> usize;
-
-    fn packed_encoding(&self) -> Vec<u8>;
-
-    fn packing_factor() -> usize;
-
-    fn cached_hash_tree_root(
-        &self,
-        other: &Item,
-        cache: &mut TreeHashCache,
-        chunk: usize,
-    ) -> Result<usize, Error>;
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct TreeHashCache {
-    cache: Vec<u8>,
-    chunk_modified: Vec<bool>,
-}
-
-impl Into<Vec<u8>> for TreeHashCache {
-    fn into(self) -> Vec<u8> {
-        self.cache
-    }
-}
-
-impl TreeHashCache {
-    pub fn new<T>(item: &T) -> Result<Self, Error>
-    where
-        T: CachedTreeHash<T>,
-    {
-        item.build_tree_hash_cache()
-    }
-
-    pub fn from_elems(cache: Vec<u8>, chunk_modified: Vec<bool>) -> Self {
-        Self {
-            cache,
-            chunk_modified,
-        }
-    }
-
-    pub fn from_leaves_and_subtrees<T>(
-        item: &T,
-        leaves_and_subtrees: Vec<Self>,
-    ) -> Result<Self, Error>
-    where
-        T: CachedTreeHash<T>,
-    {
-        let offset_handler = BTreeOverlay::new(item, 0)?;
-
-        // Note how many leaves were provided. If is not a power-of-two, we'll need to pad it out
-        // later.
-        let num_provided_leaf_nodes = leaves_and_subtrees.len();
-
-        // Allocate enough bytes to store the internal nodes and the leaves and subtrees, then fill
-        // all the to-be-built internal nodes with zeros and append the leaves and subtrees.
-        let internal_node_bytes = offset_handler.num_internal_nodes * BYTES_PER_CHUNK;
-        let leaves_and_subtrees_bytes = leaves_and_subtrees
-            .iter()
-            .fold(0, |acc, t| acc + t.bytes_len());
-        let mut cache = Vec::with_capacity(leaves_and_subtrees_bytes + internal_node_bytes);
-        cache.resize(internal_node_bytes, 0);
-
-        // Allocate enough bytes to store all the leaves.
-        let mut leaves = Vec::with_capacity(offset_handler.num_leaf_nodes * HASHSIZE);
-
-        // Iterate through all of the leaves/subtrees, adding their root as a leaf node and then
-        // concatenating their merkle trees.
-        for t in leaves_and_subtrees {
-            leaves.append(&mut t.root()?);
-            cache.append(&mut t.into_merkle_tree());
-        }
-
-        // Pad the leaves to an even power-of-two, using zeros.
-        pad_for_leaf_count(num_provided_leaf_nodes, &mut cache);
-
-        // Merkleize the leaves, then split the leaf nodes off them. Then, replace all-zeros
-        // internal nodes created earlier with the internal nodes generated by `merkleize`.
-        let mut merkleized = merkleize(leaves);
-        merkleized.split_off(internal_node_bytes);
-        cache.splice(0..internal_node_bytes, merkleized);
-
-        Ok(Self {
-            chunk_modified: vec![false; cache.len() / BYTES_PER_CHUNK],
-            cache,
-        })
-    }
-
-    pub fn from_bytes(bytes: Vec<u8>, initial_modified_state: bool) -> Result<Self, Error> {
-        if bytes.len() % BYTES_PER_CHUNK > 0 {
-            return Err(Error::BytesAreNotEvenChunks(bytes.len()));
-        }
-
-        Ok(Self {
-            chunk_modified: vec![initial_modified_state; bytes.len() / BYTES_PER_CHUNK],
-            cache: bytes,
-        })
-    }
-
-    pub fn bytes_len(&self) -> usize {
-        self.cache.len()
-    }
-
-    pub fn root(&self) -> Result<Vec<u8>, Error> {
-        self.cache
-            .get(0..HASHSIZE)
-            .ok_or_else(|| Error::NoBytesForRoot)
-            .and_then(|slice| Ok(slice.to_vec()))
-    }
-
-    pub fn splice(&mut self, chunk_range: Range<usize>, replace_with: Self) {
-        let (bytes, bools) = replace_with.into_components();
-
-        // Update the `chunk_modified` vec, marking all spliced-in nodes as changed.
-        self.chunk_modified.splice(chunk_range.clone(), bools);
-        self.cache
-            .splice(node_range_to_byte_range(&chunk_range), bytes);
-    }
-
-    pub fn maybe_update_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> {
-        let start = chunk * BYTES_PER_CHUNK;
-        let end = start + BYTES_PER_CHUNK;
-
-        if !self.chunk_equals(chunk, to)? {
-            self.cache
-                .get_mut(start..end)
-                .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))?
-                .copy_from_slice(to);
-            self.chunk_modified[chunk] = true;
-        }
-
-        Ok(())
-    }
-
-    pub fn slices(&self, chunk_range: Range<usize>) -> Option<(&[u8], &[bool])> {
-        Some((
-            self.cache.get(node_range_to_byte_range(&chunk_range))?,
-            self.chunk_modified.get(chunk_range)?,
-        ))
-    }
-
-    pub fn modify_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> {
-        let start = chunk * BYTES_PER_CHUNK;
-        let end = start + BYTES_PER_CHUNK;
-
-        self.cache
-            .get_mut(start..end)
-            .ok_or_else(|| Error::NoBytesForChunk(chunk))?
-            .copy_from_slice(to);
-
-        self.chunk_modified[chunk] = true;
-
-        Ok(())
-    }
-
-    pub fn get_chunk(&self, chunk: usize) -> Result<&[u8], Error> {
-        let start = chunk * BYTES_PER_CHUNK;
-        let end = start + BYTES_PER_CHUNK;
-
-        Ok(self
-            .cache
-            .get(start..end)
-            .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))?)
-    }
-
-    pub fn chunk_equals(&mut self, chunk: usize, other: &[u8]) -> Result<bool, Error> {
-        Ok(self.get_chunk(chunk)? == other)
-    }
-
-    pub fn changed(&self, chunk: usize) -> Result<bool, Error> {
-        self.chunk_modified
-            .get(chunk)
-            .cloned()
-            .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))
-    }
-
-    pub fn either_modified(&self, children: (&usize, &usize)) -> Result<bool, Error> {
-        Ok(self.changed(*children.0)? | self.changed(*children.1)?)
-    }
-
-    pub fn hash_children(&self, children: (&usize, &usize)) -> Result<Vec<u8>, Error> {
-        let mut child_bytes = Vec::with_capacity(BYTES_PER_CHUNK * 2);
-        child_bytes.append(&mut self.get_chunk(*children.0)?.to_vec());
-        child_bytes.append(&mut self.get_chunk(*children.1)?.to_vec());
-
-        Ok(hash(&child_bytes))
-    }
-
-    pub fn mix_in_length(&self, chunk: usize, length: usize) -> Result<Vec<u8>, Error> {
-        let mut bytes = Vec::with_capacity(2 * BYTES_PER_CHUNK);
-
-        bytes.append(&mut self.get_chunk(chunk)?.to_vec());
-        bytes.append(&mut int_to_bytes32(length as u64));
-
-        Ok(hash(&bytes))
-    }
-
-    pub fn into_merkle_tree(self) -> Vec<u8> {
-        self.cache
-    }
-
-    pub fn into_components(self) -> (Vec<u8>, Vec<bool>) {
-        (self.cache, self.chunk_modified)
-    }
-}
-
-fn children(parent: usize) -> (usize, usize) {
-    ((2 * parent + 1), (2 * parent + 2))
-}
-
-fn num_nodes(num_leaves: usize) -> usize {
-    2 * num_leaves - 1
-}
-
-fn node_range_to_byte_range(node_range: &Range<usize>) -> Range<usize> {
-    node_range.start * HASHSIZE..node_range.end * HASHSIZE
-}
-
-#[derive(Debug)]
-pub struct BTreeOverlay {
-    num_internal_nodes: usize,
-    pub num_leaf_nodes: usize,
-    first_node: usize,
-    next_node: usize,
-    offsets: Vec<usize>,
-}
-
-impl BTreeOverlay {
-    pub fn new<T>(item: &T, initial_offset: usize) -> Result<Self, Error>
-    where
-        T: CachedTreeHash<T>,
-    {
-        Self::from_lengths(initial_offset, item.offsets()?)
-    }
-
-    fn from_lengths(offset: usize, mut lengths: Vec<usize>) -> Result<Self, Error> {
-        // Extend it to the next power-of-two, if it is not already.
-        let num_leaf_nodes = if lengths.len().is_power_of_two() {
-            lengths.len()
-        } else {
-            let num_leaf_nodes = lengths.len().next_power_of_two();
-            lengths.resize(num_leaf_nodes, 1);
-            num_leaf_nodes
-        };
-
-        let num_nodes = num_nodes(num_leaf_nodes);
-        let num_internal_nodes = num_nodes - num_leaf_nodes;
-
-        let mut offsets = Vec::with_capacity(num_nodes);
-        offsets.append(&mut (offset..offset + num_internal_nodes).collect());
-
-        let mut next_node = num_internal_nodes + offset;
-        for i in 0..num_leaf_nodes {
-            offsets.push(next_node);
-            next_node += lengths[i];
-        }
-
-        Ok(Self {
-            num_internal_nodes,
-            num_leaf_nodes,
-            offsets,
-            first_node: offset,
-            next_node,
-        })
-    }
-
-    pub fn root(&self) -> usize {
-        self.first_node
-    }
-
-    pub fn height(&self) -> usize {
-        self.num_leaf_nodes.trailing_zeros() as usize
-    }
-
-    pub fn chunk_range(&self) -> Range<usize> {
-        self.first_node..self.next_node
-    }
-
-    pub fn total_chunks(&self) -> usize {
-        self.next_node - self.first_node
-    }
-
-    pub fn total_nodes(&self) -> usize {
-        self.num_internal_nodes + self.num_leaf_nodes
-    }
-
-    pub fn first_leaf_node(&self) -> Result<usize, Error> {
-        self.offsets
-            .get(self.num_internal_nodes)
-            .cloned()
-            .ok_or_else(|| Error::NoFirstNode)
-    }
-
-    pub fn next_node(&self) -> usize {
-        self.next_node
-    }
-
-    /// Returns an iterator visiting each internal node, providing the left and right child chunks
-    /// for the node.
-    pub fn iter_internal_nodes<'a>(
-        &'a self,
-    ) -> impl DoubleEndedIterator<Item = (&'a usize, (&'a usize, &'a usize))> {
-        let internal_nodes = &self.offsets[0..self.num_internal_nodes];
-
-        internal_nodes.iter().enumerate().map(move |(i, parent)| {
-            let children = children(i);
-            (
-                parent,
-                (&self.offsets[children.0], &self.offsets[children.1]),
-            )
-        })
-    }
-
-    /// Returns an iterator visiting each leaf node, providing the chunk for that node.
-    pub fn iter_leaf_nodes<'a>(&'a self) -> impl DoubleEndedIterator<Item = &'a usize> {
-        let leaf_nodes = &self.offsets[self.num_internal_nodes..];
-
-        leaf_nodes.iter()
-    }
-}
-
-/// Split `values` into a power-of-two, identical-length chunks (padding with `0`) and merkleize
-/// them, returning the entire merkle tree.
-///
-/// The root hash is `merkleize(values)[0..BYTES_PER_CHUNK]`.
-pub fn merkleize(values: Vec<u8>) -> Vec<u8> {
-    let values = sanitise_bytes(values);
-
-    let leaves = values.len() / HASHSIZE;
-
-    if leaves == 0 {
-        panic!("No full leaves");
-    }
-
-    if !leaves.is_power_of_two() {
-        panic!("leaves is not power of two");
-    }
-
-    let mut o: Vec<u8> = vec![0; (num_nodes(leaves) - leaves) * HASHSIZE];
-    o.append(&mut values.to_vec());
-
-    let mut i = o.len();
-    let mut j = o.len() - values.len();
-
-    while i >= MERKLE_HASH_CHUNCK {
-        i -= MERKLE_HASH_CHUNCK;
-        let hash = hash(&o[i..i + MERKLE_HASH_CHUNCK]);
-
-        j -= HASHSIZE;
-        o[j..j + HASHSIZE].copy_from_slice(&hash);
-    }
-
-    o
-}
-
-pub fn sanitise_bytes(mut bytes: Vec<u8>) -> Vec<u8> {
-    let present_leaves = num_unsanitized_leaves(bytes.len());
-    let required_leaves = present_leaves.next_power_of_two();
-
-    if (present_leaves != required_leaves) | last_leaf_needs_padding(bytes.len()) {
-        bytes.resize(num_bytes(required_leaves), 0);
-    }
-
-    bytes
-}
-
-fn pad_for_leaf_count(num_leaves: usize, bytes: &mut Vec<u8>) {
-    let required_leaves = num_leaves.next_power_of_two();
-
-    bytes.resize(
-        bytes.len() + (required_leaves - num_leaves) * BYTES_PER_CHUNK,
-        0,
-    );
-}
-
-fn last_leaf_needs_padding(num_bytes: usize) -> bool {
-    num_bytes % HASHSIZE != 0
-}
-
-/// Rounds up
-fn num_unsanitized_leaves(num_bytes: usize) -> usize {
-    (num_bytes + HASHSIZE - 1) / HASHSIZE
-}
-
-/// Rounds up
-fn num_sanitized_leaves(num_bytes: usize) -> usize {
-    let leaves = (num_bytes + HASHSIZE - 1) / HASHSIZE;
-    leaves.next_power_of_two()
-}
-
-fn num_bytes(num_leaves: usize) -> usize {
-    num_leaves * HASHSIZE
-}
diff --git a/eth2/utils/ssz/src/lib.rs b/eth2/utils/ssz/src/lib.rs
index f86749c66..cb3f63c48 100644
--- a/eth2/utils/ssz/src/lib.rs
+++ b/eth2/utils/ssz/src/lib.rs
@@ -10,7 +10,6 @@
 extern crate bytes;
 extern crate ethereum_types;
 
-mod cached_tree_hash;
 pub mod decode;
 pub mod encode;
 mod signed_root;
diff --git a/eth2/utils/tree_hash/Cargo.toml b/eth2/utils/tree_hash/Cargo.toml
new file mode 100644
index 000000000..243a49446
--- /dev/null
+++ b/eth2/utils/tree_hash/Cargo.toml
@@ -0,0 +1,11 @@
+[package]
+name = "tree_hash"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+[dependencies]
+ethereum-types = "0.5"
+hashing = { path = "../hashing" }
+int_to_bytes = { path = "../int_to_bytes" }
+ssz = { path = "../ssz" }
diff --git a/eth2/utils/tree_hash/src/btree_overlay.rs b/eth2/utils/tree_hash/src/btree_overlay.rs
new file mode 100644
index 000000000..e69de29bb
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash.rs b/eth2/utils/tree_hash/src/cached_tree_hash.rs
new file mode 100644
index 000000000..556ba2d21
--- /dev/null
+++ b/eth2/utils/tree_hash/src/cached_tree_hash.rs
@@ -0,0 +1,193 @@
+use super::*;
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct TreeHashCache {
+    cache: Vec<u8>,
+    chunk_modified: Vec<bool>,
+}
+
+impl Into<Vec<u8>> for TreeHashCache {
+    fn into(self) -> Vec<u8> {
+        self.cache
+    }
+}
+
+impl TreeHashCache {
+    pub fn new<T>(item: &T) -> Result<Self, Error>
+    where
+        T: CachedTreeHash<T>,
+    {
+        item.build_tree_hash_cache()
+    }
+
+    pub fn from_elems(cache: Vec<u8>, chunk_modified: Vec<bool>) -> Self {
+        Self {
+            cache,
+            chunk_modified,
+        }
+    }
+
+    pub fn from_leaves_and_subtrees<T>(
+        item: &T,
+        leaves_and_subtrees: Vec<Self>,
+    ) -> Result<Self, Error>
+    where
+        T: CachedTreeHash<T>,
+    {
+        let offset_handler = BTreeOverlay::new(item, 0)?;
+
+        // Note how many leaves were provided. If is not a power-of-two, we'll need to pad it out
+        // later.
+        let num_provided_leaf_nodes = leaves_and_subtrees.len();
+
+        // Allocate enough bytes to store the internal nodes and the leaves and subtrees, then fill
+        // all the to-be-built internal nodes with zeros and append the leaves and subtrees.
+        let internal_node_bytes = offset_handler.num_internal_nodes * BYTES_PER_CHUNK;
+        let leaves_and_subtrees_bytes = leaves_and_subtrees
+            .iter()
+            .fold(0, |acc, t| acc + t.bytes_len());
+        let mut cache = Vec::with_capacity(leaves_and_subtrees_bytes + internal_node_bytes);
+        cache.resize(internal_node_bytes, 0);
+
+        // Allocate enough bytes to store all the leaves.
+        let mut leaves = Vec::with_capacity(offset_handler.num_leaf_nodes * HASHSIZE);
+
+        // Iterate through all of the leaves/subtrees, adding their root as a leaf node and then
+        // concatenating their merkle trees.
+        for t in leaves_and_subtrees {
+            leaves.append(&mut t.root()?);
+            cache.append(&mut t.into_merkle_tree());
+        }
+
+        // Pad the leaves to an even power-of-two, using zeros.
+        pad_for_leaf_count(num_provided_leaf_nodes, &mut cache);
+
+        // Merkleize the leaves, then split the leaf nodes off them. Then, replace all-zeros
+        // internal nodes created earlier with the internal nodes generated by `merkleize`.
+        let mut merkleized = merkleize(leaves);
+        merkleized.split_off(internal_node_bytes);
+        cache.splice(0..internal_node_bytes, merkleized);
+
+        Ok(Self {
+            chunk_modified: vec![false; cache.len() / BYTES_PER_CHUNK],
+            cache,
+        })
+    }
+
+    pub fn from_bytes(bytes: Vec<u8>, initial_modified_state: bool) -> Result<Self, Error> {
+        if bytes.len() % BYTES_PER_CHUNK > 0 {
+            return Err(Error::BytesAreNotEvenChunks(bytes.len()));
+        }
+
+        Ok(Self {
+            chunk_modified: vec![initial_modified_state; bytes.len() / BYTES_PER_CHUNK],
+            cache: bytes,
+        })
+    }
+
+    pub fn bytes_len(&self) -> usize {
+        self.cache.len()
+    }
+
+    pub fn root(&self) -> Result<Vec<u8>, Error> {
+        self.cache
+            .get(0..HASHSIZE)
+            .ok_or_else(|| Error::NoBytesForRoot)
+            .and_then(|slice| Ok(slice.to_vec()))
+    }
+
+    pub fn splice(&mut self, chunk_range: Range<usize>, replace_with: Self) {
+        let (bytes, bools) = replace_with.into_components();
+
+        // Update the `chunk_modified` vec, marking all spliced-in nodes as changed.
+        self.chunk_modified.splice(chunk_range.clone(), bools);
+        self.cache
+            .splice(node_range_to_byte_range(&chunk_range), bytes);
+    }
+
+    pub fn maybe_update_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> {
+        let start = chunk * BYTES_PER_CHUNK;
+        let end = start + BYTES_PER_CHUNK;
+
+        if !self.chunk_equals(chunk, to)? {
+            self.cache
+                .get_mut(start..end)
+                .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))?
+                .copy_from_slice(to);
+            self.chunk_modified[chunk] = true;
+        }
+
+        Ok(())
+    }
+
+    pub fn slices(&self, chunk_range: Range<usize>) -> Option<(&[u8], &[bool])> {
+        Some((
+            self.cache.get(node_range_to_byte_range(&chunk_range))?,
+            self.chunk_modified.get(chunk_range)?,
+        ))
+    }
+
+    pub fn modify_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> {
+        let start = chunk * BYTES_PER_CHUNK;
+        let end = start + BYTES_PER_CHUNK;
+
+        self.cache
+            .get_mut(start..end)
+            .ok_or_else(|| Error::NoBytesForChunk(chunk))?
+            .copy_from_slice(to);
+
+        self.chunk_modified[chunk] = true;
+
+        Ok(())
+    }
+
+    pub fn get_chunk(&self, chunk: usize) -> Result<&[u8], Error> {
+        let start = chunk * BYTES_PER_CHUNK;
+        let end = start + BYTES_PER_CHUNK;
+
+        Ok(self
+            .cache
+            .get(start..end)
+            .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))?)
+    }
+
+    pub fn chunk_equals(&mut self, chunk: usize, other: &[u8]) -> Result<bool, Error> {
+        Ok(self.get_chunk(chunk)? == other)
+    }
+
+    pub fn changed(&self, chunk: usize) -> Result<bool, Error> {
+        self.chunk_modified
+            .get(chunk)
+            .cloned()
+            .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))
+    }
+
+    pub fn either_modified(&self, children: (&usize, &usize)) -> Result<bool, Error> {
+        Ok(self.changed(*children.0)? | self.changed(*children.1)?)
+    }
+
+    pub fn hash_children(&self, children: (&usize, &usize)) -> Result<Vec<u8>, Error> {
+        let mut child_bytes = Vec::with_capacity(BYTES_PER_CHUNK * 2);
+        child_bytes.append(&mut self.get_chunk(*children.0)?.to_vec());
+        child_bytes.append(&mut self.get_chunk(*children.1)?.to_vec());
+
+        Ok(hash(&child_bytes))
+    }
+
+    pub fn mix_in_length(&self, chunk: usize, length: usize) -> Result<Vec<u8>, Error> {
+        let mut bytes = Vec::with_capacity(2 * BYTES_PER_CHUNK);
+
+        bytes.append(&mut self.get_chunk(chunk)?.to_vec());
+        bytes.append(&mut int_to_bytes32(length as u64));
+
+        Ok(hash(&bytes))
+    }
+
+    pub fn into_merkle_tree(self) -> Vec<u8> {
+        self.cache
+    }
+
+    pub fn into_components(self) -> (Vec<u8>, Vec<bool>) {
+        (self.cache, self.chunk_modified)
+    }
+}
diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/tree_hash/src/impls.rs
similarity index 99%
rename from eth2/utils/ssz/src/cached_tree_hash/impls.rs
rename to eth2/utils/tree_hash/src/impls.rs
index 26905c667..d5297c38e 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/tree_hash/src/impls.rs
@@ -1,6 +1,6 @@
 use super::resize::{grow_merkle_cache, shrink_merkle_cache};
 use super::*;
-use crate::ssz_encode;
+use ssz::ssz_encode;
 
 impl CachedTreeHash<u64> for u64 {
     fn item_type() -> ItemType {
diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs
new file mode 100644
index 000000000..1b085770d
--- /dev/null
+++ b/eth2/utils/tree_hash/src/lib.rs
@@ -0,0 +1,249 @@
+use hashing::hash;
+use int_to_bytes::int_to_bytes32;
+use std::fmt::Debug;
+use std::iter::Iterator;
+use std::ops::Range;
+
+mod cached_tree_hash;
+mod impls;
+mod resize;
+
+pub use cached_tree_hash::TreeHashCache;
+
+pub const BYTES_PER_CHUNK: usize = 32;
+pub const HASHSIZE: usize = 32;
+pub const MERKLE_HASH_CHUNCK: usize = 2 * BYTES_PER_CHUNK;
+
+#[derive(Debug, PartialEq, Clone)]
+pub enum Error {
+    ShouldNotProduceBTreeOverlay,
+    NoFirstNode,
+    NoBytesForRoot,
+    UnableToObtainSlices,
+    UnableToGrowMerkleTree,
+    UnableToShrinkMerkleTree,
+    BytesAreNotEvenChunks(usize),
+    NoModifiedFieldForChunk(usize),
+    NoBytesForChunk(usize),
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub enum ItemType {
+    Basic,
+    List,
+    Composite,
+}
+
+// TODO: remove debug requirement.
+pub trait CachedTreeHash<Item>: Debug {
+    fn item_type() -> ItemType;
+
+    fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error>;
+
+    /// Return the number of bytes when this element is encoded as raw SSZ _without_ length
+    /// prefixes.
+    fn num_bytes(&self) -> usize;
+
+    fn offsets(&self) -> Result<Vec<usize>, Error>;
+
+    fn num_child_nodes(&self) -> usize;
+
+    fn packed_encoding(&self) -> Vec<u8>;
+
+    fn packing_factor() -> usize;
+
+    fn cached_hash_tree_root(
+        &self,
+        other: &Item,
+        cache: &mut TreeHashCache,
+        chunk: usize,
+    ) -> Result<usize, Error>;
+}
+
+fn children(parent: usize) -> (usize, usize) {
+    ((2 * parent + 1), (2 * parent + 2))
+}
+
+fn num_nodes(num_leaves: usize) -> usize {
+    2 * num_leaves - 1
+}
+
+fn node_range_to_byte_range(node_range: &Range<usize>) -> Range<usize> {
+    node_range.start * HASHSIZE..node_range.end * HASHSIZE
+}
+
+#[derive(Debug)]
+pub struct BTreeOverlay {
+    num_internal_nodes: usize,
+    pub num_leaf_nodes: usize,
+    first_node: usize,
+    next_node: usize,
+    offsets: Vec<usize>,
+}
+
+impl BTreeOverlay {
+    pub fn new<T>(item: &T, initial_offset: usize) -> Result<Self, Error>
+    where
+        T: CachedTreeHash<T>,
+    {
+        Self::from_lengths(initial_offset, item.offsets()?)
+    }
+
+    fn from_lengths(offset: usize, mut lengths: Vec<usize>) -> Result<Self, Error> {
+        // Extend it to the next power-of-two, if it is not already.
+        let num_leaf_nodes = if lengths.len().is_power_of_two() {
+            lengths.len()
+        } else {
+            let num_leaf_nodes = lengths.len().next_power_of_two();
+            lengths.resize(num_leaf_nodes, 1);
+            num_leaf_nodes
+        };
+
+        let num_nodes = num_nodes(num_leaf_nodes);
+        let num_internal_nodes = num_nodes - num_leaf_nodes;
+
+        let mut offsets = Vec::with_capacity(num_nodes);
+        offsets.append(&mut (offset..offset + num_internal_nodes).collect());
+
+        let mut next_node = num_internal_nodes + offset;
+        for i in 0..num_leaf_nodes {
+            offsets.push(next_node);
+            next_node += lengths[i];
+        }
+
+        Ok(Self {
+            num_internal_nodes,
+            num_leaf_nodes,
+            offsets,
+            first_node: offset,
+            next_node,
+        })
+    }
+
+    pub fn root(&self) -> usize {
+        self.first_node
+    }
+
+    pub fn height(&self) -> usize {
+        self.num_leaf_nodes.trailing_zeros() as usize
+    }
+
+    pub fn chunk_range(&self) -> Range<usize> {
+        self.first_node..self.next_node
+    }
+
+    pub fn total_chunks(&self) -> usize {
+        self.next_node - self.first_node
+    }
+
+    pub fn total_nodes(&self) -> usize {
+        self.num_internal_nodes + self.num_leaf_nodes
+    }
+
+    pub fn first_leaf_node(&self) -> Result<usize, Error> {
+        self.offsets
+            .get(self.num_internal_nodes)
+            .cloned()
+            .ok_or_else(|| Error::NoFirstNode)
+    }
+
+    pub fn next_node(&self) -> usize {
+        self.next_node
+    }
+
+    /// Returns an iterator visiting each internal node, providing the left and right child chunks
+    /// for the node.
+    pub fn iter_internal_nodes<'a>(
+        &'a self,
+    ) -> impl DoubleEndedIterator<Item = (&'a usize, (&'a usize, &'a usize))> {
+        let internal_nodes = &self.offsets[0..self.num_internal_nodes];
+
+        internal_nodes.iter().enumerate().map(move |(i, parent)| {
+            let children = children(i);
+            (
+                parent,
+                (&self.offsets[children.0], &self.offsets[children.1]),
+            )
+        })
+    }
+
+    /// Returns an iterator visiting each leaf node, providing the chunk for that node.
+    pub fn iter_leaf_nodes<'a>(&'a self) -> impl DoubleEndedIterator<Item = &'a usize> {
+        let leaf_nodes = &self.offsets[self.num_internal_nodes..];
+
+        leaf_nodes.iter()
+    }
+}
+
+/// Split `values` into a power-of-two, identical-length chunks (padding with `0`) and merkleize
+/// them, returning the entire merkle tree.
+///
+/// The root hash is `merkleize(values)[0..BYTES_PER_CHUNK]`.
+pub fn merkleize(values: Vec<u8>) -> Vec<u8> {
+    let values = sanitise_bytes(values);
+
+    let leaves = values.len() / HASHSIZE;
+
+    if leaves == 0 {
+        panic!("No full leaves");
+    }
+
+    if !leaves.is_power_of_two() {
+        panic!("leaves is not power of two");
+    }
+
+    let mut o: Vec<u8> = vec![0; (num_nodes(leaves) - leaves) * HASHSIZE];
+    o.append(&mut values.to_vec());
+
+    let mut i = o.len();
+    let mut j = o.len() - values.len();
+
+    while i >= MERKLE_HASH_CHUNCK {
+        i -= MERKLE_HASH_CHUNCK;
+        let hash = hash(&o[i..i + MERKLE_HASH_CHUNCK]);
+
+        j -= HASHSIZE;
+        o[j..j + HASHSIZE].copy_from_slice(&hash);
+    }
+
+    o
+}
+
+pub fn sanitise_bytes(mut bytes: Vec<u8>) -> Vec<u8> {
+    let present_leaves = num_unsanitized_leaves(bytes.len());
+    let required_leaves = present_leaves.next_power_of_two();
+
+    if (present_leaves != required_leaves) | last_leaf_needs_padding(bytes.len()) {
+        bytes.resize(num_bytes(required_leaves), 0);
+    }
+
+    bytes
+}
+
+fn pad_for_leaf_count(num_leaves: usize, bytes: &mut Vec<u8>) {
+    let required_leaves = num_leaves.next_power_of_two();
+
+    bytes.resize(
+        bytes.len() + (required_leaves - num_leaves) * BYTES_PER_CHUNK,
+        0,
+    );
+}
+
+fn last_leaf_needs_padding(num_bytes: usize) -> bool {
+    num_bytes % HASHSIZE != 0
+}
+
+/// Rounds up
+fn num_unsanitized_leaves(num_bytes: usize) -> usize {
+    (num_bytes + HASHSIZE - 1) / HASHSIZE
+}
+
+/// Rounds up
+fn num_sanitized_leaves(num_bytes: usize) -> usize {
+    let leaves = (num_bytes + HASHSIZE - 1) / HASHSIZE;
+    leaves.next_power_of_two()
+}
+
+fn num_bytes(num_leaves: usize) -> usize {
+    num_leaves * HASHSIZE
+}
diff --git a/eth2/utils/ssz/src/cached_tree_hash/resize.rs b/eth2/utils/tree_hash/src/resize.rs
similarity index 100%
rename from eth2/utils/ssz/src/cached_tree_hash/resize.rs
rename to eth2/utils/tree_hash/src/resize.rs
diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
similarity index 99%
rename from eth2/utils/ssz/src/cached_tree_hash/tests.rs
rename to eth2/utils/tree_hash/tests/tests.rs
index e6e2b1754..972eb1e00 100644
--- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs
+++ b/eth2/utils/tree_hash/tests/tests.rs
@@ -1,6 +1,10 @@
-#![cfg(test)]
-use super::*;
+use hashing::hash;
 use int_to_bytes::{int_to_bytes32, int_to_bytes8};
+use tree_hash::*;
+
+fn num_nodes(num_leaves: usize) -> usize {
+    2 * num_leaves - 1
+}
 
 #[derive(Clone, Debug)]
 pub struct Inner {

From c87a0fc5882c3b718827bd4865688558ab8ae611 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Mon, 15 Apr 2019 11:37:29 +1000
Subject: [PATCH 043/137] Tidy CachedTreeHash trait

---
 eth2/utils/tree_hash/src/btree_overlay.rs | 100 ++++++++++++++++++++
 eth2/utils/tree_hash/src/impls.rs         |  25 ++---
 eth2/utils/tree_hash/src/lib.rs           | 108 +---------------------
 eth2/utils/tree_hash/tests/tests.rs       |  30 +++---
 4 files changed, 131 insertions(+), 132 deletions(-)

diff --git a/eth2/utils/tree_hash/src/btree_overlay.rs b/eth2/utils/tree_hash/src/btree_overlay.rs
index e69de29bb..7d5602c0b 100644
--- a/eth2/utils/tree_hash/src/btree_overlay.rs
+++ b/eth2/utils/tree_hash/src/btree_overlay.rs
@@ -0,0 +1,100 @@
+use super::*;
+
+#[derive(Debug)]
+pub struct BTreeOverlay {
+    pub num_internal_nodes: usize,
+    pub num_leaf_nodes: usize,
+    pub first_node: usize,
+    pub next_node: usize,
+    offsets: Vec<usize>,
+}
+
+impl BTreeOverlay {
+    pub fn new<T>(item: &T, initial_offset: usize) -> Result<Self, Error>
+    where
+        T: CachedTreeHash<T>,
+    {
+        item.btree_overlay(initial_offset)
+    }
+
+    pub fn from_lengths(offset: usize, mut lengths: Vec<usize>) -> Result<Self, Error> {
+        // Extend it to the next power-of-two, if it is not already.
+        let num_leaf_nodes = if lengths.len().is_power_of_two() {
+            lengths.len()
+        } else {
+            let num_leaf_nodes = lengths.len().next_power_of_two();
+            lengths.resize(num_leaf_nodes, 1);
+            num_leaf_nodes
+        };
+
+        let num_nodes = num_nodes(num_leaf_nodes);
+        let num_internal_nodes = num_nodes - num_leaf_nodes;
+
+        let mut offsets = Vec::with_capacity(num_nodes);
+        offsets.append(&mut (offset..offset + num_internal_nodes).collect());
+
+        let mut next_node = num_internal_nodes + offset;
+        for i in 0..num_leaf_nodes {
+            offsets.push(next_node);
+            next_node += lengths[i];
+        }
+
+        Ok(Self {
+            num_internal_nodes,
+            num_leaf_nodes,
+            offsets,
+            first_node: offset,
+            next_node,
+        })
+    }
+
+    pub fn root(&self) -> usize {
+        self.first_node
+    }
+
+    pub fn height(&self) -> usize {
+        self.num_leaf_nodes.trailing_zeros() as usize
+    }
+
+    pub fn chunk_range(&self) -> Range<usize> {
+        self.first_node..self.next_node
+    }
+
+    pub fn total_chunks(&self) -> usize {
+        self.next_node - self.first_node
+    }
+
+    pub fn total_nodes(&self) -> usize {
+        self.num_internal_nodes + self.num_leaf_nodes
+    }
+
+    pub fn first_leaf_node(&self) -> Result<usize, Error> {
+        self.offsets
+            .get(self.num_internal_nodes)
+            .cloned()
+            .ok_or_else(|| Error::NoFirstNode)
+    }
+
+    /// Returns an iterator visiting each internal node, providing the left and right child chunks
+    /// for the node.
+    pub fn iter_internal_nodes<'a>(
+        &'a self,
+    ) -> impl DoubleEndedIterator<Item = (&'a usize, (&'a usize, &'a usize))> {
+        let internal_nodes = &self.offsets[0..self.num_internal_nodes];
+
+        internal_nodes.iter().enumerate().map(move |(i, parent)| {
+            let children = children(i);
+            (
+                parent,
+                (&self.offsets[children.0], &self.offsets[children.1]),
+            )
+        })
+    }
+
+    /// Returns an iterator visiting each leaf node, providing the chunk for that node.
+    pub fn iter_leaf_nodes<'a>(&'a self) -> impl DoubleEndedIterator<Item = &'a usize> {
+        let leaf_nodes = &self.offsets[self.num_internal_nodes..];
+
+        leaf_nodes.iter()
+    }
+}
diff --git a/eth2/utils/tree_hash/src/impls.rs b/eth2/utils/tree_hash/src/impls.rs
index d5297c38e..9149cf8aa 100644
--- a/eth2/utils/tree_hash/src/impls.rs
+++ b/eth2/utils/tree_hash/src/impls.rs
@@ -14,12 +14,12 @@ impl CachedTreeHash<u64> for u64 {
         )?)
     }
 
-    fn num_bytes(&self) -> usize {
-        8
+    fn btree_overlay(&self, _chunk_offset: usize) -> Result<BTreeOverlay, Error> {
+        Err(Error::ShouldNotProduceBTreeOverlay)
     }
 
-    fn offsets(&self) -> Result<Vec<usize>, Error> {
-        Err(Error::ShouldNotProduceBTreeOverlay)
+    fn num_bytes(&self) -> usize {
+        8
     }
 
     fn num_child_nodes(&self) -> usize {
@@ -71,21 +71,22 @@ where
         }
     }
 
-    fn offsets(&self) -> Result<Vec<usize>, Error> {
-        let offsets = match T::item_type() {
+    fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
+        //
+        let lengths = match T::item_type() {
             ItemType::Basic => vec![1; self.len() / T::packing_factor()],
             ItemType::Composite | ItemType::List => {
-                let mut offsets = vec![];
+                let mut lengths = vec![];
 
                 for item in self {
-                    offsets.push(BTreeOverlay::new(item, 0)?.total_nodes())
+                    lengths.push(BTreeOverlay::new(item, 0)?.total_nodes())
                 }
 
-                offsets
+                lengths
             }
         };
 
-        Ok(offsets)
+        BTreeOverlay::from_lengths(chunk_offset, lengths)
     }
 
     fn num_child_nodes(&self) -> usize {
@@ -180,7 +181,7 @@ where
                         (Some(old), None) => {
                             // Splice out the entire tree of the removed node, replacing it with a
                             // single padding node.
-                            let end_chunk = BTreeOverlay::new(old, start_chunk)?.next_node();
+                            let end_chunk = BTreeOverlay::new(old, start_chunk)?.next_node;
 
                             cache.splice(
                                 start_chunk..end_chunk,
@@ -218,7 +219,7 @@ where
             cache.modify_chunk(root_node, &cache.mix_in_length(root_node, self.len())?)?;
         }
 
-        Ok(offset_handler.next_node())
+        Ok(offset_handler.next_node)
     }
 }
 
diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs
index 1b085770d..e356210a4 100644
--- a/eth2/utils/tree_hash/src/lib.rs
+++ b/eth2/utils/tree_hash/src/lib.rs
@@ -1,13 +1,14 @@
 use hashing::hash;
 use int_to_bytes::int_to_bytes32;
 use std::fmt::Debug;
-use std::iter::Iterator;
 use std::ops::Range;
 
+mod btree_overlay;
 mod cached_tree_hash;
 mod impls;
 mod resize;
 
+pub use btree_overlay::BTreeOverlay;
 pub use cached_tree_hash::TreeHashCache;
 
 pub const BYTES_PER_CHUNK: usize = 32;
@@ -44,7 +45,7 @@ pub trait CachedTreeHash<Item>: Debug {
     /// prefixes.
     fn num_bytes(&self) -> usize;
 
-    fn offsets(&self) -> Result<Vec<usize>, Error>;
+    fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error>;
 
     fn num_child_nodes(&self) -> usize;
 
@@ -72,109 +73,6 @@ fn node_range_to_byte_range(node_range: &Range<usize>) -> Range<usize> {
     node_range.start * HASHSIZE..node_range.end * HASHSIZE
 }
 
-#[derive(Debug)]
-pub struct BTreeOverlay {
-    num_internal_nodes: usize,
-    pub num_leaf_nodes: usize,
-    first_node: usize,
-    next_node: usize,
-    offsets: Vec<usize>,
-}
-
-impl BTreeOverlay {
-    pub fn new<T>(item: &T, initial_offset: usize) -> Result<Self, Error>
-    where
-        T: CachedTreeHash<T>,
-    {
-        Self::from_lengths(initial_offset, item.offsets()?)
-    }
-
-    fn from_lengths(offset: usize, mut lengths: Vec<usize>) -> Result<Self, Error> {
-        // Extend it to the next power-of-two, if it is not already.
-        let num_leaf_nodes = if lengths.len().is_power_of_two() {
-            lengths.len()
-        } else {
-            let num_leaf_nodes = lengths.len().next_power_of_two();
-            lengths.resize(num_leaf_nodes, 1);
-            num_leaf_nodes
-        };
-
-        let num_nodes = num_nodes(num_leaf_nodes);
-        let num_internal_nodes = num_nodes - num_leaf_nodes;
-
-        let mut offsets = Vec::with_capacity(num_nodes);
-        offsets.append(&mut (offset..offset + num_internal_nodes).collect());
-
-        let mut next_node = num_internal_nodes + offset;
-        for i in 0..num_leaf_nodes {
-            offsets.push(next_node);
-            next_node += lengths[i];
-        }
-
-        Ok(Self {
-            num_internal_nodes,
-            num_leaf_nodes,
-            offsets,
-            first_node: offset,
-            next_node,
-        })
-    }
-
-    pub fn root(&self) -> usize {
-        self.first_node
-    }
-
-    pub fn height(&self) -> usize {
-        self.num_leaf_nodes.trailing_zeros() as usize
-    }
-
-    pub fn chunk_range(&self) -> Range<usize> {
-        self.first_node..self.next_node
-    }
-
-    pub fn total_chunks(&self) -> usize {
-        self.next_node - self.first_node
-    }
-
-    pub fn total_nodes(&self) -> usize {
-        self.num_internal_nodes + self.num_leaf_nodes
-    }
-
-    pub fn first_leaf_node(&self) -> Result<usize, Error> {
-        self.offsets
-            .get(self.num_internal_nodes)
-            .cloned()
-            .ok_or_else(|| Error::NoFirstNode)
-    }
-
-    pub fn next_node(&self) -> usize {
-        self.next_node
-    }
-
-    /// Returns an iterator visiting each internal node, providing the left and right child chunks
-    /// for the node.
-    pub fn iter_internal_nodes<'a>(
-        &'a self,
-    ) -> impl DoubleEndedIterator<Item = (&'a usize, (&'a usize, &'a usize))> {
-        let internal_nodes = &self.offsets[0..self.num_internal_nodes];
-
-        internal_nodes.iter().enumerate().map(move |(i, parent)| {
-            let children = children(i);
-            (
-                parent,
-                (&self.offsets[children.0], &self.offsets[children.1]),
-            )
-        })
-    }
-
-    /// Returns an iterator visiting each leaf node, providing the chunk for that node.
-    pub fn iter_leaf_nodes<'a>(&'a self) -> impl DoubleEndedIterator<Item = &'a usize> {
-        let leaf_nodes = &self.offsets[self.num_internal_nodes..];
-
-        leaf_nodes.iter()
-    }
-}
-
 /// Split `values` into a power-of-two, identical-length chunks (padding with `0`) and merkleize
 /// them, returning the entire merkle tree.
 ///
diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
index 972eb1e00..af4204cc2 100644
--- a/eth2/utils/tree_hash/tests/tests.rs
+++ b/eth2/utils/tree_hash/tests/tests.rs
@@ -44,15 +44,15 @@ impl CachedTreeHash<Inner> for Inner {
         bytes
     }
 
-    fn offsets(&self) -> Result<Vec<usize>, Error> {
-        let mut offsets = vec![];
+    fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
+        let mut lengths = vec![];
 
-        offsets.push(self.a.num_child_nodes() + 1);
-        offsets.push(self.b.num_child_nodes() + 1);
-        offsets.push(self.c.num_child_nodes() + 1);
-        offsets.push(self.d.num_child_nodes() + 1);
+        lengths.push(self.a.num_child_nodes() + 1);
+        lengths.push(self.b.num_child_nodes() + 1);
+        lengths.push(self.c.num_child_nodes() + 1);
+        lengths.push(self.d.num_child_nodes() + 1);
 
-        Ok(offsets)
+        BTreeOverlay::from_lengths(chunk_offset, lengths)
     }
 
     fn num_child_nodes(&self) -> usize {
@@ -98,7 +98,7 @@ impl CachedTreeHash<Inner> for Inner {
             }
         }
 
-        Ok(offset_handler.next_node())
+        Ok(offset_handler.next_node)
     }
 }
 
@@ -146,14 +146,14 @@ impl CachedTreeHash<Outer> for Outer {
         num_nodes(leaves) + children - 1
     }
 
-    fn offsets(&self) -> Result<Vec<usize>, Error> {
-        let mut offsets = vec![];
+    fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
+        let mut lengths = vec![];
 
-        offsets.push(self.a.num_child_nodes() + 1);
-        offsets.push(self.b.num_child_nodes() + 1);
-        offsets.push(self.c.num_child_nodes() + 1);
+        lengths.push(self.a.num_child_nodes() + 1);
+        lengths.push(self.b.num_child_nodes() + 1);
+        lengths.push(self.c.num_child_nodes() + 1);
 
-        Ok(offsets)
+        BTreeOverlay::from_lengths(chunk_offset, lengths)
     }
 
     fn packed_encoding(&self) -> Vec<u8> {
@@ -186,7 +186,7 @@ impl CachedTreeHash<Outer> for Outer {
             }
         }
 
-        Ok(offset_handler.next_node())
+        Ok(offset_handler.next_node)
     }
 }
 

From e6c33e1b60e560fb7539c0965f441dccb6af277d Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Mon, 15 Apr 2019 11:44:44 +1000
Subject: [PATCH 044/137] Remove child_nodes method from CachedTreeHash

---
 eth2/utils/tree_hash/src/impls.rs   | 14 ++---------
 eth2/utils/tree_hash/src/lib.rs     |  2 --
 eth2/utils/tree_hash/tests/tests.rs | 37 ++++++-----------------------
 3 files changed, 9 insertions(+), 44 deletions(-)

diff --git a/eth2/utils/tree_hash/src/impls.rs b/eth2/utils/tree_hash/src/impls.rs
index 9149cf8aa..4349d73d8 100644
--- a/eth2/utils/tree_hash/src/impls.rs
+++ b/eth2/utils/tree_hash/src/impls.rs
@@ -14,18 +14,14 @@ impl CachedTreeHash<u64> for u64 {
         )?)
     }
 
-    fn btree_overlay(&self, _chunk_offset: usize) -> Result<BTreeOverlay, Error> {
-        Err(Error::ShouldNotProduceBTreeOverlay)
+    fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
+        BTreeOverlay::from_lengths(chunk_offset, vec![1])
     }
 
     fn num_bytes(&self) -> usize {
         8
     }
 
-    fn num_child_nodes(&self) -> usize {
-        0
-    }
-
     fn packed_encoding(&self) -> Vec<u8> {
         ssz_encode(self)
     }
@@ -72,7 +68,6 @@ where
     }
 
     fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
-        //
         let lengths = match T::item_type() {
             ItemType::Basic => vec![1; self.len() / T::packing_factor()],
             ItemType::Composite | ItemType::List => {
@@ -89,11 +84,6 @@ where
         BTreeOverlay::from_lengths(chunk_offset, lengths)
     }
 
-    fn num_child_nodes(&self) -> usize {
-        // TODO
-        42
-    }
-
     fn num_bytes(&self) -> usize {
         self.iter().fold(0, |acc, item| acc + item.num_bytes())
     }
diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs
index e356210a4..4e1cff0e8 100644
--- a/eth2/utils/tree_hash/src/lib.rs
+++ b/eth2/utils/tree_hash/src/lib.rs
@@ -47,8 +47,6 @@ pub trait CachedTreeHash<Item>: Debug {
 
     fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error>;
 
-    fn num_child_nodes(&self) -> usize;
-
     fn packed_encoding(&self) -> Vec<u8>;
 
     fn packing_factor() -> usize;
diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
index af4204cc2..a315feeed 100644
--- a/eth2/utils/tree_hash/tests/tests.rs
+++ b/eth2/utils/tree_hash/tests/tests.rs
@@ -47,26 +47,14 @@ impl CachedTreeHash<Inner> for Inner {
     fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
         let mut lengths = vec![];
 
-        lengths.push(self.a.num_child_nodes() + 1);
-        lengths.push(self.b.num_child_nodes() + 1);
-        lengths.push(self.c.num_child_nodes() + 1);
-        lengths.push(self.d.num_child_nodes() + 1);
+        lengths.push(BTreeOverlay::new(&self.a, 0)?.total_nodes());
+        lengths.push(BTreeOverlay::new(&self.b, 0)?.total_nodes());
+        lengths.push(BTreeOverlay::new(&self.c, 0)?.total_nodes());
+        lengths.push(BTreeOverlay::new(&self.d, 0)?.total_nodes());
 
         BTreeOverlay::from_lengths(chunk_offset, lengths)
     }
 
-    fn num_child_nodes(&self) -> usize {
-        let mut children = 0;
-        let leaves = 4;
-
-        children += self.a.num_child_nodes();
-        children += self.b.num_child_nodes();
-        children += self.c.num_child_nodes();
-        children += self.d.num_child_nodes();
-
-        num_nodes(leaves) + children - 1
-    }
-
     fn packed_encoding(&self) -> Vec<u8> {
         panic!("Struct should never be packed")
     }
@@ -135,23 +123,12 @@ impl CachedTreeHash<Outer> for Outer {
         bytes
     }
 
-    fn num_child_nodes(&self) -> usize {
-        let mut children = 0;
-        let leaves = 3;
-
-        children += self.a.num_child_nodes();
-        children += self.b.num_child_nodes();
-        children += self.c.num_child_nodes();
-
-        num_nodes(leaves) + children - 1
-    }
-
     fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
         let mut lengths = vec![];
 
-        lengths.push(self.a.num_child_nodes() + 1);
-        lengths.push(self.b.num_child_nodes() + 1);
-        lengths.push(self.c.num_child_nodes() + 1);
+        lengths.push(BTreeOverlay::new(&self.a, 0)?.total_nodes());
+        lengths.push(BTreeOverlay::new(&self.b, 0)?.total_nodes());
+        lengths.push(BTreeOverlay::new(&self.c, 0)?.total_nodes());
 
         BTreeOverlay::from_lengths(chunk_offset, lengths)
     }

From cb9b59b93d90a727c9a4e3eac0423a7a42d0eca0 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Mon, 15 Apr 2019 11:49:50 +1000
Subject: [PATCH 045/137] Remove panics from packed_encoding

---
 eth2/utils/tree_hash/src/impls.rs   | 20 +++++++++++---------
 eth2/utils/tree_hash/src/lib.rs     |  3 ++-
 eth2/utils/tree_hash/tests/tests.rs |  8 ++++----
 3 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/eth2/utils/tree_hash/src/impls.rs b/eth2/utils/tree_hash/src/impls.rs
index 4349d73d8..05da2753a 100644
--- a/eth2/utils/tree_hash/src/impls.rs
+++ b/eth2/utils/tree_hash/src/impls.rs
@@ -22,8 +22,8 @@ impl CachedTreeHash<u64> for u64 {
         8
     }
 
-    fn packed_encoding(&self) -> Vec<u8> {
-        ssz_encode(self)
+    fn packed_encoding(&self) -> Result<Vec<u8>, Error> {
+        Ok(ssz_encode(self))
     }
 
     fn packing_factor() -> usize {
@@ -55,7 +55,9 @@ where
 
     fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
         match T::item_type() {
-            ItemType::Basic => TreeHashCache::from_bytes(merkleize(get_packed_leaves(self)), false),
+            ItemType::Basic => {
+                TreeHashCache::from_bytes(merkleize(get_packed_leaves(self)?), false)
+            }
             ItemType::Composite | ItemType::List => {
                 let subtrees = self
                     .iter()
@@ -88,8 +90,8 @@ where
         self.iter().fold(0, |acc, item| acc + item.num_bytes())
     }
 
-    fn packed_encoding(&self) -> Vec<u8> {
-        panic!("List should never be packed")
+    fn packed_encoding(&self) -> Result<Vec<u8>, Error> {
+        Err(Error::ShouldNeverBePacked(Self::item_type()))
     }
 
     fn packing_factor() -> usize {
@@ -142,7 +144,7 @@ where
 
         match T::item_type() {
             ItemType::Basic => {
-                let leaves = get_packed_leaves(self);
+                let leaves = get_packed_leaves(self)?;
 
                 for (i, chunk) in offset_handler.iter_leaf_nodes().enumerate() {
                     if let Some(latest) = leaves.get(i * HASHSIZE..(i + 1) * HASHSIZE) {
@@ -213,7 +215,7 @@ where
     }
 }
 
-fn get_packed_leaves<T>(vec: &Vec<T>) -> Vec<u8>
+fn get_packed_leaves<T>(vec: &Vec<T>) -> Result<Vec<u8>, Error>
 where
     T: CachedTreeHash<T>,
 {
@@ -223,8 +225,8 @@ where
     let mut packed = Vec::with_capacity(num_leaves * HASHSIZE);
 
     for item in vec {
-        packed.append(&mut item.packed_encoding());
+        packed.append(&mut item.packed_encoding()?);
     }
 
-    sanitise_bytes(packed)
+    Ok(sanitise_bytes(packed))
 }
diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs
index 4e1cff0e8..76752e5b2 100644
--- a/eth2/utils/tree_hash/src/lib.rs
+++ b/eth2/utils/tree_hash/src/lib.rs
@@ -23,6 +23,7 @@ pub enum Error {
     UnableToObtainSlices,
     UnableToGrowMerkleTree,
     UnableToShrinkMerkleTree,
+    ShouldNeverBePacked(ItemType),
     BytesAreNotEvenChunks(usize),
     NoModifiedFieldForChunk(usize),
     NoBytesForChunk(usize),
@@ -47,7 +48,7 @@ pub trait CachedTreeHash<Item>: Debug {
 
     fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error>;
 
-    fn packed_encoding(&self) -> Vec<u8>;
+    fn packed_encoding(&self) -> Result<Vec<u8>, Error>;
 
     fn packing_factor() -> usize;
 
diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
index a315feeed..17ec121a8 100644
--- a/eth2/utils/tree_hash/tests/tests.rs
+++ b/eth2/utils/tree_hash/tests/tests.rs
@@ -55,8 +55,8 @@ impl CachedTreeHash<Inner> for Inner {
         BTreeOverlay::from_lengths(chunk_offset, lengths)
     }
 
-    fn packed_encoding(&self) -> Vec<u8> {
-        panic!("Struct should never be packed")
+    fn packed_encoding(&self) -> Result<Vec<u8>, Error> {
+        Err(Error::ShouldNeverBePacked(Self::item_type()))
     }
 
     fn packing_factor() -> usize {
@@ -133,8 +133,8 @@ impl CachedTreeHash<Outer> for Outer {
         BTreeOverlay::from_lengths(chunk_offset, lengths)
     }
 
-    fn packed_encoding(&self) -> Vec<u8> {
-        panic!("Struct should never be packed")
+    fn packed_encoding(&self) -> Result<Vec<u8>, Error> {
+        Err(Error::ShouldNeverBePacked(Self::item_type()))
     }
 
     fn packing_factor() -> usize {

From c18cdf2abf7478e4d9535aa9d2581ea35df5931a Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Mon, 15 Apr 2019 11:55:56 +1000
Subject: [PATCH 046/137] Remove num_bytes method

---
 eth2/utils/tree_hash/src/impls.rs   | 12 ++----------
 eth2/utils/tree_hash/src/lib.rs     |  4 ----
 eth2/utils/tree_hash/tests/tests.rs | 19 -------------------
 3 files changed, 2 insertions(+), 33 deletions(-)

diff --git a/eth2/utils/tree_hash/src/impls.rs b/eth2/utils/tree_hash/src/impls.rs
index 05da2753a..53490551f 100644
--- a/eth2/utils/tree_hash/src/impls.rs
+++ b/eth2/utils/tree_hash/src/impls.rs
@@ -18,16 +18,12 @@ impl CachedTreeHash<u64> for u64 {
         BTreeOverlay::from_lengths(chunk_offset, vec![1])
     }
 
-    fn num_bytes(&self) -> usize {
-        8
-    }
-
     fn packed_encoding(&self) -> Result<Vec<u8>, Error> {
         Ok(ssz_encode(self))
     }
 
     fn packing_factor() -> usize {
-        32 / 8
+        HASHSIZE / 8
     }
 
     fn cached_hash_tree_root(
@@ -86,10 +82,6 @@ where
         BTreeOverlay::from_lengths(chunk_offset, lengths)
     }
 
-    fn num_bytes(&self) -> usize {
-        self.iter().fold(0, |acc, item| acc + item.num_bytes())
-    }
-
     fn packed_encoding(&self) -> Result<Vec<u8>, Error> {
         Err(Error::ShouldNeverBePacked(Self::item_type()))
     }
@@ -219,7 +211,7 @@ fn get_packed_leaves<T>(vec: &Vec<T>) -> Result<Vec<u8>, Error>
 where
     T: CachedTreeHash<T>,
 {
-    let num_packed_bytes = vec.num_bytes();
+    let num_packed_bytes = (BYTES_PER_CHUNK / T::packing_factor()) * vec.len();
     let num_leaves = num_sanitized_leaves(num_packed_bytes);
 
     let mut packed = Vec::with_capacity(num_leaves * HASHSIZE);
diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs
index 76752e5b2..0fd75dc5a 100644
--- a/eth2/utils/tree_hash/src/lib.rs
+++ b/eth2/utils/tree_hash/src/lib.rs
@@ -42,10 +42,6 @@ pub trait CachedTreeHash<Item>: Debug {
 
     fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error>;
 
-    /// Return the number of bytes when this element is encoded as raw SSZ _without_ length
-    /// prefixes.
-    fn num_bytes(&self) -> usize;
-
     fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error>;
 
     fn packed_encoding(&self) -> Result<Vec<u8>, Error>;
diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
index 17ec121a8..701bf8ec1 100644
--- a/eth2/utils/tree_hash/tests/tests.rs
+++ b/eth2/utils/tree_hash/tests/tests.rs
@@ -33,17 +33,6 @@ impl CachedTreeHash<Inner> for Inner {
         Ok(tree)
     }
 
-    fn num_bytes(&self) -> usize {
-        let mut bytes = 0;
-
-        bytes += self.a.num_bytes();
-        bytes += self.b.num_bytes();
-        bytes += self.c.num_bytes();
-        bytes += self.d.num_bytes();
-
-        bytes
-    }
-
     fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
         let mut lengths = vec![];
 
@@ -115,14 +104,6 @@ impl CachedTreeHash<Outer> for Outer {
         Ok(tree)
     }
 
-    fn num_bytes(&self) -> usize {
-        let mut bytes = 0;
-        bytes += self.a.num_bytes();
-        bytes += self.b.num_bytes();
-        bytes += self.c.num_bytes();
-        bytes
-    }
-
     fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
         let mut lengths = vec![];
 

From 8e5b79452ad5878e5f512d9e3cc35af1882b3bbf Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Mon, 15 Apr 2019 12:01:12 +1000
Subject: [PATCH 047/137] Further tidy cached tree hash

---
 eth2/utils/tree_hash/src/cached_tree_hash.rs |   2 +-
 eth2/utils/tree_hash/src/impls.rs            | 188 +------------------
 eth2/utils/tree_hash/src/impls/vec.rs        | 183 ++++++++++++++++++
 eth2/utils/tree_hash/src/lib.rs              |   6 +-
 eth2/utils/tree_hash/tests/tests.rs          |  48 +++--
 5 files changed, 214 insertions(+), 213 deletions(-)
 create mode 100644 eth2/utils/tree_hash/src/impls/vec.rs

diff --git a/eth2/utils/tree_hash/src/cached_tree_hash.rs b/eth2/utils/tree_hash/src/cached_tree_hash.rs
index 556ba2d21..4022f6b7b 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash.rs
@@ -17,7 +17,7 @@ impl TreeHashCache {
     where
         T: CachedTreeHash<T>,
     {
-        item.build_tree_hash_cache()
+        item.new_cache()
     }
 
     pub fn from_elems(cache: Vec<u8>, chunk_modified: Vec<bool>) -> Self {
diff --git a/eth2/utils/tree_hash/src/impls.rs b/eth2/utils/tree_hash/src/impls.rs
index 53490551f..6849fd55c 100644
--- a/eth2/utils/tree_hash/src/impls.rs
+++ b/eth2/utils/tree_hash/src/impls.rs
@@ -2,12 +2,14 @@ use super::resize::{grow_merkle_cache, shrink_merkle_cache};
 use super::*;
 use ssz::ssz_encode;
 
+mod vec;
+
 impl CachedTreeHash<u64> for u64 {
     fn item_type() -> ItemType {
         ItemType::Basic
     }
 
-    fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
+    fn new_cache(&self) -> Result<TreeHashCache, Error> {
         Ok(TreeHashCache::from_bytes(
             merkleize(ssz_encode(self)),
             false,
@@ -26,7 +28,7 @@ impl CachedTreeHash<u64> for u64 {
         HASHSIZE / 8
     }
 
-    fn cached_hash_tree_root(
+    fn update_cache(
         &self,
         other: &Self,
         cache: &mut TreeHashCache,
@@ -40,185 +42,3 @@ impl CachedTreeHash<u64> for u64 {
         Ok(chunk + 1)
     }
 }
-
-impl<T> CachedTreeHash<Vec<T>> for Vec<T>
-where
-    T: CachedTreeHash<T>,
-{
-    fn item_type() -> ItemType {
-        ItemType::List
-    }
-
-    fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
-        match T::item_type() {
-            ItemType::Basic => {
-                TreeHashCache::from_bytes(merkleize(get_packed_leaves(self)?), false)
-            }
-            ItemType::Composite | ItemType::List => {
-                let subtrees = self
-                    .iter()
-                    .map(|item| TreeHashCache::new(item))
-                    .collect::<Result<Vec<TreeHashCache>, _>>()?;
-
-                TreeHashCache::from_leaves_and_subtrees(self, subtrees)
-            }
-        }
-    }
-
-    fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
-        let lengths = match T::item_type() {
-            ItemType::Basic => vec![1; self.len() / T::packing_factor()],
-            ItemType::Composite | ItemType::List => {
-                let mut lengths = vec![];
-
-                for item in self {
-                    lengths.push(BTreeOverlay::new(item, 0)?.total_nodes())
-                }
-
-                lengths
-            }
-        };
-
-        BTreeOverlay::from_lengths(chunk_offset, lengths)
-    }
-
-    fn packed_encoding(&self) -> Result<Vec<u8>, Error> {
-        Err(Error::ShouldNeverBePacked(Self::item_type()))
-    }
-
-    fn packing_factor() -> usize {
-        1
-    }
-
-    fn cached_hash_tree_root(
-        &self,
-        other: &Vec<T>,
-        cache: &mut TreeHashCache,
-        chunk: usize,
-    ) -> Result<usize, Error> {
-        let offset_handler = BTreeOverlay::new(self, chunk)?;
-        let old_offset_handler = BTreeOverlay::new(other, chunk)?;
-
-        if offset_handler.num_leaf_nodes != old_offset_handler.num_leaf_nodes {
-            let old_offset_handler = BTreeOverlay::new(other, chunk)?;
-
-            // Get slices of the exsiting tree from the cache.
-            let (old_bytes, old_flags) = cache
-                .slices(old_offset_handler.chunk_range())
-                .ok_or_else(|| Error::UnableToObtainSlices)?;
-
-            let (new_bytes, new_flags) =
-                if offset_handler.num_leaf_nodes > old_offset_handler.num_leaf_nodes {
-                    grow_merkle_cache(
-                        old_bytes,
-                        old_flags,
-                        old_offset_handler.height(),
-                        offset_handler.height(),
-                    )
-                    .ok_or_else(|| Error::UnableToGrowMerkleTree)?
-                } else {
-                    shrink_merkle_cache(
-                        old_bytes,
-                        old_flags,
-                        old_offset_handler.height(),
-                        offset_handler.height(),
-                        offset_handler.total_chunks(),
-                    )
-                    .ok_or_else(|| Error::UnableToShrinkMerkleTree)?
-                };
-
-            // Create a `TreeHashCache` from the raw elements.
-            let modified_cache = TreeHashCache::from_elems(new_bytes, new_flags);
-
-            // Splice the newly created `TreeHashCache` over the existing elements.
-            cache.splice(old_offset_handler.chunk_range(), modified_cache);
-        }
-
-        match T::item_type() {
-            ItemType::Basic => {
-                let leaves = get_packed_leaves(self)?;
-
-                for (i, chunk) in offset_handler.iter_leaf_nodes().enumerate() {
-                    if let Some(latest) = leaves.get(i * HASHSIZE..(i + 1) * HASHSIZE) {
-                        cache.maybe_update_chunk(*chunk, latest)?;
-                    }
-                }
-                let first_leaf_chunk = offset_handler.first_leaf_node()?;
-
-                cache.splice(
-                    first_leaf_chunk..offset_handler.next_node,
-                    TreeHashCache::from_bytes(leaves, true)?,
-                );
-            }
-            ItemType::Composite | ItemType::List => {
-                let mut i = offset_handler.num_leaf_nodes;
-                for &start_chunk in offset_handler.iter_leaf_nodes().rev() {
-                    i -= 1;
-                    match (other.get(i), self.get(i)) {
-                        // The item existed in the previous list and exsits in the current list.
-                        (Some(old), Some(new)) => {
-                            new.cached_hash_tree_root(old, cache, start_chunk)?;
-                        }
-                        // The item existed in the previous list but does not exist in this list.
-                        //
-                        // I.e., the list has been shortened.
-                        (Some(old), None) => {
-                            // Splice out the entire tree of the removed node, replacing it with a
-                            // single padding node.
-                            let end_chunk = BTreeOverlay::new(old, start_chunk)?.next_node;
-
-                            cache.splice(
-                                start_chunk..end_chunk,
-                                TreeHashCache::from_bytes(vec![0; HASHSIZE], true)?,
-                            );
-                        }
-                        // The item existed in the previous list but does exist in this list.
-                        //
-                        // I.e., the list has been lengthened.
-                        (None, Some(new)) => {
-                            let bytes: Vec<u8> = TreeHashCache::new(new)?.into();
-
-                            cache.splice(
-                                start_chunk..start_chunk + 1,
-                                TreeHashCache::from_bytes(bytes, true)?,
-                            );
-                        }
-                        // The item didn't exist in the old list and doesn't exist in the new list,
-                        // nothing to do.
-                        (None, None) => {}
-                    };
-                }
-            }
-        }
-
-        for (&parent, children) in offset_handler.iter_internal_nodes().rev() {
-            if cache.either_modified(children)? {
-                cache.modify_chunk(parent, &cache.hash_children(children)?)?;
-            }
-        }
-
-        // If the root node or the length has changed, mix in the length of the list.
-        let root_node = offset_handler.root();
-        if cache.changed(root_node)? | (self.len() != other.len()) {
-            cache.modify_chunk(root_node, &cache.mix_in_length(root_node, self.len())?)?;
-        }
-
-        Ok(offset_handler.next_node)
-    }
-}
-
-fn get_packed_leaves<T>(vec: &Vec<T>) -> Result<Vec<u8>, Error>
-where
-    T: CachedTreeHash<T>,
-{
-    let num_packed_bytes = (BYTES_PER_CHUNK / T::packing_factor()) * vec.len();
-    let num_leaves = num_sanitized_leaves(num_packed_bytes);
-
-    let mut packed = Vec::with_capacity(num_leaves * HASHSIZE);
-
-    for item in vec {
-        packed.append(&mut item.packed_encoding()?);
-    }
-
-    Ok(sanitise_bytes(packed))
-}
diff --git a/eth2/utils/tree_hash/src/impls/vec.rs b/eth2/utils/tree_hash/src/impls/vec.rs
new file mode 100644
index 000000000..7c0993f43
--- /dev/null
+++ b/eth2/utils/tree_hash/src/impls/vec.rs
@@ -0,0 +1,183 @@
+use super::*;
+
+impl<T> CachedTreeHash<Vec<T>> for Vec<T>
+where
+    T: CachedTreeHash<T>,
+{
+    fn item_type() -> ItemType {
+        ItemType::List
+    }
+
+    fn new_cache(&self) -> Result<TreeHashCache, Error> {
+        match T::item_type() {
+            ItemType::Basic => {
+                TreeHashCache::from_bytes(merkleize(get_packed_leaves(self)?), false)
+            }
+            ItemType::Composite | ItemType::List => {
+                let subtrees = self
+                    .iter()
+                    .map(|item| TreeHashCache::new(item))
+                    .collect::<Result<Vec<TreeHashCache>, _>>()?;
+
+                TreeHashCache::from_leaves_and_subtrees(self, subtrees)
+            }
+        }
+    }
+
+    fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
+        let lengths = match T::item_type() {
+            ItemType::Basic => vec![1; self.len() / T::packing_factor()],
+            ItemType::Composite | ItemType::List => {
+                let mut lengths = vec![];
+
+                for item in self {
+                    lengths.push(BTreeOverlay::new(item, 0)?.total_nodes())
+                }
+
+                lengths
+            }
+        };
+
+        BTreeOverlay::from_lengths(chunk_offset, lengths)
+    }
+
+    fn packed_encoding(&self) -> Result<Vec<u8>, Error> {
+        Err(Error::ShouldNeverBePacked(Self::item_type()))
+    }
+
+    fn packing_factor() -> usize {
+        1
+    }
+
+    fn update_cache(
+        &self,
+        other: &Vec<T>,
+        cache: &mut TreeHashCache,
+        chunk: usize,
+    ) -> Result<usize, Error> {
+        let offset_handler = BTreeOverlay::new(self, chunk)?;
+        let old_offset_handler = BTreeOverlay::new(other, chunk)?;
+
+        if offset_handler.num_leaf_nodes != old_offset_handler.num_leaf_nodes {
+            let old_offset_handler = BTreeOverlay::new(other, chunk)?;
+
+            // Get slices of the exsiting tree from the cache.
+            let (old_bytes, old_flags) = cache
+                .slices(old_offset_handler.chunk_range())
+                .ok_or_else(|| Error::UnableToObtainSlices)?;
+
+            let (new_bytes, new_flags) =
+                if offset_handler.num_leaf_nodes > old_offset_handler.num_leaf_nodes {
+                    grow_merkle_cache(
+                        old_bytes,
+                        old_flags,
+                        old_offset_handler.height(),
+                        offset_handler.height(),
+                    )
+                    .ok_or_else(|| Error::UnableToGrowMerkleTree)?
+                } else {
+                    shrink_merkle_cache(
+                        old_bytes,
+                        old_flags,
+                        old_offset_handler.height(),
+                        offset_handler.height(),
+                        offset_handler.total_chunks(),
+                    )
+                    .ok_or_else(|| Error::UnableToShrinkMerkleTree)?
+                };
+
+            // Create a `TreeHashCache` from the raw elements.
+            let modified_cache = TreeHashCache::from_elems(new_bytes, new_flags);
+
+            // Splice the newly created `TreeHashCache` over the existing elements.
+            cache.splice(old_offset_handler.chunk_range(), modified_cache);
+        }
+
+        match T::item_type() {
+            ItemType::Basic => {
+                let leaves = get_packed_leaves(self)?;
+
+                for (i, chunk) in offset_handler.iter_leaf_nodes().enumerate() {
+                    if let Some(latest) = leaves.get(i * HASHSIZE..(i + 1) * HASHSIZE) {
+                        cache.maybe_update_chunk(*chunk, latest)?;
+                    }
+                }
+                let first_leaf_chunk = offset_handler.first_leaf_node()?;
+
+                cache.splice(
+                    first_leaf_chunk..offset_handler.next_node,
+                    TreeHashCache::from_bytes(leaves, true)?,
+                );
+            }
+            ItemType::Composite | ItemType::List => {
+                let mut i = offset_handler.num_leaf_nodes;
+                for &start_chunk in offset_handler.iter_leaf_nodes().rev() {
+                    i -= 1;
+                    match (other.get(i), self.get(i)) {
+                        // The item existed in the previous list and exsits in the current list.
+                        (Some(old), Some(new)) => {
+                            new.update_cache(old, cache, start_chunk)?;
+                        }
+                        // The item existed in the previous list but does not exist in this list.
+                        //
+                        // I.e., the list has been shortened.
+                        (Some(old), None) => {
+                            // Splice out the entire tree of the removed node, replacing it with a
+                            // single padding node.
+                            let end_chunk = BTreeOverlay::new(old, start_chunk)?.next_node;
+
+                            cache.splice(
+                                start_chunk..end_chunk,
+                                TreeHashCache::from_bytes(vec![0; HASHSIZE], true)?,
+                            );
+                        }
+                        // The item existed in the previous list but does exist in this list.
+                        //
+                        // I.e., the list has been lengthened.
+                        (None, Some(new)) => {
+                            let bytes: Vec<u8> = TreeHashCache::new(new)?.into();
+
+                            cache.splice(
+                                start_chunk..start_chunk + 1,
+                                TreeHashCache::from_bytes(bytes, true)?,
+                            );
+                        }
+                        // The item didn't exist in the old list and doesn't exist in the new list,
+                        // nothing to do.
+                        (None, None) => {}
+                    };
+                }
+            }
+        }
+
+        for (&parent, children) in offset_handler.iter_internal_nodes().rev() {
+            if cache.either_modified(children)? {
+                cache.modify_chunk(parent, &cache.hash_children(children)?)?;
+            }
+        }
+
+        // If the root node or the length has changed, mix in the length of the list.
+        let root_node = offset_handler.root();
+        if cache.changed(root_node)? | (self.len() != other.len()) {
+            cache.modify_chunk(root_node, &cache.mix_in_length(root_node, self.len())?)?;
+        }
+
+        Ok(offset_handler.next_node)
+    }
+}
+
+fn get_packed_leaves<T>(vec: &Vec<T>) -> Result<Vec<u8>, Error>
+where
+    T: CachedTreeHash<T>,
+{
+    let num_packed_bytes = (BYTES_PER_CHUNK / T::packing_factor()) * vec.len();
+    let num_leaves = num_sanitized_leaves(num_packed_bytes);
+
+    let mut packed = Vec::with_capacity(num_leaves * HASHSIZE);
+
+    for item in vec {
+        packed.append(&mut item.packed_encoding()?);
+    }
+
+    Ok(sanitise_bytes(packed))
+}
diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs
index 0fd75dc5a..b3167a37d 100644
--- a/eth2/utils/tree_hash/src/lib.rs
+++ b/eth2/utils/tree_hash/src/lib.rs
@@ -40,15 +40,15 @@ pub enum ItemType {
 pub trait CachedTreeHash<Item>: Debug {
     fn item_type() -> ItemType;
 
-    fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error>;
-
     fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error>;
 
     fn packed_encoding(&self) -> Result<Vec<u8>, Error>;
 
     fn packing_factor() -> usize;
 
-    fn cached_hash_tree_root(
+    fn new_cache(&self) -> Result<TreeHashCache, Error>;
+
+    fn update_cache(
         &self,
         other: &Item,
         cache: &mut TreeHashCache,
diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
index 701bf8ec1..22780bcac 100644
--- a/eth2/utils/tree_hash/tests/tests.rs
+++ b/eth2/utils/tree_hash/tests/tests.rs
@@ -19,14 +19,14 @@ impl CachedTreeHash<Inner> for Inner {
         ItemType::Composite
     }
 
-    fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
+    fn new_cache(&self) -> Result<TreeHashCache, Error> {
         let tree = TreeHashCache::from_leaves_and_subtrees(
             self,
             vec![
-                self.a.build_tree_hash_cache()?,
-                self.b.build_tree_hash_cache()?,
-                self.c.build_tree_hash_cache()?,
-                self.d.build_tree_hash_cache()?,
+                self.a.new_cache()?,
+                self.b.new_cache()?,
+                self.c.new_cache()?,
+                self.d.new_cache()?,
             ],
         )?;
 
@@ -52,7 +52,7 @@ impl CachedTreeHash<Inner> for Inner {
         1
     }
 
-    fn cached_hash_tree_root(
+    fn update_cache(
         &self,
         other: &Self,
         cache: &mut TreeHashCache,
@@ -63,10 +63,10 @@ impl CachedTreeHash<Inner> for Inner {
         // Skip past the internal nodes and update any changed leaf nodes.
         {
             let chunk = offset_handler.first_leaf_node()?;
-            let chunk = self.a.cached_hash_tree_root(&other.a, cache, chunk)?;
-            let chunk = self.b.cached_hash_tree_root(&other.b, cache, chunk)?;
-            let chunk = self.c.cached_hash_tree_root(&other.c, cache, chunk)?;
-            let _chunk = self.d.cached_hash_tree_root(&other.d, cache, chunk)?;
+            let chunk = self.a.update_cache(&other.a, cache, chunk)?;
+            let chunk = self.b.update_cache(&other.b, cache, chunk)?;
+            let chunk = self.c.update_cache(&other.c, cache, chunk)?;
+            let _chunk = self.d.update_cache(&other.d, cache, chunk)?;
         }
 
         for (&parent, children) in offset_handler.iter_internal_nodes().rev() {
@@ -91,13 +91,13 @@ impl CachedTreeHash<Outer> for Outer {
         ItemType::Composite
     }
 
-    fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
+    fn new_cache(&self) -> Result<TreeHashCache, Error> {
         let tree = TreeHashCache::from_leaves_and_subtrees(
             self,
             vec![
-                self.a.build_tree_hash_cache()?,
-                self.b.build_tree_hash_cache()?,
-                self.c.build_tree_hash_cache()?,
+                self.a.new_cache()?,
+                self.b.new_cache()?,
+                self.c.new_cache()?,
             ],
         )?;
 
@@ -122,7 +122,7 @@ impl CachedTreeHash<Outer> for Outer {
         1
     }
 
-    fn cached_hash_tree_root(
+    fn update_cache(
         &self,
         other: &Self,
         cache: &mut TreeHashCache,
@@ -133,9 +133,9 @@ impl CachedTreeHash<Outer> for Outer {
         // Skip past the internal nodes and update any changed leaf nodes.
         {
             let chunk = offset_handler.first_leaf_node()?;
-            let chunk = self.a.cached_hash_tree_root(&other.a, cache, chunk)?;
-            let chunk = self.b.cached_hash_tree_root(&other.b, cache, chunk)?;
-            let _chunk = self.c.cached_hash_tree_root(&other.c, cache, chunk)?;
+            let chunk = self.a.update_cache(&other.a, cache, chunk)?;
+            let chunk = self.b.update_cache(&other.b, cache, chunk)?;
+            let _chunk = self.c.update_cache(&other.c, cache, chunk)?;
         }
 
         for (&parent, children) in offset_handler.iter_internal_nodes().rev() {
@@ -186,7 +186,7 @@ fn partial_modification_to_inner_struct() {
     let mut cache_struct = TreeHashCache::new(&original_outer).unwrap();
 
     modified_outer
-        .cached_hash_tree_root(&original_outer, &mut cache_struct, 0)
+        .update_cache(&original_outer, &mut cache_struct, 0)
         .unwrap();
 
     let modified_cache: Vec<u8> = cache_struct.into();
@@ -240,7 +240,7 @@ fn partial_modification_to_outer() {
     let mut cache_struct = TreeHashCache::new(&original_outer).unwrap();
 
     modified_outer
-        .cached_hash_tree_root(&original_outer, &mut cache_struct, 0)
+        .update_cache(&original_outer, &mut cache_struct, 0)
         .unwrap();
 
     let modified_cache: Vec<u8> = cache_struct.into();
@@ -326,7 +326,7 @@ fn test_u64_vec_modifications(original: Vec<u64>, modified: Vec<u64>) {
     // Perform a differential hash
     let mut cache_struct = TreeHashCache::from_bytes(original_cache.clone(), false).unwrap();
     modified
-        .cached_hash_tree_root(&original, &mut cache_struct, 0)
+        .update_cache(&original, &mut cache_struct, 0)
         .unwrap();
     let modified_cache: Vec<u8> = cache_struct.into();
 
@@ -430,9 +430,7 @@ fn large_vec_of_u64_builds() {
 fn test_inner_vec_modifications(original: Vec<Inner>, modified: Vec<Inner>, reference: Vec<u64>) {
     let mut cache = TreeHashCache::new(&original).unwrap();
 
-    modified
-        .cached_hash_tree_root(&original, &mut cache, 0)
-        .unwrap();
+    modified.update_cache(&original, &mut cache, 0).unwrap();
     let modified_cache: Vec<u8> = cache.into();
 
     // Build the reference vec.
@@ -792,7 +790,7 @@ fn generic_test(index: usize) {
     let mut cache_struct = TreeHashCache::from_bytes(cache.clone(), false).unwrap();
 
     changed_inner
-        .cached_hash_tree_root(&inner, &mut cache_struct, 0)
+        .update_cache(&inner, &mut cache_struct, 0)
         .unwrap();
 
     // assert_eq!(*cache_struct.hash_count, 3);

From 354f823c1628e56de6627961dc3746a1f09c7a2b Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Mon, 15 Apr 2019 15:13:02 +1000
Subject: [PATCH 048/137] Tidy tree hash cache, add new trait

---
 eth2/utils/tree_hash/src/btree_overlay.rs    |   2 +-
 eth2/utils/tree_hash/src/cached_tree_hash.rs |  13 +-
 eth2/utils/tree_hash/src/impls.rs            |   2 +-
 eth2/utils/tree_hash/src/impls/vec.rs        |   6 +-
 eth2/utils/tree_hash/src/lib.rs              |  12 +-
 eth2/utils/tree_hash/tests/tests.rs          | 121 ++++++++++++++++++-
 6 files changed, 138 insertions(+), 18 deletions(-)

diff --git a/eth2/utils/tree_hash/src/btree_overlay.rs b/eth2/utils/tree_hash/src/btree_overlay.rs
index 7d5602c0b..8c859d046 100644
--- a/eth2/utils/tree_hash/src/btree_overlay.rs
+++ b/eth2/utils/tree_hash/src/btree_overlay.rs
@@ -12,7 +12,7 @@ pub struct BTreeOverlay {
 impl BTreeOverlay {
     pub fn new<T>(item: &T, initial_offset: usize) -> Result<Self, Error>
     where
-        T: CachedTreeHash<T>,
+        T: CachedTreeHashSubtree<T>,
     {
         item.btree_overlay(initial_offset)
     }
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash.rs b/eth2/utils/tree_hash/src/cached_tree_hash.rs
index 4022f6b7b..97f9388a1 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash.rs
@@ -15,7 +15,7 @@ impl Into<Vec<u8>> for TreeHashCache {
 impl TreeHashCache {
     pub fn new<T>(item: &T) -> Result<Self, Error>
     where
-        T: CachedTreeHash<T>,
+        T: CachedTreeHashSubtree<T>,
     {
         item.new_cache()
     }
@@ -32,7 +32,7 @@ impl TreeHashCache {
         leaves_and_subtrees: Vec<Self>,
     ) -> Result<Self, Error>
     where
-        T: CachedTreeHash<T>,
+        T: CachedTreeHashSubtree<T>,
     {
         let offset_handler = BTreeOverlay::new(item, 0)?;
 
@@ -55,7 +55,7 @@ impl TreeHashCache {
         // Iterate through all of the leaves/subtrees, adding their root as a leaf node and then
         // concatenating their merkle trees.
         for t in leaves_and_subtrees {
-            leaves.append(&mut t.root()?);
+            leaves.append(&mut t.root().ok_or_else(|| Error::NoBytesForRoot)?.to_vec());
             cache.append(&mut t.into_merkle_tree());
         }
 
@@ -89,11 +89,8 @@ impl TreeHashCache {
         self.cache.len()
     }
 
-    pub fn root(&self) -> Result<Vec<u8>, Error> {
-        self.cache
-            .get(0..HASHSIZE)
-            .ok_or_else(|| Error::NoBytesForRoot)
-            .and_then(|slice| Ok(slice.to_vec()))
+    pub fn root(&self) -> Option<&[u8]> {
+        self.cache.get(0..HASHSIZE)
     }
 
     pub fn splice(&mut self, chunk_range: Range<usize>, replace_with: Self) {
diff --git a/eth2/utils/tree_hash/src/impls.rs b/eth2/utils/tree_hash/src/impls.rs
index 6849fd55c..bd5c352c9 100644
--- a/eth2/utils/tree_hash/src/impls.rs
+++ b/eth2/utils/tree_hash/src/impls.rs
@@ -4,7 +4,7 @@ use ssz::ssz_encode;
 
 mod vec;
 
-impl CachedTreeHash<u64> for u64 {
+impl CachedTreeHashSubtree<u64> for u64 {
     fn item_type() -> ItemType {
         ItemType::Basic
     }
diff --git a/eth2/utils/tree_hash/src/impls/vec.rs b/eth2/utils/tree_hash/src/impls/vec.rs
index 7c0993f43..c02460cf3 100644
--- a/eth2/utils/tree_hash/src/impls/vec.rs
+++ b/eth2/utils/tree_hash/src/impls/vec.rs
@@ -1,8 +1,8 @@
 use super::*;
 
-impl<T> CachedTreeHash<Vec<T>> for Vec<T>
+impl<T> CachedTreeHashSubtree<Vec<T>> for Vec<T>
 where
-    T: CachedTreeHash<T>,
+    T: CachedTreeHashSubtree<T>,
 {
     fn item_type() -> ItemType {
         ItemType::List
@@ -168,7 +168,7 @@ where
 
 fn get_packed_leaves<T>(vec: &Vec<T>) -> Result<Vec<u8>, Error>
 where
-    T: CachedTreeHash<T>,
+    T: CachedTreeHashSubtree<T>,
 {
     let num_packed_bytes = (BYTES_PER_CHUNK / T::packing_factor()) * vec.len();
     let num_leaves = num_sanitized_leaves(num_packed_bytes);
diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs
index b3167a37d..179f557ce 100644
--- a/eth2/utils/tree_hash/src/lib.rs
+++ b/eth2/utils/tree_hash/src/lib.rs
@@ -1,6 +1,5 @@
 use hashing::hash;
 use int_to_bytes::int_to_bytes32;
-use std::fmt::Debug;
 use std::ops::Range;
 
 mod btree_overlay;
@@ -36,8 +35,15 @@ pub enum ItemType {
     Composite,
 }
 
-// TODO: remove debug requirement.
-pub trait CachedTreeHash<Item>: Debug {
+pub trait CachedTreeHash<T>: CachedTreeHashSubtree<T> + Sized {
+    fn update_internal_tree_hash_cache(self, old: T) -> Result<(Self, Self), Error>;
+
+    fn cached_tree_hash_root(&self) -> Option<Vec<u8>>;
+
+    fn clone_without_tree_hash_cache(&self) -> Self;
+}
+
+pub trait CachedTreeHashSubtree<Item> {
     fn item_type() -> ItemType;
 
     fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error>;
diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
index 22780bcac..c61a010ca 100644
--- a/eth2/utils/tree_hash/tests/tests.rs
+++ b/eth2/utils/tree_hash/tests/tests.rs
@@ -2,6 +2,123 @@ use hashing::hash;
 use int_to_bytes::{int_to_bytes32, int_to_bytes8};
 use tree_hash::*;
 
+#[derive(Clone, Debug)]
+pub struct InternalCache {
+    pub a: u64,
+    pub b: u64,
+    pub cache: Option<TreeHashCache>,
+}
+
+impl CachedTreeHash<InternalCache> for InternalCache {
+    fn update_internal_tree_hash_cache(mut self, mut old: Self) -> Result<(Self, Self), Error> {
+        let mut local_cache = old.cache;
+        old.cache = None;
+
+        if let Some(ref mut local_cache) = local_cache {
+            self.update_cache(&old, local_cache, 0)?;
+        } else {
+            local_cache = Some(self.new_cache()?)
+        }
+
+        self.cache = local_cache;
+
+        Ok((old, self))
+    }
+
+    fn cached_tree_hash_root(&self) -> Option<Vec<u8>> {
+        match &self.cache {
+            None => None,
+            Some(c) => Some(c.root()?.to_vec()),
+        }
+    }
+
+    fn clone_without_tree_hash_cache(&self) -> Self {
+        Self {
+            a: self.a,
+            b: self.b,
+            cache: None,
+        }
+    }
+}
+
+#[test]
+fn works_when_embedded() {
+    let old = InternalCache {
+        a: 99,
+        b: 99,
+        cache: None,
+    };
+
+    let mut new = old.clone_without_tree_hash_cache();
+    new.a = 1;
+    new.b = 2;
+
+    let (_old, new) = new.update_internal_tree_hash_cache(old).unwrap();
+
+    let root = new.cached_tree_hash_root().unwrap();
+
+    let leaves = vec![int_to_bytes32(1), int_to_bytes32(2)];
+    let merkle = merkleize(join(leaves));
+
+    assert_eq!(&merkle[0..32], &root[..]);
+}
+
+impl CachedTreeHashSubtree<InternalCache> for InternalCache {
+    fn item_type() -> ItemType {
+        ItemType::Composite
+    }
+
+    fn new_cache(&self) -> Result<TreeHashCache, Error> {
+        let tree = TreeHashCache::from_leaves_and_subtrees(
+            self,
+            vec![self.a.new_cache()?, self.b.new_cache()?],
+        )?;
+
+        Ok(tree)
+    }
+
+    fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
+        let mut lengths = vec![];
+
+        lengths.push(BTreeOverlay::new(&self.a, 0)?.total_nodes());
+        lengths.push(BTreeOverlay::new(&self.b, 0)?.total_nodes());
+
+        BTreeOverlay::from_lengths(chunk_offset, lengths)
+    }
+
+    fn packed_encoding(&self) -> Result<Vec<u8>, Error> {
+        Err(Error::ShouldNeverBePacked(Self::item_type()))
+    }
+
+    fn packing_factor() -> usize {
+        1
+    }
+
+    fn update_cache(
+        &self,
+        other: &Self,
+        cache: &mut TreeHashCache,
+        chunk: usize,
+    ) -> Result<usize, Error> {
+        let offset_handler = BTreeOverlay::new(self, chunk)?;
+
+        // Skip past the internal nodes and update any changed leaf nodes.
+        {
+            let chunk = offset_handler.first_leaf_node()?;
+            let chunk = self.a.update_cache(&other.a, cache, chunk)?;
+            let _chunk = self.b.update_cache(&other.b, cache, chunk)?;
+        }
+
+        for (&parent, children) in offset_handler.iter_internal_nodes().rev() {
+            if cache.either_modified(children)? {
+                cache.modify_chunk(parent, &cache.hash_children(children)?)?;
+            }
+        }
+
+        Ok(offset_handler.next_node)
+    }
+}
+
 fn num_nodes(num_leaves: usize) -> usize {
     2 * num_leaves - 1
 }
@@ -14,7 +131,7 @@ pub struct Inner {
     pub d: u64,
 }
 
-impl CachedTreeHash<Inner> for Inner {
+impl CachedTreeHashSubtree<Inner> for Inner {
     fn item_type() -> ItemType {
         ItemType::Composite
     }
@@ -86,7 +203,7 @@ pub struct Outer {
     pub c: u64,
 }
 
-impl CachedTreeHash<Outer> for Outer {
+impl CachedTreeHashSubtree<Outer> for Outer {
     fn item_type() -> ItemType {
         ItemType::Composite
     }

From 2be05a466f0e0cddc56fa807c6e70b28913fafe3 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Mon, 15 Apr 2019 15:45:05 +1000
Subject: [PATCH 049/137] Add tree_hash_derive crate

---
 Cargo.toml                                   |   1 +
 eth2/utils/tree_hash/src/btree_overlay.rs    |   2 +-
 eth2/utils/tree_hash/src/cached_tree_hash.rs |   4 +-
 eth2/utils/tree_hash/src/impls.rs            |   2 +-
 eth2/utils/tree_hash/src/impls/vec.rs        |   6 +-
 eth2/utils/tree_hash/src/lib.rs              |   4 +-
 eth2/utils/tree_hash/tests/tests.rs          |   6 +-
 eth2/utils/tree_hash_derive/Cargo.toml       |  16 +++
 eth2/utils/tree_hash_derive/src/lib.rs       | 125 +++++++++++++++++++
 eth2/utils/tree_hash_derive/tests/tests.rs   |   9 ++
 10 files changed, 163 insertions(+), 12 deletions(-)
 create mode 100644 eth2/utils/tree_hash_derive/Cargo.toml
 create mode 100644 eth2/utils/tree_hash_derive/src/lib.rs
 create mode 100644 eth2/utils/tree_hash_derive/tests/tests.rs

diff --git a/Cargo.toml b/Cargo.toml
index 2574d328f..b419d32e4 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -19,6 +19,7 @@ members = [
 	"eth2/utils/ssz_derive",
 	"eth2/utils/swap_or_not_shuffle",
 	"eth2/utils/tree_hash",
+	"eth2/utils/tree_hash_derive",
 	"eth2/utils/fisher_yates_shuffle",
     "eth2/utils/test_random_derive",
 	"beacon_node",
diff --git a/eth2/utils/tree_hash/src/btree_overlay.rs b/eth2/utils/tree_hash/src/btree_overlay.rs
index 8c859d046..1e188da60 100644
--- a/eth2/utils/tree_hash/src/btree_overlay.rs
+++ b/eth2/utils/tree_hash/src/btree_overlay.rs
@@ -12,7 +12,7 @@ pub struct BTreeOverlay {
 impl BTreeOverlay {
     pub fn new<T>(item: &T, initial_offset: usize) -> Result<Self, Error>
     where
-        T: CachedTreeHashSubtree<T>,
+        T: CachedTreeHashSubTree<T>,
     {
         item.btree_overlay(initial_offset)
     }
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash.rs b/eth2/utils/tree_hash/src/cached_tree_hash.rs
index 97f9388a1..048d4bab5 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash.rs
@@ -15,7 +15,7 @@ impl Into<Vec<u8>> for TreeHashCache {
 impl TreeHashCache {
     pub fn new<T>(item: &T) -> Result<Self, Error>
     where
-        T: CachedTreeHashSubtree<T>,
+        T: CachedTreeHashSubTree<T>,
     {
         item.new_cache()
     }
@@ -32,7 +32,7 @@ impl TreeHashCache {
         leaves_and_subtrees: Vec<Self>,
     ) -> Result<Self, Error>
     where
-        T: CachedTreeHashSubtree<T>,
+        T: CachedTreeHashSubTree<T>,
     {
         let offset_handler = BTreeOverlay::new(item, 0)?;
 
diff --git a/eth2/utils/tree_hash/src/impls.rs b/eth2/utils/tree_hash/src/impls.rs
index bd5c352c9..982e98724 100644
--- a/eth2/utils/tree_hash/src/impls.rs
+++ b/eth2/utils/tree_hash/src/impls.rs
@@ -4,7 +4,7 @@ use ssz::ssz_encode;
 
 mod vec;
 
-impl CachedTreeHashSubtree<u64> for u64 {
+impl CachedTreeHashSubTree<u64> for u64 {
     fn item_type() -> ItemType {
         ItemType::Basic
     }
diff --git a/eth2/utils/tree_hash/src/impls/vec.rs b/eth2/utils/tree_hash/src/impls/vec.rs
index c02460cf3..a6fad9ba6 100644
--- a/eth2/utils/tree_hash/src/impls/vec.rs
+++ b/eth2/utils/tree_hash/src/impls/vec.rs
@@ -1,8 +1,8 @@
 use super::*;
 
-impl<T> CachedTreeHashSubtree<Vec<T>> for Vec<T>
+impl<T> CachedTreeHashSubTree<Vec<T>> for Vec<T>
 where
-    T: CachedTreeHashSubtree<T>,
+    T: CachedTreeHashSubTree<T>,
 {
     fn item_type() -> ItemType {
         ItemType::List
@@ -168,7 +168,7 @@ where
 
 fn get_packed_leaves<T>(vec: &Vec<T>) -> Result<Vec<u8>, Error>
 where
-    T: CachedTreeHashSubtree<T>,
+    T: CachedTreeHashSubTree<T>,
 {
     let num_packed_bytes = (BYTES_PER_CHUNK / T::packing_factor()) * vec.len();
     let num_leaves = num_sanitized_leaves(num_packed_bytes);
diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs
index 179f557ce..5ec2b0283 100644
--- a/eth2/utils/tree_hash/src/lib.rs
+++ b/eth2/utils/tree_hash/src/lib.rs
@@ -35,7 +35,7 @@ pub enum ItemType {
     Composite,
 }
 
-pub trait CachedTreeHash<T>: CachedTreeHashSubtree<T> + Sized {
+pub trait CachedTreeHash<T>: CachedTreeHashSubTree<T> + Sized {
     fn update_internal_tree_hash_cache(self, old: T) -> Result<(Self, Self), Error>;
 
     fn cached_tree_hash_root(&self) -> Option<Vec<u8>>;
@@ -43,7 +43,7 @@ pub trait CachedTreeHash<T>: CachedTreeHashSubtree<T> + Sized {
     fn clone_without_tree_hash_cache(&self) -> Self;
 }
 
-pub trait CachedTreeHashSubtree<Item> {
+pub trait CachedTreeHashSubTree<Item> {
     fn item_type() -> ItemType;
 
     fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error>;
diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
index c61a010ca..ead6d8c00 100644
--- a/eth2/utils/tree_hash/tests/tests.rs
+++ b/eth2/utils/tree_hash/tests/tests.rs
@@ -63,7 +63,7 @@ fn works_when_embedded() {
     assert_eq!(&merkle[0..32], &root[..]);
 }
 
-impl CachedTreeHashSubtree<InternalCache> for InternalCache {
+impl CachedTreeHashSubTree<InternalCache> for InternalCache {
     fn item_type() -> ItemType {
         ItemType::Composite
     }
@@ -131,7 +131,7 @@ pub struct Inner {
     pub d: u64,
 }
 
-impl CachedTreeHashSubtree<Inner> for Inner {
+impl CachedTreeHashSubTree<Inner> for Inner {
     fn item_type() -> ItemType {
         ItemType::Composite
     }
@@ -203,7 +203,7 @@ pub struct Outer {
     pub c: u64,
 }
 
-impl CachedTreeHashSubtree<Outer> for Outer {
+impl CachedTreeHashSubTree<Outer> for Outer {
     fn item_type() -> ItemType {
         ItemType::Composite
     }
diff --git a/eth2/utils/tree_hash_derive/Cargo.toml b/eth2/utils/tree_hash_derive/Cargo.toml
new file mode 100644
index 000000000..f227d7954
--- /dev/null
+++ b/eth2/utils/tree_hash_derive/Cargo.toml
@@ -0,0 +1,16 @@
+[package]
+name = "tree_hash_derive"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+description = "Procedural derive macros for SSZ tree hashing."
+
+[lib]
+proc-macro = true
+
+[dev-dependencies]
+tree_hash = { path = "../tree_hash" }
+
+[dependencies]
+syn = "0.15"
+quote = "0.6"
diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs
new file mode 100644
index 000000000..217e91c24
--- /dev/null
+++ b/eth2/utils/tree_hash_derive/src/lib.rs
@@ -0,0 +1,125 @@
+#![recursion_limit = "256"]
+extern crate proc_macro;
+
+use proc_macro::TokenStream;
+use quote::quote;
+use syn::{parse_macro_input, DeriveInput};
+
+/// Returns a Vec of `syn::Ident` for each named field in the struct, whilst filtering out fields
+/// that should not be hashed.
+///
+/// # Panics
+/// Any unnamed struct field (like in a tuple struct) will raise a panic at compile time.
+fn get_hashable_named_field_idents<'a>(struct_data: &'a syn::DataStruct) -> Vec<&'a syn::Ident> {
+    struct_data
+        .fields
+        .iter()
+        .filter_map(|f| {
+            if should_skip_hashing(&f) {
+                None
+            } else {
+                Some(match &f.ident {
+                    Some(ref ident) => ident,
+                    _ => panic!("tree_hash_derive only supports named struct fields."),
+                })
+            }
+        })
+        .collect()
+}
+
+/// Returns true if some field has an attribute declaring it should not be hashedd.
+///
+/// The field attribute is: `#[tree_hash(skip_hashing)]`
+fn should_skip_hashing(field: &syn::Field) -> bool {
+    for attr in &field.attrs {
+        if attr.tts.to_string() == "( skip_hashing )" {
+            return true;
+        }
+    }
+    false
+}
+
+/// Implements `ssz::Encodable` for some `struct`.
+///
+/// Fields are encoded in the order they are defined.
+#[proc_macro_derive(CachedTreeHashSubTree, attributes(tree_hash))]
+pub fn subtree_derive(input: TokenStream) -> TokenStream {
+    let item = parse_macro_input!(input as DeriveInput);
+
+    let name = &item.ident;
+
+    let struct_data = match &item.data {
+        syn::Data::Struct(s) => s,
+        _ => panic!("tree_hash_derive only supports structs."),
+    };
+
+    let idents_a = get_hashable_named_field_idents(&struct_data);
+    let idents_b = idents_a.clone();
+    let idents_c = idents_a.clone();
+    let idents_d = idents_a.clone();
+
+    let output = quote! {
+        impl tree_hash::CachedTreeHashSubTree<#name> for #name {
+            fn item_type() -> tree_hash::ItemType {
+                tree_hash::ItemType::Composite
+            }
+
+            fn new_cache(&self) -> Result<tree_hash::TreeHashCache, tree_hash::Error> {
+                let tree = tree_hash::TreeHashCache::from_leaves_and_subtrees(
+                    self,
+                    vec![
+                        #(
+                            self.#idents_a.new_cache()?,
+                        )*
+                    ],
+                )?;
+
+                Ok(tree)
+            }
+
+            fn btree_overlay(&self, chunk_offset: usize) -> Result<tree_hash::BTreeOverlay, tree_hash::Error> {
+                let mut lengths = vec![];
+
+                #(
+                    lengths.push(tree_hash::BTreeOverlay::new(&self.#idents_b, 0)?.total_nodes());
+                )*
+
+                tree_hash::BTreeOverlay::from_lengths(chunk_offset, lengths)
+            }
+
+            fn packed_encoding(&self) -> Result<Vec<u8>, tree_hash::Error> {
+                Err(tree_hash::Error::ShouldNeverBePacked(Self::item_type()))
+            }
+
+            fn packing_factor() -> usize {
+                1
+            }
+
+            fn update_cache(
+                &self,
+                other: &Self,
+                cache: &mut tree_hash::TreeHashCache,
+                chunk: usize,
+            ) -> Result<usize, tree_hash::Error> {
+                let offset_handler = tree_hash::BTreeOverlay::new(self, chunk)?;
+
+                // Skip past the internal nodes and update any changed leaf nodes.
+                {
+                    let chunk = offset_handler.first_leaf_node()?;
+                    #(
+                        let chunk = self.#idents_c.update_cache(&other.#idents_d, cache, chunk)?;
+                    )*
+                }
+
+                for (&parent, children) in offset_handler.iter_internal_nodes().rev() {
+                    if cache.either_modified(children)? {
+                        cache.modify_chunk(parent, &cache.hash_children(children)?)?;
+                    }
+                }
+
+                Ok(offset_handler.next_node)
+            }
+        }
+    };
+    output.into()
+}
diff --git a/eth2/utils/tree_hash_derive/tests/tests.rs b/eth2/utils/tree_hash_derive/tests/tests.rs
new file mode 100644
index 000000000..a5ab112a2
--- /dev/null
+++ b/eth2/utils/tree_hash_derive/tests/tests.rs
@@ -0,0 +1,9 @@
+use tree_hash_derive::CachedTreeHashSubTree;
+
+#[derive(Clone, Debug, CachedTreeHashSubTree)]
+pub struct Inner {
+    pub a: u64,
+    pub b: u64,
+    pub c: u64,
+    pub d: u64,
+}

From 291146eeb4fea4bbe0aa3c6aa37eadd566d7e1d4 Mon Sep 17 00:00:00 2001
From: Mehdi Zerouali <zedt3ster@users.noreply.github.com>
Date: Mon, 15 Apr 2019 16:47:35 +1000
Subject: [PATCH 050/137] Update License to Apache 2.0

See Issue #326
---
 LICENSE | 476 ++++++++++++++++++++------------------------------------
 1 file changed, 169 insertions(+), 307 deletions(-)

diff --git a/LICENSE b/LICENSE
index d159169d1..98016b543 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,339 +1,201 @@
-                    GNU GENERAL PUBLIC LICENSE
-                       Version 2, June 1991
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
 
- Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
-                            Preamble
+   1. Definitions.
 
-  The licenses for most software are designed to take away your
-freedom to share and change it.  By contrast, the GNU General Public
-License is intended to guarantee your freedom to share and change free
-software--to make sure the software is free for all its users.  This
-General Public License applies to most of the Free Software
-Foundation's software and to any other program whose authors commit to
-using it.  (Some other Free Software Foundation software is covered by
-the GNU Lesser General Public License instead.)  You can apply it to
-your programs, too.
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
 
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-this service if you wish), that you receive source code or can get it
-if you want it, that you can change the software or use pieces of it
-in new free programs; and that you know you can do these things.
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
 
-  To protect your rights, we need to make restrictions that forbid
-anyone to deny you these rights or to ask you to surrender the rights.
-These restrictions translate to certain responsibilities for you if you
-distribute copies of the software, or if you modify it.
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
 
-  For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must give the recipients all the rights that
-you have.  You must make sure that they, too, receive or can get the
-source code.  And you must show them these terms so they know their
-rights.
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
 
-  We protect your rights with two steps: (1) copyright the software, and
-(2) offer you this license which gives you legal permission to copy,
-distribute and/or modify the software.
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
 
-  Also, for each author's protection and ours, we want to make certain
-that everyone understands that there is no warranty for this free
-software.  If the software is modified by someone else and passed on, we
-want its recipients to know that what they have is not the original, so
-that any problems introduced by others will not reflect on the original
-authors' reputations.
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
 
-  Finally, any free program is threatened constantly by software
-patents.  We wish to avoid the danger that redistributors of a free
-program will individually obtain patent licenses, in effect making the
-program proprietary.  To prevent this, we have made it clear that any
-patent must be licensed for everyone's free use or not licensed at all.
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
 
-  The precise terms and conditions for copying, distribution and
-modification follow.
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
 
-                    GNU GENERAL PUBLIC LICENSE
-   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
 
-  0. This License applies to any program or other work which contains
-a notice placed by the copyright holder saying it may be distributed
-under the terms of this General Public License.  The "Program", below,
-refers to any such program or work, and a "work based on the Program"
-means either the Program or any derivative work under copyright law:
-that is to say, a work containing the Program or a portion of it,
-either verbatim or with modifications and/or translated into another
-language.  (Hereinafter, translation is included without limitation in
-the term "modification".)  Each licensee is addressed as "you".
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
 
-Activities other than copying, distribution and modification are not
-covered by this License; they are outside its scope.  The act of
-running the Program is not restricted, and the output from the Program
-is covered only if its contents constitute a work based on the
-Program (independent of having been made by running the Program).
-Whether that is true depends on what the Program does.
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
 
-  1. You may copy and distribute verbatim copies of the Program's
-source code as you receive it, in any medium, provided that you
-conspicuously and appropriately publish on each copy an appropriate
-copyright notice and disclaimer of warranty; keep intact all the
-notices that refer to this License and to the absence of any warranty;
-and give any other recipients of the Program a copy of this License
-along with the Program.
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
 
-You may charge a fee for the physical act of transferring a copy, and
-you may at your option offer warranty protection in exchange for a fee.
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
 
-  2. You may modify your copy or copies of the Program or any portion
-of it, thus forming a work based on the Program, and copy and
-distribute such modifications or work under the terms of Section 1
-above, provided that you also meet all of these conditions:
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
 
-    a) You must cause the modified files to carry prominent notices
-    stating that you changed the files and the date of any change.
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
 
-    b) You must cause any work that you distribute or publish, that in
-    whole or in part contains or is derived from the Program or any
-    part thereof, to be licensed as a whole at no charge to all third
-    parties under the terms of this License.
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
 
-    c) If the modified program normally reads commands interactively
-    when run, you must cause it, when started running for such
-    interactive use in the most ordinary way, to print or display an
-    announcement including an appropriate copyright notice and a
-    notice that there is no warranty (or else, saying that you provide
-    a warranty) and that users may redistribute the program under
-    these conditions, and telling the user how to view a copy of this
-    License.  (Exception: if the Program itself is interactive but
-    does not normally print such an announcement, your work based on
-    the Program is not required to print an announcement.)
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
 
-These requirements apply to the modified work as a whole.  If
-identifiable sections of that work are not derived from the Program,
-and can be reasonably considered independent and separate works in
-themselves, then this License, and its terms, do not apply to those
-sections when you distribute them as separate works.  But when you
-distribute the same sections as part of a whole which is a work based
-on the Program, the distribution of the whole must be on the terms of
-this License, whose permissions for other licensees extend to the
-entire whole, and thus to each and every part regardless of who wrote it.
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
 
-Thus, it is not the intent of this section to claim rights or contest
-your rights to work written entirely by you; rather, the intent is to
-exercise the right to control the distribution of derivative or
-collective works based on the Program.
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
 
-In addition, mere aggregation of another work not based on the Program
-with the Program (or with a work based on the Program) on a volume of
-a storage or distribution medium does not bring the other work under
-the scope of this License.
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
 
-  3. You may copy and distribute the Program (or a work based on it,
-under Section 2) in object code or executable form under the terms of
-Sections 1 and 2 above provided that you also do one of the following:
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
 
-    a) Accompany it with the complete corresponding machine-readable
-    source code, which must be distributed under the terms of Sections
-    1 and 2 above on a medium customarily used for software interchange; or,
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
 
-    b) Accompany it with a written offer, valid for at least three
-    years, to give any third party, for a charge no more than your
-    cost of physically performing source distribution, a complete
-    machine-readable copy of the corresponding source code, to be
-    distributed under the terms of Sections 1 and 2 above on a medium
-    customarily used for software interchange; or,
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
 
-    c) Accompany it with the information you received as to the offer
-    to distribute corresponding source code.  (This alternative is
-    allowed only for noncommercial distribution and only if you
-    received the program in object code or executable form with such
-    an offer, in accord with Subsection b above.)
+   END OF TERMS AND CONDITIONS
 
-The source code for a work means the preferred form of the work for
-making modifications to it.  For an executable work, complete source
-code means all the source code for all modules it contains, plus any
-associated interface definition files, plus the scripts used to
-control compilation and installation of the executable.  However, as a
-special exception, the source code distributed need not include
-anything that is normally distributed (in either source or binary
-form) with the major components (compiler, kernel, and so on) of the
-operating system on which the executable runs, unless that component
-itself accompanies the executable.
+   APPENDIX: How to apply the Apache License to your work.
 
-If distribution of executable or object code is made by offering
-access to copy from a designated place, then offering equivalent
-access to copy the source code from the same place counts as
-distribution of the source code, even though third parties are not
-compelled to copy the source along with the object code.
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
 
-  4. You may not copy, modify, sublicense, or distribute the Program
-except as expressly provided under this License.  Any attempt
-otherwise to copy, modify, sublicense or distribute the Program is
-void, and will automatically terminate your rights under this License.
-However, parties who have received copies, or rights, from you under
-this License will not have their licenses terminated so long as such
-parties remain in full compliance.
+   Copyright 2018 Sigma Prime Pty Ltd
 
-  5. You are not required to accept this License, since you have not
-signed it.  However, nothing else grants you permission to modify or
-distribute the Program or its derivative works.  These actions are
-prohibited by law if you do not accept this License.  Therefore, by
-modifying or distributing the Program (or any work based on the
-Program), you indicate your acceptance of this License to do so, and
-all its terms and conditions for copying, distributing or modifying
-the Program or works based on it.
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
 
-  6. Each time you redistribute the Program (or any work based on the
-Program), the recipient automatically receives a license from the
-original licensor to copy, distribute or modify the Program subject to
-these terms and conditions.  You may not impose any further
-restrictions on the recipients' exercise of the rights granted herein.
-You are not responsible for enforcing compliance by third parties to
-this License.
+       http://www.apache.org/licenses/LICENSE-2.0
 
-  7. If, as a consequence of a court judgment or allegation of patent
-infringement or for any other reason (not limited to patent issues),
-conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot
-distribute so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you
-may not distribute the Program at all.  For example, if a patent
-license would not permit royalty-free redistribution of the Program by
-all those who receive copies directly or indirectly through you, then
-the only way you could satisfy both it and this License would be to
-refrain entirely from distribution of the Program.
-
-If any portion of this section is held invalid or unenforceable under
-any particular circumstance, the balance of the section is intended to
-apply and the section as a whole is intended to apply in other
-circumstances.
-
-It is not the purpose of this section to induce you to infringe any
-patents or other property right claims or to contest validity of any
-such claims; this section has the sole purpose of protecting the
-integrity of the free software distribution system, which is
-implemented by public license practices.  Many people have made
-generous contributions to the wide range of software distributed
-through that system in reliance on consistent application of that
-system; it is up to the author/donor to decide if he or she is willing
-to distribute software through any other system and a licensee cannot
-impose that choice.
-
-This section is intended to make thoroughly clear what is believed to
-be a consequence of the rest of this License.
-
-  8. If the distribution and/or use of the Program is restricted in
-certain countries either by patents or by copyrighted interfaces, the
-original copyright holder who places the Program under this License
-may add an explicit geographical distribution limitation excluding
-those countries, so that distribution is permitted only in or among
-countries not thus excluded.  In such case, this License incorporates
-the limitation as if written in the body of this License.
-
-  9. The Free Software Foundation may publish revised and/or new versions
-of the General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-Each version is given a distinguishing version number.  If the Program
-specifies a version number of this License which applies to it and "any
-later version", you have the option of following the terms and conditions
-either of that version or of any later version published by the Free
-Software Foundation.  If the Program does not specify a version number of
-this License, you may choose any version ever published by the Free Software
-Foundation.
-
-  10. If you wish to incorporate parts of the Program into other free
-programs whose distribution conditions are different, write to the author
-to ask for permission.  For software which is copyrighted by the Free
-Software Foundation, write to the Free Software Foundation; we sometimes
-make exceptions for this.  Our decision will be guided by the two goals
-of preserving the free status of all derivatives of our free software and
-of promoting the sharing and reuse of software generally.
-
-                            NO WARRANTY
-
-  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
-FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
-OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
-PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
-OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
-TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
-PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
-REPAIR OR CORRECTION.
-
-  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
-REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
-INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
-OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
-TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
-YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
-PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGES.
-
-                     END OF TERMS AND CONDITIONS
-
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-convey the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-Also add information on how to contact you by electronic and paper mail.
-
-If the program is interactive, make it output a short notice like this
-when it starts in an interactive mode:
-
-    Gnomovision version 69, Copyright (C) year name of author
-    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, the commands you use may
-be called something other than `show w' and `show c'; they could even be
-mouse-clicks or menu items--whatever suits your program.
-
-You should also get your employer (if you work as a programmer) or your
-school, if any, to sign a "copyright disclaimer" for the program, if
-necessary.  Here is a sample; alter the names:
-
-  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
-  `Gnomovision' (which makes passes at compilers) written by James Hacker.
-
-  <signature of Ty Coon>, 1 April 1989
-  Ty Coon, President of Vice
-
-This General Public License does not permit incorporating your program into
-proprietary programs.  If your program is a subroutine library, you may
-consider it more useful to permit linking proprietary applications with the
-library.  If this is what you want to do, use the GNU Lesser General
-Public License instead of this License.
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

From 8cb6368fe6ae62d6263a72025bd2c97628ffa678 Mon Sep 17 00:00:00 2001
From: Sean Yu <seansu4you87@gmail.com>
Date: Wed, 27 Mar 2019 14:57:50 -0700
Subject: [PATCH 051/137] Adding a #[signed_root(skip_hashing)] macro

Lets the user annotate fields of a struct to skip for signed root
hashing.

Also added tests in a `eth2/utils/tests` crate, so that we can test
whether these derived macros work as intended.
---
 eth2/types/src/attestation.rs               |  1 +
 eth2/types/src/beacon_block.rs              |  1 +
 eth2/types/src/beacon_block_header.rs       |  1 +
 eth2/types/src/deposit_input.rs             |  1 +
 eth2/types/src/slashable_attestation.rs     |  1 +
 eth2/types/src/transfer.rs                  |  1 +
 eth2/types/src/voluntary_exit.rs            |  1 +
 eth2/utils/ssz_derive/src/lib.rs            | 44 ++++++----
 eth2/utils/ssz_derive/tests/test_derives.rs | 94 +++++++++++++++++++++
 9 files changed, 129 insertions(+), 16 deletions(-)
 create mode 100644 eth2/utils/ssz_derive/tests/test_derives.rs

diff --git a/eth2/types/src/attestation.rs b/eth2/types/src/attestation.rs
index dabccfde7..a8eeea909 100644
--- a/eth2/types/src/attestation.rs
+++ b/eth2/types/src/attestation.rs
@@ -25,6 +25,7 @@ pub struct Attestation {
     pub aggregation_bitfield: Bitfield,
     pub data: AttestationData,
     pub custody_bitfield: Bitfield,
+    #[signed_root(skip_hashing)]
     pub aggregate_signature: AggregateSignature,
 }
 
diff --git a/eth2/types/src/beacon_block.rs b/eth2/types/src/beacon_block.rs
index 6a3f1a354..77c1620f3 100644
--- a/eth2/types/src/beacon_block.rs
+++ b/eth2/types/src/beacon_block.rs
@@ -27,6 +27,7 @@ pub struct BeaconBlock {
     pub previous_block_root: Hash256,
     pub state_root: Hash256,
     pub body: BeaconBlockBody,
+    #[signed_root(skip_hashing)]
     pub signature: Signature,
 }
 
diff --git a/eth2/types/src/beacon_block_header.rs b/eth2/types/src/beacon_block_header.rs
index f4bee27e1..090d0a965 100644
--- a/eth2/types/src/beacon_block_header.rs
+++ b/eth2/types/src/beacon_block_header.rs
@@ -27,6 +27,7 @@ pub struct BeaconBlockHeader {
     pub previous_block_root: Hash256,
     pub state_root: Hash256,
     pub block_body_root: Hash256,
+    #[signed_root(skip_hashing)]
     pub signature: Signature,
 }
 
diff --git a/eth2/types/src/deposit_input.rs b/eth2/types/src/deposit_input.rs
index 3f8a6177a..380528dc0 100644
--- a/eth2/types/src/deposit_input.rs
+++ b/eth2/types/src/deposit_input.rs
@@ -25,6 +25,7 @@ use test_random_derive::TestRandom;
 pub struct DepositInput {
     pub pubkey: PublicKey,
     pub withdrawal_credentials: Hash256,
+    #[signed_root(skip_hashing)]
     pub proof_of_possession: Signature,
 }
 
diff --git a/eth2/types/src/slashable_attestation.rs b/eth2/types/src/slashable_attestation.rs
index 05c41a72b..e557285b8 100644
--- a/eth2/types/src/slashable_attestation.rs
+++ b/eth2/types/src/slashable_attestation.rs
@@ -27,6 +27,7 @@ pub struct SlashableAttestation {
     pub validator_indices: Vec<u64>,
     pub data: AttestationData,
     pub custody_bitfield: Bitfield,
+    #[signed_root(skip_hashing)]
     pub aggregate_signature: AggregateSignature,
 }
 
diff --git a/eth2/types/src/transfer.rs b/eth2/types/src/transfer.rs
index 4b10ce1ca..f291190b2 100644
--- a/eth2/types/src/transfer.rs
+++ b/eth2/types/src/transfer.rs
@@ -32,6 +32,7 @@ pub struct Transfer {
     pub slot: Slot,
     pub pubkey: PublicKey,
     #[derivative(Hash = "ignore")]
+    #[signed_root(skip_hashing)]
     pub signature: Signature,
 }
 
diff --git a/eth2/types/src/voluntary_exit.rs b/eth2/types/src/voluntary_exit.rs
index f64f950cb..0cdc63149 100644
--- a/eth2/types/src/voluntary_exit.rs
+++ b/eth2/types/src/voluntary_exit.rs
@@ -24,6 +24,7 @@ use test_random_derive::TestRandom;
 pub struct VoluntaryExit {
     pub epoch: Epoch,
     pub validator_index: u64,
+    #[signed_root(skip_hashing)]
     pub signature: Signature,
 }
 
diff --git a/eth2/utils/ssz_derive/src/lib.rs b/eth2/utils/ssz_derive/src/lib.rs
index 9ba1de416..ce2538785 100644
--- a/eth2/utils/ssz_derive/src/lib.rs
+++ b/eth2/utils/ssz_derive/src/lib.rs
@@ -38,7 +38,7 @@
 extern crate proc_macro;
 
 use proc_macro::TokenStream;
-use quote::quote;
+use quote::{quote, ToTokens};
 use syn::{parse_macro_input, DeriveInput};
 
 /// Returns a Vec of `syn::Ident` for each named field in the struct.
@@ -218,7 +218,7 @@ fn get_tree_hashable_named_field_idents<'a>(
 /// The field attribute is: `#[tree_hash(skip_hashing)]`
 fn should_skip_tree_hash(field: &syn::Field) -> bool {
     for attr in &field.attrs {
-        if attr.tts.to_string() == "( skip_hashing )" {
+        if attr.into_token_stream().to_string() == "# [ tree_hash ( skip_hashing ) ]" {
             return true;
         }
     }
@@ -289,7 +289,7 @@ fn final_type_ident(field: &syn::Field) -> &syn::Ident {
 /// If you can think of a better way to do this, please make an issue!
 ///
 /// Fields are processed in the order they are defined.
-#[proc_macro_derive(SignedRoot)]
+#[proc_macro_derive(SignedRoot, attributes(signed_root))]
 pub fn ssz_signed_root_derive(input: TokenStream) -> TokenStream {
     let item = parse_macro_input!(input as DeriveInput);
 
@@ -302,19 +302,7 @@ pub fn ssz_signed_root_derive(input: TokenStream) -> TokenStream {
 
     let mut field_idents: Vec<&syn::Ident> = vec![];
 
-    for field in struct_data.fields.iter() {
-        let final_type_ident = final_type_ident(&field);
-
-        if type_ident_is_signature(final_type_ident) {
-            break;
-        } else {
-            let ident = field
-                .ident
-                .as_ref()
-                .expect("ssz_derive only supports named_struct fields.");
-            field_idents.push(ident);
-        }
-    }
+    let field_idents = get_signed_root_named_field_idents(&struct_data);
 
     let output = quote! {
         impl ssz::SignedRoot for #name {
@@ -330,3 +318,27 @@ pub fn ssz_signed_root_derive(input: TokenStream) -> TokenStream {
     };
     output.into()
 }
+
+fn get_signed_root_named_field_idents(struct_data: &syn::DataStruct) -> Vec<&syn::Ident> {
+    struct_data
+        .fields
+        .iter()
+        .filter_map(|f| {
+            if should_skip_signed_root(&f) {
+                None
+            } else {
+                Some(match &f.ident {
+                    Some(ref ident) => ident,
+                    _ => panic!("ssz_derive only supports named struct fields"),
+                })
+            }
+        })
+        .collect()
+}
+
+fn should_skip_signed_root(field: &syn::Field) -> bool {
+    field
+        .attrs
+        .iter()
+        .any(|attr| attr.into_token_stream().to_string() == "# [ signed_root ( skip_hashing ) ]")
+}
diff --git a/eth2/utils/ssz_derive/tests/test_derives.rs b/eth2/utils/ssz_derive/tests/test_derives.rs
new file mode 100644
index 000000000..e025dc3a5
--- /dev/null
+++ b/eth2/utils/ssz_derive/tests/test_derives.rs
@@ -0,0 +1,94 @@
+use ssz::{SignedRoot, TreeHash};
+use ssz_derive::{SignedRoot, TreeHash};
+
+#[derive(TreeHash, SignedRoot)]
+struct CryptoKitties {
+    best_kitty: u64,
+    worst_kitty: u8,
+    kitties: Vec<u32>,
+}
+
+impl CryptoKitties {
+    fn new() -> Self {
+        CryptoKitties {
+            best_kitty: 9999,
+            worst_kitty: 1,
+            kitties: vec![2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43],
+        }
+    }
+
+    fn hash(&self) -> Vec<u8> {
+        let mut list: Vec<Vec<u8>> = Vec::new();
+        list.push(self.best_kitty.hash_tree_root());
+        list.push(self.worst_kitty.hash_tree_root());
+        list.push(self.kitties.hash_tree_root());
+        ssz::merkle_hash(&mut list)
+    }
+}
+
+#[test]
+fn test_cryptokitties_hash() {
+    let kitties = CryptoKitties::new();
+    let expected_hash = vec![
+        201, 9, 139, 14, 24, 247, 21, 55, 132, 211, 51, 125, 183, 186, 177, 33, 147, 210, 42, 108,
+        174, 162, 221, 227, 157, 179, 15, 7, 97, 239, 82, 220,
+    ];
+    assert_eq!(kitties.hash(), expected_hash);
+}
+
+#[test]
+fn test_simple_tree_hash_derive() {
+    let kitties = CryptoKitties::new();
+    assert_eq!(kitties.hash_tree_root(), kitties.hash());
+}
+
+#[test]
+fn test_simple_signed_root_derive() {
+    let kitties = CryptoKitties::new();
+    assert_eq!(kitties.signed_root(), kitties.hash());
+}
+
+#[derive(TreeHash, SignedRoot)]
+struct Casper {
+    friendly: bool,
+    #[tree_hash(skip_hashing)]
+    friends: Vec<u32>,
+    #[signed_root(skip_hashing)]
+    dead: bool,
+}
+
+impl Casper {
+    fn new() -> Self {
+        Casper {
+            friendly: true,
+            friends: vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+            dead: true,
+        }
+    }
+
+    fn expected_signed_hash(&self) -> Vec<u8> {
+        let mut list = Vec::new();
+        list.push(self.friendly.hash_tree_root());
+        list.push(self.friends.hash_tree_root());
+        ssz::merkle_hash(&mut list)
+    }
+
+    fn expected_tree_hash(&self) -> Vec<u8> {
+        let mut list = Vec::new();
+        list.push(self.friendly.hash_tree_root());
+        list.push(self.dead.hash_tree_root());
+        ssz::merkle_hash(&mut list)
+    }
+}
+
+#[test]
+fn test_annotated_tree_hash_derive() {
+    let casper = Casper::new();
+    assert_eq!(casper.hash_tree_root(), casper.expected_tree_hash());
+}
+
+#[test]
+fn test_annotated_signed_root_derive() {
+    let casper = Casper::new();
+    assert_eq!(casper.signed_root(), casper.expected_signed_hash());
+}

From 93f3fc858d97791563ab52eb1a1dea78b8f1ec46 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Tue, 16 Apr 2019 09:14:33 +1000
Subject: [PATCH 052/137] Add uncached tree hashing

---
 eth2/utils/tree_hash/src/cached_tree_hash.rs  | 125 ++++++++++++++++
 .../{ => cached_tree_hash}/btree_overlay.rs   |   0
 .../src/{ => cached_tree_hash}/impls.rs       |   0
 .../src/{ => cached_tree_hash}/impls/vec.rs   |   0
 .../src/{ => cached_tree_hash}/resize.rs      |   0
 eth2/utils/tree_hash/src/lib.rs               | 134 +-----------------
 .../utils/tree_hash/src/standard_tree_hash.rs | 114 +++++++++++++++
 eth2/utils/tree_hash/tests/tests.rs           |  25 ++++
 8 files changed, 268 insertions(+), 130 deletions(-)
 rename eth2/utils/tree_hash/src/{ => cached_tree_hash}/btree_overlay.rs (100%)
 rename eth2/utils/tree_hash/src/{ => cached_tree_hash}/impls.rs (100%)
 rename eth2/utils/tree_hash/src/{ => cached_tree_hash}/impls/vec.rs (100%)
 rename eth2/utils/tree_hash/src/{ => cached_tree_hash}/resize.rs (100%)
 create mode 100644 eth2/utils/tree_hash/src/standard_tree_hash.rs

diff --git a/eth2/utils/tree_hash/src/cached_tree_hash.rs b/eth2/utils/tree_hash/src/cached_tree_hash.rs
index 048d4bab5..fc12cfbba 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash.rs
@@ -1,4 +1,129 @@
 use super::*;
+use hashing::hash;
+use int_to_bytes::int_to_bytes32;
+use std::ops::Range;
+
+pub mod btree_overlay;
+pub mod impls;
+pub mod resize;
+
+pub use btree_overlay::BTreeOverlay;
+
+#[derive(Debug, PartialEq, Clone)]
+pub enum Error {
+    ShouldNotProduceBTreeOverlay,
+    NoFirstNode,
+    NoBytesForRoot,
+    UnableToObtainSlices,
+    UnableToGrowMerkleTree,
+    UnableToShrinkMerkleTree,
+    ShouldNeverBePacked(ItemType),
+    BytesAreNotEvenChunks(usize),
+    NoModifiedFieldForChunk(usize),
+    NoBytesForChunk(usize),
+}
+
+pub trait CachedTreeHash<T>: CachedTreeHashSubTree<T> + Sized {
+    fn update_internal_tree_hash_cache(self, old: T) -> Result<(Self, Self), Error>;
+
+    fn cached_tree_hash_root(&self) -> Option<Vec<u8>>;
+
+    fn clone_without_tree_hash_cache(&self) -> Self;
+}
+
+pub trait CachedTreeHashSubTree<Item> {
+    fn item_type() -> ItemType;
+
+    fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error>;
+
+    fn packed_encoding(&self) -> Result<Vec<u8>, Error>;
+
+    fn packing_factor() -> usize;
+
+    fn new_cache(&self) -> Result<TreeHashCache, Error>;
+
+    fn update_cache(
+        &self,
+        other: &Item,
+        cache: &mut TreeHashCache,
+        chunk: usize,
+    ) -> Result<usize, Error>;
+}
+
+fn children(parent: usize) -> (usize, usize) {
+    ((2 * parent + 1), (2 * parent + 2))
+}
+
+fn node_range_to_byte_range(node_range: &Range<usize>) -> Range<usize> {
+    node_range.start * HASHSIZE..node_range.end * HASHSIZE
+}
+
+/// Split `values` into a power-of-two, identical-length chunks (padding with `0`) and merkleize
+/// them, returning the entire merkle tree.
+///
+/// The root hash is `merkleize(values)[0..BYTES_PER_CHUNK]`.
+pub fn merkleize(values: Vec<u8>) -> Vec<u8> {
+    let values = sanitise_bytes(values);
+
+    let leaves = values.len() / HASHSIZE;
+
+    if leaves == 0 {
+        panic!("No full leaves");
+    }
+
+    if !leaves.is_power_of_two() {
+        panic!("leaves is not power of two");
+    }
+
+    let mut o: Vec<u8> = vec![0; (num_nodes(leaves) - leaves) * HASHSIZE];
+    o.append(&mut values.to_vec());
+
+    let mut i = o.len();
+    let mut j = o.len() - values.len();
+
+    while i >= MERKLE_HASH_CHUNCK {
+        i -= MERKLE_HASH_CHUNCK;
+        let hash = hash(&o[i..i + MERKLE_HASH_CHUNCK]);
+
+        j -= HASHSIZE;
+        o[j..j + HASHSIZE].copy_from_slice(&hash);
+    }
+
+    o
+}
+
+pub fn sanitise_bytes(mut bytes: Vec<u8>) -> Vec<u8> {
+    let present_leaves = num_unsanitized_leaves(bytes.len());
+    let required_leaves = present_leaves.next_power_of_two();
+
+    if (present_leaves != required_leaves) | last_leaf_needs_padding(bytes.len()) {
+        bytes.resize(num_bytes(required_leaves), 0);
+    }
+
+    bytes
+}
+
+fn pad_for_leaf_count(num_leaves: usize, bytes: &mut Vec<u8>) {
+    let required_leaves = num_leaves.next_power_of_two();
+
+    bytes.resize(
+        bytes.len() + (required_leaves - num_leaves) * BYTES_PER_CHUNK,
+        0,
+    );
+}
+
+fn last_leaf_needs_padding(num_bytes: usize) -> bool {
+    num_bytes % HASHSIZE != 0
+}
+
+/// Rounds up
+fn num_unsanitized_leaves(num_bytes: usize) -> usize {
+    (num_bytes + HASHSIZE - 1) / HASHSIZE
+}
+
+fn num_bytes(num_leaves: usize) -> usize {
+    num_leaves * HASHSIZE
+}
 
 #[derive(Debug, PartialEq, Clone)]
 pub struct TreeHashCache {
diff --git a/eth2/utils/tree_hash/src/btree_overlay.rs b/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
similarity index 100%
rename from eth2/utils/tree_hash/src/btree_overlay.rs
rename to eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
diff --git a/eth2/utils/tree_hash/src/impls.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
similarity index 100%
rename from eth2/utils/tree_hash/src/impls.rs
rename to eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
diff --git a/eth2/utils/tree_hash/src/impls/vec.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
similarity index 100%
rename from eth2/utils/tree_hash/src/impls/vec.rs
rename to eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
diff --git a/eth2/utils/tree_hash/src/resize.rs b/eth2/utils/tree_hash/src/cached_tree_hash/resize.rs
similarity index 100%
rename from eth2/utils/tree_hash/src/resize.rs
rename to eth2/utils/tree_hash/src/cached_tree_hash/resize.rs
diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs
index 5ec2b0283..4e5302bca 100644
--- a/eth2/utils/tree_hash/src/lib.rs
+++ b/eth2/utils/tree_hash/src/lib.rs
@@ -1,33 +1,10 @@
-use hashing::hash;
-use int_to_bytes::int_to_bytes32;
-use std::ops::Range;
-
-mod btree_overlay;
-mod cached_tree_hash;
-mod impls;
-mod resize;
-
-pub use btree_overlay::BTreeOverlay;
-pub use cached_tree_hash::TreeHashCache;
+pub mod cached_tree_hash;
+pub mod standard_tree_hash;
 
 pub const BYTES_PER_CHUNK: usize = 32;
 pub const HASHSIZE: usize = 32;
 pub const MERKLE_HASH_CHUNCK: usize = 2 * BYTES_PER_CHUNK;
 
-#[derive(Debug, PartialEq, Clone)]
-pub enum Error {
-    ShouldNotProduceBTreeOverlay,
-    NoFirstNode,
-    NoBytesForRoot,
-    UnableToObtainSlices,
-    UnableToGrowMerkleTree,
-    UnableToShrinkMerkleTree,
-    ShouldNeverBePacked(ItemType),
-    BytesAreNotEvenChunks(usize),
-    NoModifiedFieldForChunk(usize),
-    NoBytesForChunk(usize),
-}
-
 #[derive(Debug, PartialEq, Clone)]
 pub enum ItemType {
     Basic,
@@ -35,114 +12,11 @@ pub enum ItemType {
     Composite,
 }
 
-pub trait CachedTreeHash<T>: CachedTreeHashSubTree<T> + Sized {
-    fn update_internal_tree_hash_cache(self, old: T) -> Result<(Self, Self), Error>;
-
-    fn cached_tree_hash_root(&self) -> Option<Vec<u8>>;
-
-    fn clone_without_tree_hash_cache(&self) -> Self;
-}
-
-pub trait CachedTreeHashSubTree<Item> {
-    fn item_type() -> ItemType;
-
-    fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error>;
-
-    fn packed_encoding(&self) -> Result<Vec<u8>, Error>;
-
-    fn packing_factor() -> usize;
-
-    fn new_cache(&self) -> Result<TreeHashCache, Error>;
-
-    fn update_cache(
-        &self,
-        other: &Item,
-        cache: &mut TreeHashCache,
-        chunk: usize,
-    ) -> Result<usize, Error>;
-}
-
-fn children(parent: usize) -> (usize, usize) {
-    ((2 * parent + 1), (2 * parent + 2))
-}
-
-fn num_nodes(num_leaves: usize) -> usize {
-    2 * num_leaves - 1
-}
-
-fn node_range_to_byte_range(node_range: &Range<usize>) -> Range<usize> {
-    node_range.start * HASHSIZE..node_range.end * HASHSIZE
-}
-
-/// Split `values` into a power-of-two, identical-length chunks (padding with `0`) and merkleize
-/// them, returning the entire merkle tree.
-///
-/// The root hash is `merkleize(values)[0..BYTES_PER_CHUNK]`.
-pub fn merkleize(values: Vec<u8>) -> Vec<u8> {
-    let values = sanitise_bytes(values);
-
-    let leaves = values.len() / HASHSIZE;
-
-    if leaves == 0 {
-        panic!("No full leaves");
-    }
-
-    if !leaves.is_power_of_two() {
-        panic!("leaves is not power of two");
-    }
-
-    let mut o: Vec<u8> = vec![0; (num_nodes(leaves) - leaves) * HASHSIZE];
-    o.append(&mut values.to_vec());
-
-    let mut i = o.len();
-    let mut j = o.len() - values.len();
-
-    while i >= MERKLE_HASH_CHUNCK {
-        i -= MERKLE_HASH_CHUNCK;
-        let hash = hash(&o[i..i + MERKLE_HASH_CHUNCK]);
-
-        j -= HASHSIZE;
-        o[j..j + HASHSIZE].copy_from_slice(&hash);
-    }
-
-    o
-}
-
-pub fn sanitise_bytes(mut bytes: Vec<u8>) -> Vec<u8> {
-    let present_leaves = num_unsanitized_leaves(bytes.len());
-    let required_leaves = present_leaves.next_power_of_two();
-
-    if (present_leaves != required_leaves) | last_leaf_needs_padding(bytes.len()) {
-        bytes.resize(num_bytes(required_leaves), 0);
-    }
-
-    bytes
-}
-
-fn pad_for_leaf_count(num_leaves: usize, bytes: &mut Vec<u8>) {
-    let required_leaves = num_leaves.next_power_of_two();
-
-    bytes.resize(
-        bytes.len() + (required_leaves - num_leaves) * BYTES_PER_CHUNK,
-        0,
-    );
-}
-
-fn last_leaf_needs_padding(num_bytes: usize) -> bool {
-    num_bytes % HASHSIZE != 0
-}
-
-/// Rounds up
-fn num_unsanitized_leaves(num_bytes: usize) -> usize {
-    (num_bytes + HASHSIZE - 1) / HASHSIZE
-}
-
-/// Rounds up
 fn num_sanitized_leaves(num_bytes: usize) -> usize {
     let leaves = (num_bytes + HASHSIZE - 1) / HASHSIZE;
     leaves.next_power_of_two()
 }
 
-fn num_bytes(num_leaves: usize) -> usize {
-    num_leaves * HASHSIZE
+fn num_nodes(num_leaves: usize) -> usize {
+    2 * num_leaves - 1
 }
diff --git a/eth2/utils/tree_hash/src/standard_tree_hash.rs b/eth2/utils/tree_hash/src/standard_tree_hash.rs
new file mode 100644
index 000000000..c8119a790
--- /dev/null
+++ b/eth2/utils/tree_hash/src/standard_tree_hash.rs
@@ -0,0 +1,114 @@
+use super::*;
+use hashing::hash;
+use int_to_bytes::int_to_bytes32;
+use ssz::ssz_encode;
+
+pub trait TreeHash {
+    fn tree_hash_item_type() -> ItemType;
+
+    fn tree_hash_packed_encoding(&self) -> Vec<u8>;
+
+    fn hash_tree_root(&self) -> Vec<u8>;
+}
+
+impl TreeHash for u64 {
+    fn tree_hash_item_type() -> ItemType {
+        ItemType::Basic
+    }
+
+    fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+        ssz_encode(self)
+    }
+
+    fn hash_tree_root(&self) -> Vec<u8> {
+        int_to_bytes32(*self)
+    }
+}
+
+impl<T> TreeHash for Vec<T>
+where
+    T: TreeHash,
+{
+    fn tree_hash_item_type() -> ItemType {
+        ItemType::List
+    }
+
+    fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+        unreachable!("List should never be packed.")
+    }
+
+    fn hash_tree_root(&self) -> Vec<u8> {
+        let leaves = match T::tree_hash_item_type() {
+            ItemType::Basic => {
+                let mut leaves = vec![];
+
+                for item in self {
+                    leaves.append(&mut item.tree_hash_packed_encoding());
+                }
+
+                leaves
+            }
+            ItemType::Composite | ItemType::List => {
+                let mut leaves = Vec::with_capacity(self.len() * HASHSIZE);
+
+                for item in self {
+                    leaves.append(&mut item.hash_tree_root())
+                }
+
+                leaves
+            }
+        };
+
+        // Mix in the length
+        let mut root_and_len = Vec::with_capacity(HASHSIZE * 2);
+        root_and_len.append(&mut efficient_merkleize(&leaves)[0..32].to_vec());
+        root_and_len.append(&mut int_to_bytes32(self.len() as u64));
+
+        hash(&root_and_len)
+    }
+}
+
+pub fn efficient_merkleize(bytes: &[u8]) -> Vec<u8> {
+    let leaves = num_sanitized_leaves(bytes.len());
+    let nodes = num_nodes(leaves);
+    let internal_nodes = nodes - leaves;
+
+    let num_bytes = internal_nodes * HASHSIZE + bytes.len();
+
+    let mut o: Vec<u8> = vec![0; internal_nodes * HASHSIZE];
+    o.append(&mut bytes.to_vec());
+
+    assert_eq!(o.len(), num_bytes);
+
+    let empty_chunk_hash = hash(&[0; MERKLE_HASH_CHUNCK]);
+
+    let mut i = nodes * HASHSIZE;
+    let mut j = internal_nodes * HASHSIZE;
+
+    while i >= MERKLE_HASH_CHUNCK {
+        i -= MERKLE_HASH_CHUNCK;
+
+        j -= HASHSIZE;
+        let hash = match o.get(i..i + MERKLE_HASH_CHUNCK) {
+            // All bytes are available, hash as ususal.
+            Some(slice) => hash(slice),
+            // Unable to get all the bytes.
+            None => {
+                match o.get(i..) {
+                    // Able to get some of the bytes, pad them out.
+                    Some(slice) => {
+                        let mut bytes = slice.to_vec();
+                        bytes.resize(MERKLE_HASH_CHUNCK, 0);
+                        hash(&bytes)
+                    }
+                    // Unable to get any bytes, use the empty-chunk hash.
+                    None => empty_chunk_hash.clone(),
+                }
+            }
+        };
+
+        o[j..j + HASHSIZE].copy_from_slice(&hash);
+    }
+
+    o
+}
diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
index ead6d8c00..d65192cd5 100644
--- a/eth2/utils/tree_hash/tests/tests.rs
+++ b/eth2/utils/tree_hash/tests/tests.rs
@@ -1,5 +1,7 @@
 use hashing::hash;
 use int_to_bytes::{int_to_bytes32, int_to_bytes8};
+use tree_hash::cached_tree_hash::*;
+use tree_hash::standard_tree_hash::*;
 use tree_hash::*;
 
 #[derive(Clone, Debug)]
@@ -131,6 +133,27 @@ pub struct Inner {
     pub d: u64,
 }
 
+impl TreeHash for Inner {
+    fn tree_hash_item_type() -> ItemType {
+        ItemType::Composite
+    }
+
+    fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+        unreachable!("Struct should never be packed.")
+    }
+
+    fn hash_tree_root(&self) -> Vec<u8> {
+        let mut leaves = Vec::with_capacity(4 * HASHSIZE);
+
+        leaves.append(&mut self.a.hash_tree_root());
+        leaves.append(&mut self.b.hash_tree_root());
+        leaves.append(&mut self.c.hash_tree_root());
+        leaves.append(&mut self.d.hash_tree_root());
+
+        efficient_merkleize(&leaves)[0..32].to_vec()
+    }
+}
+
 impl CachedTreeHashSubTree<Inner> for Inner {
     fn item_type() -> ItemType {
         ItemType::Composite
@@ -458,6 +481,7 @@ fn test_u64_vec_modifications(original: Vec<u64>, modified: Vec<u64>) {
     mix_in_length(&mut expected[0..HASHSIZE], modified.len());
 
     assert_eq!(expected, modified_cache);
+    assert_eq!(&expected[0..32], &modified.hash_tree_root()[..]);
 }
 
 #[test]
@@ -580,6 +604,7 @@ fn test_inner_vec_modifications(original: Vec<Inner>, modified: Vec<Inner>, refe
 
     // Compare the cached tree to the reference tree.
     assert_trees_eq(&expected, &modified_cache);
+    assert_eq!(&expected[0..32], &modified.hash_tree_root()[..]);
 }
 
 #[test]

From d311b48a9f35f4463939bbf50efbe65eea7a5261 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Tue, 16 Apr 2019 09:34:23 +1000
Subject: [PATCH 053/137] Unify tree hash methods

---
 eth2/utils/tree_hash/src/cached_tree_hash.rs  |  10 +-
 .../tree_hash/src/cached_tree_hash/impls.rs   |  12 --
 .../src/cached_tree_hash/impls/vec.rs         |  36 ++----
 eth2/utils/tree_hash/src/lib.rs               |   5 +-
 .../utils/tree_hash/src/standard_tree_hash.rs |  37 ++++---
 eth2/utils/tree_hash/tests/tests.rs           | 103 ++++++++++--------
 6 files changed, 101 insertions(+), 102 deletions(-)

diff --git a/eth2/utils/tree_hash/src/cached_tree_hash.rs b/eth2/utils/tree_hash/src/cached_tree_hash.rs
index fc12cfbba..43c0ba2fe 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash.rs
@@ -17,7 +17,7 @@ pub enum Error {
     UnableToObtainSlices,
     UnableToGrowMerkleTree,
     UnableToShrinkMerkleTree,
-    ShouldNeverBePacked(ItemType),
+    ShouldNeverBePacked(TreeHashType),
     BytesAreNotEvenChunks(usize),
     NoModifiedFieldForChunk(usize),
     NoBytesForChunk(usize),
@@ -31,15 +31,9 @@ pub trait CachedTreeHash<T>: CachedTreeHashSubTree<T> + Sized {
     fn clone_without_tree_hash_cache(&self) -> Self;
 }
 
-pub trait CachedTreeHashSubTree<Item> {
-    fn item_type() -> ItemType;
-
+pub trait CachedTreeHashSubTree<Item>: TreeHash {
     fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error>;
 
-    fn packed_encoding(&self) -> Result<Vec<u8>, Error>;
-
-    fn packing_factor() -> usize;
-
     fn new_cache(&self) -> Result<TreeHashCache, Error>;
 
     fn update_cache(
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
index 982e98724..190deaf27 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
@@ -5,10 +5,6 @@ use ssz::ssz_encode;
 mod vec;
 
 impl CachedTreeHashSubTree<u64> for u64 {
-    fn item_type() -> ItemType {
-        ItemType::Basic
-    }
-
     fn new_cache(&self) -> Result<TreeHashCache, Error> {
         Ok(TreeHashCache::from_bytes(
             merkleize(ssz_encode(self)),
@@ -20,14 +16,6 @@ impl CachedTreeHashSubTree<u64> for u64 {
         BTreeOverlay::from_lengths(chunk_offset, vec![1])
     }
 
-    fn packed_encoding(&self) -> Result<Vec<u8>, Error> {
-        Ok(ssz_encode(self))
-    }
-
-    fn packing_factor() -> usize {
-        HASHSIZE / 8
-    }
-
     fn update_cache(
         &self,
         other: &Self,
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
index a6fad9ba6..bc86e6054 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
@@ -2,18 +2,14 @@ use super::*;
 
 impl<T> CachedTreeHashSubTree<Vec<T>> for Vec<T>
 where
-    T: CachedTreeHashSubTree<T>,
+    T: CachedTreeHashSubTree<T> + TreeHash,
 {
-    fn item_type() -> ItemType {
-        ItemType::List
-    }
-
     fn new_cache(&self) -> Result<TreeHashCache, Error> {
-        match T::item_type() {
-            ItemType::Basic => {
+        match T::tree_hash_type() {
+            TreeHashType::Basic => {
                 TreeHashCache::from_bytes(merkleize(get_packed_leaves(self)?), false)
             }
-            ItemType::Composite | ItemType::List => {
+            TreeHashType::Composite | TreeHashType::List => {
                 let subtrees = self
                     .iter()
                     .map(|item| TreeHashCache::new(item))
@@ -25,9 +21,9 @@ where
     }
 
     fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
-        let lengths = match T::item_type() {
-            ItemType::Basic => vec![1; self.len() / T::packing_factor()],
-            ItemType::Composite | ItemType::List => {
+        let lengths = match T::tree_hash_type() {
+            TreeHashType::Basic => vec![1; self.len() / T::tree_hash_packing_factor()],
+            TreeHashType::Composite | TreeHashType::List => {
                 let mut lengths = vec![];
 
                 for item in self {
@@ -41,14 +37,6 @@ where
         BTreeOverlay::from_lengths(chunk_offset, lengths)
     }
 
-    fn packed_encoding(&self) -> Result<Vec<u8>, Error> {
-        Err(Error::ShouldNeverBePacked(Self::item_type()))
-    }
-
-    fn packing_factor() -> usize {
-        1
-    }
-
     fn update_cache(
         &self,
         other: &Vec<T>,
@@ -93,8 +81,8 @@ where
             cache.splice(old_offset_handler.chunk_range(), modified_cache);
         }
 
-        match T::item_type() {
-            ItemType::Basic => {
+        match T::tree_hash_type() {
+            TreeHashType::Basic => {
                 let leaves = get_packed_leaves(self)?;
 
                 for (i, chunk) in offset_handler.iter_leaf_nodes().enumerate() {
@@ -109,7 +97,7 @@ where
                     TreeHashCache::from_bytes(leaves, true)?,
                 );
             }
-            ItemType::Composite | ItemType::List => {
+            TreeHashType::Composite | TreeHashType::List => {
                 let mut i = offset_handler.num_leaf_nodes;
                 for &start_chunk in offset_handler.iter_leaf_nodes().rev() {
                     i -= 1;
@@ -170,13 +158,13 @@ fn get_packed_leaves<T>(vec: &Vec<T>) -> Result<Vec<u8>, Error>
 where
     T: CachedTreeHashSubTree<T>,
 {
-    let num_packed_bytes = (BYTES_PER_CHUNK / T::packing_factor()) * vec.len();
+    let num_packed_bytes = (BYTES_PER_CHUNK / T::tree_hash_packing_factor()) * vec.len();
     let num_leaves = num_sanitized_leaves(num_packed_bytes);
 
     let mut packed = Vec::with_capacity(num_leaves * HASHSIZE);
 
     for item in vec {
-        packed.append(&mut item.packed_encoding()?);
+        packed.append(&mut item.tree_hash_packed_encoding());
     }
 
     Ok(sanitise_bytes(packed))
diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs
index 4e5302bca..0f0fb60f4 100644
--- a/eth2/utils/tree_hash/src/lib.rs
+++ b/eth2/utils/tree_hash/src/lib.rs
@@ -5,8 +5,11 @@ pub const BYTES_PER_CHUNK: usize = 32;
 pub const HASHSIZE: usize = 32;
 pub const MERKLE_HASH_CHUNCK: usize = 2 * BYTES_PER_CHUNK;
 
+pub use cached_tree_hash::CachedTreeHashSubTree;
+pub use standard_tree_hash::TreeHash;
+
 #[derive(Debug, PartialEq, Clone)]
-pub enum ItemType {
+pub enum TreeHashType {
     Basic,
     List,
     Composite,
diff --git a/eth2/utils/tree_hash/src/standard_tree_hash.rs b/eth2/utils/tree_hash/src/standard_tree_hash.rs
index c8119a790..e7f94560b 100644
--- a/eth2/utils/tree_hash/src/standard_tree_hash.rs
+++ b/eth2/utils/tree_hash/src/standard_tree_hash.rs
@@ -4,23 +4,29 @@ use int_to_bytes::int_to_bytes32;
 use ssz::ssz_encode;
 
 pub trait TreeHash {
-    fn tree_hash_item_type() -> ItemType;
+    fn tree_hash_type() -> TreeHashType;
 
     fn tree_hash_packed_encoding(&self) -> Vec<u8>;
 
-    fn hash_tree_root(&self) -> Vec<u8>;
+    fn tree_hash_packing_factor() -> usize;
+
+    fn tree_hash_root(&self) -> Vec<u8>;
 }
 
 impl TreeHash for u64 {
-    fn tree_hash_item_type() -> ItemType {
-        ItemType::Basic
+    fn tree_hash_type() -> TreeHashType {
+        TreeHashType::Basic
     }
 
     fn tree_hash_packed_encoding(&self) -> Vec<u8> {
         ssz_encode(self)
     }
 
-    fn hash_tree_root(&self) -> Vec<u8> {
+    fn tree_hash_packing_factor() -> usize {
+        HASHSIZE / 8
+    }
+
+    fn tree_hash_root(&self) -> Vec<u8> {
         int_to_bytes32(*self)
     }
 }
@@ -29,18 +35,23 @@ impl<T> TreeHash for Vec<T>
 where
     T: TreeHash,
 {
-    fn tree_hash_item_type() -> ItemType {
-        ItemType::List
+    fn tree_hash_type() -> TreeHashType {
+        TreeHashType::List
     }
 
     fn tree_hash_packed_encoding(&self) -> Vec<u8> {
         unreachable!("List should never be packed.")
     }
 
-    fn hash_tree_root(&self) -> Vec<u8> {
-        let leaves = match T::tree_hash_item_type() {
-            ItemType::Basic => {
-                let mut leaves = vec![];
+    fn tree_hash_packing_factor() -> usize {
+        unreachable!("List should never be packed.")
+    }
+
+    fn tree_hash_root(&self) -> Vec<u8> {
+        let leaves = match T::tree_hash_type() {
+            TreeHashType::Basic => {
+                let mut leaves =
+                    Vec::with_capacity((HASHSIZE / T::tree_hash_packing_factor()) * self.len());
 
                 for item in self {
                     leaves.append(&mut item.tree_hash_packed_encoding());
@@ -48,11 +59,11 @@ where
 
                 leaves
             }
-            ItemType::Composite | ItemType::List => {
+            TreeHashType::Composite | TreeHashType::List => {
                 let mut leaves = Vec::with_capacity(self.len() * HASHSIZE);
 
                 for item in self {
-                    leaves.append(&mut item.hash_tree_root())
+                    leaves.append(&mut item.tree_hash_root())
                 }
 
                 leaves
diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
index d65192cd5..f52a17272 100644
--- a/eth2/utils/tree_hash/tests/tests.rs
+++ b/eth2/utils/tree_hash/tests/tests.rs
@@ -11,6 +11,29 @@ pub struct InternalCache {
     pub cache: Option<TreeHashCache>,
 }
 
+impl TreeHash for InternalCache {
+    fn tree_hash_type() -> TreeHashType {
+        TreeHashType::Composite
+    }
+
+    fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+        unreachable!("Struct should never be packed.")
+    }
+
+    fn tree_hash_packing_factor() -> usize {
+        unreachable!("Struct should never be packed.")
+    }
+
+    fn tree_hash_root(&self) -> Vec<u8> {
+        let mut leaves = Vec::with_capacity(4 * HASHSIZE);
+
+        leaves.append(&mut self.a.tree_hash_root());
+        leaves.append(&mut self.b.tree_hash_root());
+
+        efficient_merkleize(&leaves)[0..32].to_vec()
+    }
+}
+
 impl CachedTreeHash<InternalCache> for InternalCache {
     fn update_internal_tree_hash_cache(mut self, mut old: Self) -> Result<(Self, Self), Error> {
         let mut local_cache = old.cache;
@@ -66,10 +89,6 @@ fn works_when_embedded() {
 }
 
 impl CachedTreeHashSubTree<InternalCache> for InternalCache {
-    fn item_type() -> ItemType {
-        ItemType::Composite
-    }
-
     fn new_cache(&self) -> Result<TreeHashCache, Error> {
         let tree = TreeHashCache::from_leaves_and_subtrees(
             self,
@@ -88,14 +107,6 @@ impl CachedTreeHashSubTree<InternalCache> for InternalCache {
         BTreeOverlay::from_lengths(chunk_offset, lengths)
     }
 
-    fn packed_encoding(&self) -> Result<Vec<u8>, Error> {
-        Err(Error::ShouldNeverBePacked(Self::item_type()))
-    }
-
-    fn packing_factor() -> usize {
-        1
-    }
-
     fn update_cache(
         &self,
         other: &Self,
@@ -134,31 +145,31 @@ pub struct Inner {
 }
 
 impl TreeHash for Inner {
-    fn tree_hash_item_type() -> ItemType {
-        ItemType::Composite
+    fn tree_hash_type() -> TreeHashType {
+        TreeHashType::Composite
     }
 
     fn tree_hash_packed_encoding(&self) -> Vec<u8> {
         unreachable!("Struct should never be packed.")
     }
 
-    fn hash_tree_root(&self) -> Vec<u8> {
+    fn tree_hash_packing_factor() -> usize {
+        unreachable!("Struct should never be packed.")
+    }
+
+    fn tree_hash_root(&self) -> Vec<u8> {
         let mut leaves = Vec::with_capacity(4 * HASHSIZE);
 
-        leaves.append(&mut self.a.hash_tree_root());
-        leaves.append(&mut self.b.hash_tree_root());
-        leaves.append(&mut self.c.hash_tree_root());
-        leaves.append(&mut self.d.hash_tree_root());
+        leaves.append(&mut self.a.tree_hash_root());
+        leaves.append(&mut self.b.tree_hash_root());
+        leaves.append(&mut self.c.tree_hash_root());
+        leaves.append(&mut self.d.tree_hash_root());
 
         efficient_merkleize(&leaves)[0..32].to_vec()
     }
 }
 
 impl CachedTreeHashSubTree<Inner> for Inner {
-    fn item_type() -> ItemType {
-        ItemType::Composite
-    }
-
     fn new_cache(&self) -> Result<TreeHashCache, Error> {
         let tree = TreeHashCache::from_leaves_and_subtrees(
             self,
@@ -184,14 +195,6 @@ impl CachedTreeHashSubTree<Inner> for Inner {
         BTreeOverlay::from_lengths(chunk_offset, lengths)
     }
 
-    fn packed_encoding(&self) -> Result<Vec<u8>, Error> {
-        Err(Error::ShouldNeverBePacked(Self::item_type()))
-    }
-
-    fn packing_factor() -> usize {
-        1
-    }
-
     fn update_cache(
         &self,
         other: &Self,
@@ -226,11 +229,31 @@ pub struct Outer {
     pub c: u64,
 }
 
-impl CachedTreeHashSubTree<Outer> for Outer {
-    fn item_type() -> ItemType {
-        ItemType::Composite
+impl TreeHash for Outer {
+    fn tree_hash_type() -> TreeHashType {
+        TreeHashType::Composite
     }
 
+    fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+        unreachable!("Struct should never be packed.")
+    }
+
+    fn tree_hash_packing_factor() -> usize {
+        unreachable!("Struct should never be packed.")
+    }
+
+    fn tree_hash_root(&self) -> Vec<u8> {
+        let mut leaves = Vec::with_capacity(4 * HASHSIZE);
+
+        leaves.append(&mut self.a.tree_hash_root());
+        leaves.append(&mut self.b.tree_hash_root());
+        leaves.append(&mut self.c.tree_hash_root());
+
+        efficient_merkleize(&leaves)[0..32].to_vec()
+    }
+}
+
+impl CachedTreeHashSubTree<Outer> for Outer {
     fn new_cache(&self) -> Result<TreeHashCache, Error> {
         let tree = TreeHashCache::from_leaves_and_subtrees(
             self,
@@ -254,14 +277,6 @@ impl CachedTreeHashSubTree<Outer> for Outer {
         BTreeOverlay::from_lengths(chunk_offset, lengths)
     }
 
-    fn packed_encoding(&self) -> Result<Vec<u8>, Error> {
-        Err(Error::ShouldNeverBePacked(Self::item_type()))
-    }
-
-    fn packing_factor() -> usize {
-        1
-    }
-
     fn update_cache(
         &self,
         other: &Self,
@@ -481,7 +496,7 @@ fn test_u64_vec_modifications(original: Vec<u64>, modified: Vec<u64>) {
     mix_in_length(&mut expected[0..HASHSIZE], modified.len());
 
     assert_eq!(expected, modified_cache);
-    assert_eq!(&expected[0..32], &modified.hash_tree_root()[..]);
+    assert_eq!(&expected[0..32], &modified.tree_hash_root()[..]);
 }
 
 #[test]
@@ -604,7 +619,7 @@ fn test_inner_vec_modifications(original: Vec<Inner>, modified: Vec<Inner>, refe
 
     // Compare the cached tree to the reference tree.
     assert_trees_eq(&expected, &modified_cache);
-    assert_eq!(&expected[0..32], &modified.hash_tree_root()[..]);
+    assert_eq!(&expected[0..32], &modified.tree_hash_root()[..]);
 }
 
 #[test]

From 8a1bde3e2f9d64f06000208e1940082007c241de Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Tue, 16 Apr 2019 10:47:58 +1000
Subject: [PATCH 054/137] Update naming for tree_hash fns/structs/traits

---
 eth2/utils/tree_hash/src/cached_tree_hash.rs  |  8 +--
 .../src/cached_tree_hash/btree_overlay.rs     |  2 +-
 .../tree_hash/src/cached_tree_hash/impls.rs   |  6 +-
 .../src/cached_tree_hash/impls/vec.rs         |  8 +--
 eth2/utils/tree_hash/src/lib.rs               |  4 +-
 eth2/utils/tree_hash/tests/tests.rs           | 72 ++++++++++---------
 eth2/utils/tree_hash_derive/src/lib.rs        | 70 +++++++++++++-----
 eth2/utils/tree_hash_derive/tests/tests.rs    | 66 ++++++++++++++++-
 8 files changed, 166 insertions(+), 70 deletions(-)

diff --git a/eth2/utils/tree_hash/src/cached_tree_hash.rs b/eth2/utils/tree_hash/src/cached_tree_hash.rs
index 43c0ba2fe..e093b2dd7 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash.rs
@@ -32,11 +32,11 @@ pub trait CachedTreeHash<T>: CachedTreeHashSubTree<T> + Sized {
 }
 
 pub trait CachedTreeHashSubTree<Item>: TreeHash {
-    fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error>;
+    fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error>;
 
-    fn new_cache(&self) -> Result<TreeHashCache, Error>;
+    fn new_tree_hash_cache(&self) -> Result<TreeHashCache, Error>;
 
-    fn update_cache(
+    fn update_tree_hash_cache(
         &self,
         other: &Item,
         cache: &mut TreeHashCache,
@@ -136,7 +136,7 @@ impl TreeHashCache {
     where
         T: CachedTreeHashSubTree<T>,
     {
-        item.new_cache()
+        item.new_tree_hash_cache()
     }
 
     pub fn from_elems(cache: Vec<u8>, chunk_modified: Vec<bool>) -> Self {
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs b/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
index 1e188da60..e8c04a91e 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
@@ -14,7 +14,7 @@ impl BTreeOverlay {
     where
         T: CachedTreeHashSubTree<T>,
     {
-        item.btree_overlay(initial_offset)
+        item.tree_hash_cache_overlay(initial_offset)
     }
 
     pub fn from_lengths(offset: usize, mut lengths: Vec<usize>) -> Result<Self, Error> {
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
index 190deaf27..62d013881 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
@@ -5,18 +5,18 @@ use ssz::ssz_encode;
 mod vec;
 
 impl CachedTreeHashSubTree<u64> for u64 {
-    fn new_cache(&self) -> Result<TreeHashCache, Error> {
+    fn new_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
         Ok(TreeHashCache::from_bytes(
             merkleize(ssz_encode(self)),
             false,
         )?)
     }
 
-    fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
+    fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
         BTreeOverlay::from_lengths(chunk_offset, vec![1])
     }
 
-    fn update_cache(
+    fn update_tree_hash_cache(
         &self,
         other: &Self,
         cache: &mut TreeHashCache,
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
index bc86e6054..6c0970cef 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
@@ -4,7 +4,7 @@ impl<T> CachedTreeHashSubTree<Vec<T>> for Vec<T>
 where
     T: CachedTreeHashSubTree<T> + TreeHash,
 {
-    fn new_cache(&self) -> Result<TreeHashCache, Error> {
+    fn new_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
         match T::tree_hash_type() {
             TreeHashType::Basic => {
                 TreeHashCache::from_bytes(merkleize(get_packed_leaves(self)?), false)
@@ -20,7 +20,7 @@ where
         }
     }
 
-    fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
+    fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
         let lengths = match T::tree_hash_type() {
             TreeHashType::Basic => vec![1; self.len() / T::tree_hash_packing_factor()],
             TreeHashType::Composite | TreeHashType::List => {
@@ -37,7 +37,7 @@ where
         BTreeOverlay::from_lengths(chunk_offset, lengths)
     }
 
-    fn update_cache(
+    fn update_tree_hash_cache(
         &self,
         other: &Vec<T>,
         cache: &mut TreeHashCache,
@@ -104,7 +104,7 @@ where
                     match (other.get(i), self.get(i)) {
                         // The item existed in the previous list and exsits in the current list.
                         (Some(old), Some(new)) => {
-                            new.update_cache(old, cache, start_chunk)?;
+                            new.update_tree_hash_cache(old, cache, start_chunk)?;
                         }
                         // The item existed in the previous list but does not exist in this list.
                         //
diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs
index 0f0fb60f4..04eb6d80f 100644
--- a/eth2/utils/tree_hash/src/lib.rs
+++ b/eth2/utils/tree_hash/src/lib.rs
@@ -5,8 +5,8 @@ pub const BYTES_PER_CHUNK: usize = 32;
 pub const HASHSIZE: usize = 32;
 pub const MERKLE_HASH_CHUNCK: usize = 2 * BYTES_PER_CHUNK;
 
-pub use cached_tree_hash::CachedTreeHashSubTree;
-pub use standard_tree_hash::TreeHash;
+pub use cached_tree_hash::{BTreeOverlay, CachedTreeHashSubTree, Error, TreeHashCache};
+pub use standard_tree_hash::{efficient_merkleize, TreeHash};
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum TreeHashType {
diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
index f52a17272..db33709ac 100644
--- a/eth2/utils/tree_hash/tests/tests.rs
+++ b/eth2/utils/tree_hash/tests/tests.rs
@@ -40,9 +40,9 @@ impl CachedTreeHash<InternalCache> for InternalCache {
         old.cache = None;
 
         if let Some(ref mut local_cache) = local_cache {
-            self.update_cache(&old, local_cache, 0)?;
+            self.update_tree_hash_cache(&old, local_cache, 0)?;
         } else {
-            local_cache = Some(self.new_cache()?)
+            local_cache = Some(self.new_tree_hash_cache()?)
         }
 
         self.cache = local_cache;
@@ -89,16 +89,16 @@ fn works_when_embedded() {
 }
 
 impl CachedTreeHashSubTree<InternalCache> for InternalCache {
-    fn new_cache(&self) -> Result<TreeHashCache, Error> {
+    fn new_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
         let tree = TreeHashCache::from_leaves_and_subtrees(
             self,
-            vec![self.a.new_cache()?, self.b.new_cache()?],
+            vec![self.a.new_tree_hash_cache()?, self.b.new_tree_hash_cache()?],
         )?;
 
         Ok(tree)
     }
 
-    fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
+    fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
         let mut lengths = vec![];
 
         lengths.push(BTreeOverlay::new(&self.a, 0)?.total_nodes());
@@ -107,7 +107,7 @@ impl CachedTreeHashSubTree<InternalCache> for InternalCache {
         BTreeOverlay::from_lengths(chunk_offset, lengths)
     }
 
-    fn update_cache(
+    fn update_tree_hash_cache(
         &self,
         other: &Self,
         cache: &mut TreeHashCache,
@@ -118,8 +118,8 @@ impl CachedTreeHashSubTree<InternalCache> for InternalCache {
         // Skip past the internal nodes and update any changed leaf nodes.
         {
             let chunk = offset_handler.first_leaf_node()?;
-            let chunk = self.a.update_cache(&other.a, cache, chunk)?;
-            let _chunk = self.b.update_cache(&other.b, cache, chunk)?;
+            let chunk = self.a.update_tree_hash_cache(&other.a, cache, chunk)?;
+            let _chunk = self.b.update_tree_hash_cache(&other.b, cache, chunk)?;
         }
 
         for (&parent, children) in offset_handler.iter_internal_nodes().rev() {
@@ -170,21 +170,21 @@ impl TreeHash for Inner {
 }
 
 impl CachedTreeHashSubTree<Inner> for Inner {
-    fn new_cache(&self) -> Result<TreeHashCache, Error> {
+    fn new_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
         let tree = TreeHashCache::from_leaves_and_subtrees(
             self,
             vec![
-                self.a.new_cache()?,
-                self.b.new_cache()?,
-                self.c.new_cache()?,
-                self.d.new_cache()?,
+                self.a.new_tree_hash_cache()?,
+                self.b.new_tree_hash_cache()?,
+                self.c.new_tree_hash_cache()?,
+                self.d.new_tree_hash_cache()?,
             ],
         )?;
 
         Ok(tree)
     }
 
-    fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
+    fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
         let mut lengths = vec![];
 
         lengths.push(BTreeOverlay::new(&self.a, 0)?.total_nodes());
@@ -195,7 +195,7 @@ impl CachedTreeHashSubTree<Inner> for Inner {
         BTreeOverlay::from_lengths(chunk_offset, lengths)
     }
 
-    fn update_cache(
+    fn update_tree_hash_cache(
         &self,
         other: &Self,
         cache: &mut TreeHashCache,
@@ -206,10 +206,10 @@ impl CachedTreeHashSubTree<Inner> for Inner {
         // Skip past the internal nodes and update any changed leaf nodes.
         {
             let chunk = offset_handler.first_leaf_node()?;
-            let chunk = self.a.update_cache(&other.a, cache, chunk)?;
-            let chunk = self.b.update_cache(&other.b, cache, chunk)?;
-            let chunk = self.c.update_cache(&other.c, cache, chunk)?;
-            let _chunk = self.d.update_cache(&other.d, cache, chunk)?;
+            let chunk = self.a.update_tree_hash_cache(&other.a, cache, chunk)?;
+            let chunk = self.b.update_tree_hash_cache(&other.b, cache, chunk)?;
+            let chunk = self.c.update_tree_hash_cache(&other.c, cache, chunk)?;
+            let _chunk = self.d.update_tree_hash_cache(&other.d, cache, chunk)?;
         }
 
         for (&parent, children) in offset_handler.iter_internal_nodes().rev() {
@@ -254,20 +254,20 @@ impl TreeHash for Outer {
 }
 
 impl CachedTreeHashSubTree<Outer> for Outer {
-    fn new_cache(&self) -> Result<TreeHashCache, Error> {
+    fn new_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
         let tree = TreeHashCache::from_leaves_and_subtrees(
             self,
             vec![
-                self.a.new_cache()?,
-                self.b.new_cache()?,
-                self.c.new_cache()?,
+                self.a.new_tree_hash_cache()?,
+                self.b.new_tree_hash_cache()?,
+                self.c.new_tree_hash_cache()?,
             ],
         )?;
 
         Ok(tree)
     }
 
-    fn btree_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
+    fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
         let mut lengths = vec![];
 
         lengths.push(BTreeOverlay::new(&self.a, 0)?.total_nodes());
@@ -277,7 +277,7 @@ impl CachedTreeHashSubTree<Outer> for Outer {
         BTreeOverlay::from_lengths(chunk_offset, lengths)
     }
 
-    fn update_cache(
+    fn update_tree_hash_cache(
         &self,
         other: &Self,
         cache: &mut TreeHashCache,
@@ -288,9 +288,9 @@ impl CachedTreeHashSubTree<Outer> for Outer {
         // Skip past the internal nodes and update any changed leaf nodes.
         {
             let chunk = offset_handler.first_leaf_node()?;
-            let chunk = self.a.update_cache(&other.a, cache, chunk)?;
-            let chunk = self.b.update_cache(&other.b, cache, chunk)?;
-            let _chunk = self.c.update_cache(&other.c, cache, chunk)?;
+            let chunk = self.a.update_tree_hash_cache(&other.a, cache, chunk)?;
+            let chunk = self.b.update_tree_hash_cache(&other.b, cache, chunk)?;
+            let _chunk = self.c.update_tree_hash_cache(&other.c, cache, chunk)?;
         }
 
         for (&parent, children) in offset_handler.iter_internal_nodes().rev() {
@@ -341,7 +341,7 @@ fn partial_modification_to_inner_struct() {
     let mut cache_struct = TreeHashCache::new(&original_outer).unwrap();
 
     modified_outer
-        .update_cache(&original_outer, &mut cache_struct, 0)
+        .update_tree_hash_cache(&original_outer, &mut cache_struct, 0)
         .unwrap();
 
     let modified_cache: Vec<u8> = cache_struct.into();
@@ -395,7 +395,7 @@ fn partial_modification_to_outer() {
     let mut cache_struct = TreeHashCache::new(&original_outer).unwrap();
 
     modified_outer
-        .update_cache(&original_outer, &mut cache_struct, 0)
+        .update_tree_hash_cache(&original_outer, &mut cache_struct, 0)
         .unwrap();
 
     let modified_cache: Vec<u8> = cache_struct.into();
@@ -481,7 +481,7 @@ fn test_u64_vec_modifications(original: Vec<u64>, modified: Vec<u64>) {
     // Perform a differential hash
     let mut cache_struct = TreeHashCache::from_bytes(original_cache.clone(), false).unwrap();
     modified
-        .update_cache(&original, &mut cache_struct, 0)
+        .update_tree_hash_cache(&original, &mut cache_struct, 0)
         .unwrap();
     let modified_cache: Vec<u8> = cache_struct.into();
 
@@ -586,7 +586,9 @@ fn large_vec_of_u64_builds() {
 fn test_inner_vec_modifications(original: Vec<Inner>, modified: Vec<Inner>, reference: Vec<u64>) {
     let mut cache = TreeHashCache::new(&original).unwrap();
 
-    modified.update_cache(&original, &mut cache, 0).unwrap();
+    modified
+        .update_tree_hash_cache(&original, &mut cache, 0)
+        .unwrap();
     let modified_cache: Vec<u8> = cache.into();
 
     // Build the reference vec.
@@ -947,12 +949,12 @@ fn generic_test(index: usize) {
     let mut cache_struct = TreeHashCache::from_bytes(cache.clone(), false).unwrap();
 
     changed_inner
-        .update_cache(&inner, &mut cache_struct, 0)
+        .update_tree_hash_cache(&inner, &mut cache_struct, 0)
         .unwrap();
 
     // assert_eq!(*cache_struct.hash_count, 3);
 
-    let new_cache: Vec<u8> = cache_struct.into();
+    let new_tree_hash_cache: Vec<u8> = cache_struct.into();
 
     let data1 = int_to_bytes32(1);
     let data2 = int_to_bytes32(2);
@@ -965,7 +967,7 @@ fn generic_test(index: usize) {
 
     let expected = merkleize(join(data));
 
-    assert_eq!(expected, new_cache);
+    assert_eq!(expected, new_tree_hash_cache);
 }
 
 #[test]
diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs
index 217e91c24..b2afabaa9 100644
--- a/eth2/utils/tree_hash_derive/src/lib.rs
+++ b/eth2/utils/tree_hash_derive/src/lib.rs
@@ -39,9 +39,9 @@ fn should_skip_hashing(field: &syn::Field) -> bool {
     false
 }
 
-/// Implements `ssz::Encodable` for some `struct`.
+/// Implements `tree_hash::CachedTreeHashSubTree` for some `struct`.
 ///
-/// Fields are encoded in the order they are defined.
+/// Fields are hashed in the order they are defined.
 #[proc_macro_derive(CachedTreeHashSubTree, attributes(tree_hash))]
 pub fn subtree_derive(input: TokenStream) -> TokenStream {
     let item = parse_macro_input!(input as DeriveInput);
@@ -60,16 +60,12 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
 
     let output = quote! {
         impl tree_hash::CachedTreeHashSubTree<#name> for #name {
-            fn item_type() -> tree_hash::ItemType {
-                tree_hash::ItemType::Composite
-            }
-
-            fn new_cache(&self) -> Result<tree_hash::TreeHashCache, tree_hash::Error> {
+            fn new_tree_hash_cache(&self) -> Result<tree_hash::TreeHashCache, tree_hash::Error> {
                 let tree = tree_hash::TreeHashCache::from_leaves_and_subtrees(
                     self,
                     vec![
                         #(
-                            self.#idents_a.new_cache()?,
+                            self.#idents_a.new_tree_hash_cache()?,
                         )*
                     ],
                 )?;
@@ -77,7 +73,7 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
                 Ok(tree)
             }
 
-            fn btree_overlay(&self, chunk_offset: usize) -> Result<tree_hash::BTreeOverlay, tree_hash::Error> {
+            fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result<tree_hash::BTreeOverlay, tree_hash::Error> {
                 let mut lengths = vec![];
 
                 #(
@@ -87,15 +83,7 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
                 tree_hash::BTreeOverlay::from_lengths(chunk_offset, lengths)
             }
 
-            fn packed_encoding(&self) -> Result<Vec<u8>, tree_hash::Error> {
-                Err(tree_hash::Error::ShouldNeverBePacked(Self::item_type()))
-            }
-
-            fn packing_factor() -> usize {
-                1
-            }
-
-            fn update_cache(
+            fn update_tree_hash_cache(
                 &self,
                 other: &Self,
                 cache: &mut tree_hash::TreeHashCache,
@@ -107,7 +95,7 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
                 {
                     let chunk = offset_handler.first_leaf_node()?;
                     #(
-                        let chunk = self.#idents_c.update_cache(&other.#idents_d, cache, chunk)?;
+                        let chunk = self.#idents_c.update_tree_hash_cache(&other.#idents_d, cache, chunk)?;
                     )*
                 }
 
@@ -123,3 +111,47 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
     };
     output.into()
 }
+
+/// Implements `tree_hash::TreeHash` for some `struct`.
+///
+/// Fields are hashed in the order they are defined.
+#[proc_macro_derive(TreeHash, attributes(tree_hash))]
+pub fn tree_hash_derive(input: TokenStream) -> TokenStream {
+    let item = parse_macro_input!(input as DeriveInput);
+
+    let name = &item.ident;
+
+    let struct_data = match &item.data {
+        syn::Data::Struct(s) => s,
+        _ => panic!("tree_hash_derive only supports structs."),
+    };
+
+    let idents = get_hashable_named_field_idents(&struct_data);
+
+    let output = quote! {
+        impl tree_hash::TreeHash for #name {
+            fn tree_hash_type() -> tree_hash::TreeHashType {
+                tree_hash::TreeHashType::Composite
+            }
+
+            fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+                unreachable!("Struct should never be packed.")
+            }
+
+            fn tree_hash_packing_factor() -> usize {
+                unreachable!("Struct should never be packed.")
+            }
+
+            fn tree_hash_root(&self) -> Vec<u8> {
+                let mut leaves = Vec::with_capacity(4 * tree_hash::HASHSIZE);
+
+                #(
+                    leaves.append(&mut self.#idents.tree_hash_root());
+                )*
+
+                tree_hash::efficient_merkleize(&leaves)[0..32].to_vec()
+            }
+        }
+    };
+    output.into()
+}
diff --git a/eth2/utils/tree_hash_derive/tests/tests.rs b/eth2/utils/tree_hash_derive/tests/tests.rs
index a5ab112a2..5f065c982 100644
--- a/eth2/utils/tree_hash_derive/tests/tests.rs
+++ b/eth2/utils/tree_hash_derive/tests/tests.rs
@@ -1,9 +1,71 @@
-use tree_hash_derive::CachedTreeHashSubTree;
+use tree_hash::CachedTreeHashSubTree;
+use tree_hash_derive::{CachedTreeHashSubTree, TreeHash};
 
-#[derive(Clone, Debug, CachedTreeHashSubTree)]
+#[derive(Clone, Debug, TreeHash, CachedTreeHashSubTree)]
 pub struct Inner {
     pub a: u64,
     pub b: u64,
     pub c: u64,
     pub d: u64,
 }
+
+fn test_standard_and_cached<T>(original: &T, modified: &T)
+where
+    T: CachedTreeHashSubTree<T>,
+{
+    let mut cache = original.new_tree_hash_cache().unwrap();
+
+    let standard_root = original.tree_hash_root();
+    let cached_root = cache.root().unwrap().to_vec();
+    assert_eq!(standard_root, cached_root);
+
+    // Test after a modification
+    modified
+        .update_tree_hash_cache(&original, &mut cache, 0)
+        .unwrap();
+    let standard_root = modified.tree_hash_root();
+    let cached_root = cache.root().unwrap().to_vec();
+    assert_eq!(standard_root, cached_root);
+}
+
+#[test]
+fn inner_standard_vs_cached() {
+    let original = Inner {
+        a: 1,
+        b: 2,
+        c: 3,
+        d: 4,
+    };
+    let modified = Inner {
+        b: 42,
+        ..original.clone()
+    };
+
+    test_standard_and_cached(&original, &modified);
+}
+
+#[derive(Clone, Debug, TreeHash, CachedTreeHashSubTree)]
+pub struct Uneven {
+    pub a: u64,
+    pub b: u64,
+    pub c: u64,
+    pub d: u64,
+    pub e: u64,
+}
+
+#[test]
+fn uneven_standard_vs_cached() {
+    let original = Uneven {
+        a: 1,
+        b: 2,
+        c: 3,
+        d: 4,
+        e: 5,
+    };
+    let modified = Uneven {
+        e: 42,
+        ..original.clone()
+    };
+
+    test_standard_and_cached(&original, &modified);
+}

From 024b9e315ad2edcf71611fb8361b607d25a74dbe Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Tue, 16 Apr 2019 11:14:28 +1000
Subject: [PATCH 055/137] Add signed_root to tree_hash crate

---
 eth2/utils/tree_hash/src/lib.rs            |  2 +
 eth2/utils/tree_hash/src/signed_root.rs    |  5 ++
 eth2/utils/tree_hash_derive/src/lib.rs     | 68 +++++++++++++++++++++-
 eth2/utils/tree_hash_derive/tests/tests.rs | 33 ++++++++++-
 4 files changed, 105 insertions(+), 3 deletions(-)
 create mode 100644 eth2/utils/tree_hash/src/signed_root.rs

diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs
index 04eb6d80f..ac7e7633d 100644
--- a/eth2/utils/tree_hash/src/lib.rs
+++ b/eth2/utils/tree_hash/src/lib.rs
@@ -1,4 +1,5 @@
 pub mod cached_tree_hash;
+pub mod signed_root;
 pub mod standard_tree_hash;
 
 pub const BYTES_PER_CHUNK: usize = 32;
@@ -6,6 +7,7 @@ pub const HASHSIZE: usize = 32;
 pub const MERKLE_HASH_CHUNCK: usize = 2 * BYTES_PER_CHUNK;
 
 pub use cached_tree_hash::{BTreeOverlay, CachedTreeHashSubTree, Error, TreeHashCache};
+pub use signed_root::SignedRoot;
 pub use standard_tree_hash::{efficient_merkleize, TreeHash};
 
 #[derive(Debug, PartialEq, Clone)]
diff --git a/eth2/utils/tree_hash/src/signed_root.rs b/eth2/utils/tree_hash/src/signed_root.rs
new file mode 100644
index 000000000..f7aeca4af
--- /dev/null
+++ b/eth2/utils/tree_hash/src/signed_root.rs
@@ -0,0 +1,5 @@
+use crate::TreeHash;
+
+pub trait SignedRoot: TreeHash {
+    fn signed_root(&self) -> Vec<u8>;
+}
diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs
index b2afabaa9..ff5bc0d47 100644
--- a/eth2/utils/tree_hash_derive/src/lib.rs
+++ b/eth2/utils/tree_hash_derive/src/lib.rs
@@ -2,7 +2,7 @@
 extern crate proc_macro;
 
 use proc_macro::TokenStream;
-use quote::quote;
+use quote::{quote, ToTokens};
 use syn::{parse_macro_input, DeriveInput};
 
 /// Returns a Vec of `syn::Ident` for each named field in the struct, whilst filtering out fields
@@ -155,3 +155,69 @@ pub fn tree_hash_derive(input: TokenStream) -> TokenStream {
     };
     output.into()
 }
+
+/// Implements `tree_hash::TreeHash` for some `struct`, whilst excluding any fields following and
+/// including a field that is of type "Signature" or "AggregateSignature".
+///
+/// See:
+/// https://github.com/ethereum/eth2.0-specs/blob/master/specs/simple-serialize.md#signed-roots
+///
+/// This is a rather horrendous macro, it will read the type of the object as a string and decide
+/// if it's a signature by matching that string against "Signature" or "AggregateSignature". So,
+/// it's important that you use those exact words as your type -- don't alias it to something else.
+///
+/// If you can think of a better way to do this, please make an issue!
+///
+/// Fields are processed in the order they are defined.
+#[proc_macro_derive(SignedRoot, attributes(signed_root))]
+pub fn tree_hash_signed_root_derive(input: TokenStream) -> TokenStream {
+    let item = parse_macro_input!(input as DeriveInput);
+
+    let name = &item.ident;
+
+    let struct_data = match &item.data {
+        syn::Data::Struct(s) => s,
+        _ => panic!("tree_hash_derive only supports structs."),
+    };
+
+    let idents = get_signed_root_named_field_idents(&struct_data);
+
+    let output = quote! {
+        impl tree_hash::SignedRoot for #name {
+            fn signed_root(&self) -> Vec<u8> {
+                let mut leaves = Vec::with_capacity(4 * tree_hash::HASHSIZE);
+
+                #(
+                    leaves.append(&mut self.#idents.tree_hash_root());
+                )*
+
+                tree_hash::efficient_merkleize(&leaves)[0..32].to_vec()
+            }
+        }
+    };
+    output.into()
+}
+
+fn get_signed_root_named_field_idents(struct_data: &syn::DataStruct) -> Vec<&syn::Ident> {
+    struct_data
+        .fields
+        .iter()
+        .filter_map(|f| {
+            if should_skip_signed_root(&f) {
+                None
+            } else {
+                Some(match &f.ident {
+                    Some(ref ident) => ident,
+                    _ => panic!("tree_hash_derive only supports named struct fields"),
+                })
+            }
+        })
+        .collect()
+}
+
+fn should_skip_signed_root(field: &syn::Field) -> bool {
+    field
+        .attrs
+        .iter()
+        .any(|attr| attr.into_token_stream().to_string() == "# [ signed_root ( skip_hashing ) ]")
+}
diff --git a/eth2/utils/tree_hash_derive/tests/tests.rs b/eth2/utils/tree_hash_derive/tests/tests.rs
index 5f065c982..721e77715 100644
--- a/eth2/utils/tree_hash_derive/tests/tests.rs
+++ b/eth2/utils/tree_hash_derive/tests/tests.rs
@@ -1,5 +1,5 @@
-use tree_hash::CachedTreeHashSubTree;
-use tree_hash_derive::{CachedTreeHashSubTree, TreeHash};
+use tree_hash::{CachedTreeHashSubTree, SignedRoot, TreeHash};
+use tree_hash_derive::{CachedTreeHashSubTree, SignedRoot, TreeHash};
 
 #[derive(Clone, Debug, TreeHash, CachedTreeHashSubTree)]
 pub struct Inner {
@@ -69,3 +69,32 @@ fn uneven_standard_vs_cached() {
 
     test_standard_and_cached(&original, &modified);
 }
+
+#[derive(Clone, Debug, TreeHash, SignedRoot)]
+pub struct SignedInner {
+    pub a: u64,
+    pub b: u64,
+    pub c: u64,
+    pub d: u64,
+    #[signed_root(skip_hashing)]
+    pub e: u64,
+}
+
+#[test]
+fn signed_root() {
+    let unsigned = Inner {
+        a: 1,
+        b: 2,
+        c: 3,
+        d: 4,
+    };
+    let signed = SignedInner {
+        a: 1,
+        b: 2,
+        c: 3,
+        d: 4,
+        e: 5,
+    };
+
+    assert_eq!(unsigned.tree_hash_root(), signed.signed_root());
+}

From 3eaa06d758152cc69ce934622fcd887f28e6c9c8 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Tue, 16 Apr 2019 12:29:39 +1000
Subject: [PATCH 056/137] Remove tree hashing from ssz crate

---
 eth2/utils/ssz/src/impl_tree_hash.rs          |  85 ----------
 eth2/utils/ssz/src/lib.rs                     |   5 -
 eth2/utils/ssz/src/signed_root.rs             |   5 -
 eth2/utils/ssz/src/tree_hash.rs               | 107 ------------
 eth2/utils/ssz_derive/src/lib.rs              | 154 ------------------
 eth2/utils/tree_hash/src/lib.rs               |  25 ++-
 .../utils/tree_hash/src/standard_tree_hash.rs |  69 +-------
 .../tree_hash/src/standard_tree_hash/impls.rs |  97 +++++++++++
 eth2/utils/tree_hash_derive/src/lib.rs        |   4 +-
 9 files changed, 128 insertions(+), 423 deletions(-)
 delete mode 100644 eth2/utils/ssz/src/impl_tree_hash.rs
 delete mode 100644 eth2/utils/ssz/src/signed_root.rs
 delete mode 100644 eth2/utils/ssz/src/tree_hash.rs
 create mode 100644 eth2/utils/tree_hash/src/standard_tree_hash/impls.rs

diff --git a/eth2/utils/ssz/src/impl_tree_hash.rs b/eth2/utils/ssz/src/impl_tree_hash.rs
deleted file mode 100644
index 03976f637..000000000
--- a/eth2/utils/ssz/src/impl_tree_hash.rs
+++ /dev/null
@@ -1,85 +0,0 @@
-use super::ethereum_types::{Address, H256};
-use super::{merkle_hash, ssz_encode, TreeHash};
-use hashing::hash;
-
-impl TreeHash for u8 {
-    fn hash_tree_root(&self) -> Vec<u8> {
-        ssz_encode(self)
-    }
-}
-
-impl TreeHash for u16 {
-    fn hash_tree_root(&self) -> Vec<u8> {
-        ssz_encode(self)
-    }
-}
-
-impl TreeHash for u32 {
-    fn hash_tree_root(&self) -> Vec<u8> {
-        ssz_encode(self)
-    }
-}
-
-impl TreeHash for u64 {
-    fn hash_tree_root(&self) -> Vec<u8> {
-        ssz_encode(self)
-    }
-}
-
-impl TreeHash for usize {
-    fn hash_tree_root(&self) -> Vec<u8> {
-        ssz_encode(self)
-    }
-}
-
-impl TreeHash for bool {
-    fn hash_tree_root(&self) -> Vec<u8> {
-        ssz_encode(self)
-    }
-}
-
-impl TreeHash for Address {
-    fn hash_tree_root(&self) -> Vec<u8> {
-        ssz_encode(self)
-    }
-}
-
-impl TreeHash for H256 {
-    fn hash_tree_root(&self) -> Vec<u8> {
-        ssz_encode(self)
-    }
-}
-
-impl TreeHash for [u8] {
-    fn hash_tree_root(&self) -> Vec<u8> {
-        if self.len() > 32 {
-            return hash(&self);
-        }
-        self.to_vec()
-    }
-}
-
-impl<T> TreeHash for Vec<T>
-where
-    T: TreeHash,
-{
-    /// Returns the merkle_hash of a list of hash_tree_root values created
-    /// from the given list.
-    /// Note: A byte vector, Vec<u8>, must be converted to a slice (as_slice())
-    ///       to be handled properly (i.e. hashed) as byte array.
-    fn hash_tree_root(&self) -> Vec<u8> {
-        let mut tree_hashes = self.iter().map(|x| x.hash_tree_root()).collect();
-        merkle_hash(&mut tree_hashes)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_impl_tree_hash_vec() {
-        let result = vec![1u32, 2, 3, 4, 5, 6, 7].hash_tree_root();
-        assert_eq!(result.len(), 32);
-    }
-}
diff --git a/eth2/utils/ssz/src/lib.rs b/eth2/utils/ssz/src/lib.rs
index cb3f63c48..0a00efa5d 100644
--- a/eth2/utils/ssz/src/lib.rs
+++ b/eth2/utils/ssz/src/lib.rs
@@ -12,17 +12,12 @@ extern crate ethereum_types;
 
 pub mod decode;
 pub mod encode;
-mod signed_root;
-pub mod tree_hash;
 
 mod impl_decode;
 mod impl_encode;
-mod impl_tree_hash;
 
 pub use crate::decode::{decode, decode_ssz_list, Decodable, DecodeError};
 pub use crate::encode::{Encodable, SszStream};
-pub use crate::signed_root::SignedRoot;
-pub use crate::tree_hash::{merkle_hash, TreeHash};
 
 pub use hashing::hash;
 
diff --git a/eth2/utils/ssz/src/signed_root.rs b/eth2/utils/ssz/src/signed_root.rs
deleted file mode 100644
index f7aeca4af..000000000
--- a/eth2/utils/ssz/src/signed_root.rs
+++ /dev/null
@@ -1,5 +0,0 @@
-use crate::TreeHash;
-
-pub trait SignedRoot: TreeHash {
-    fn signed_root(&self) -> Vec<u8>;
-}
diff --git a/eth2/utils/ssz/src/tree_hash.rs b/eth2/utils/ssz/src/tree_hash.rs
deleted file mode 100644
index 85e56924c..000000000
--- a/eth2/utils/ssz/src/tree_hash.rs
+++ /dev/null
@@ -1,107 +0,0 @@
-use hashing::hash;
-
-const BYTES_PER_CHUNK: usize = 32;
-const HASHSIZE: usize = 32;
-
-pub trait TreeHash {
-    fn hash_tree_root(&self) -> Vec<u8>;
-}
-
-/// Returns a 32 byte hash of 'list' - a vector of byte vectors.
-/// Note that this will consume 'list'.
-pub fn merkle_hash(list: &mut Vec<Vec<u8>>) -> Vec<u8> {
-    // flatten list
-    let mut chunkz = list_to_blob(list);
-
-    // get data_len as bytes. It will hashed will the merkle root
-    let mut datalen = list.len().to_le_bytes().to_vec();
-    zpad(&mut datalen, 32);
-
-    // merklelize
-    while chunkz.len() > HASHSIZE {
-        let mut new_chunkz: Vec<u8> = Vec::new();
-
-        for two_chunks in chunkz.chunks(BYTES_PER_CHUNK * 2) {
-            // Hash two chuncks together
-            new_chunkz.append(&mut hash(two_chunks));
-        }
-
-        chunkz = new_chunkz;
-    }
-
-    chunkz.append(&mut datalen);
-    hash(&chunkz)
-}
-
-fn list_to_blob(list: &mut Vec<Vec<u8>>) -> Vec<u8> {
-    // pack - fit as many many items per chunk as we can and then
-    // right pad to BYTES_PER_CHUNCK
-    let (items_per_chunk, chunk_count) = if list.is_empty() {
-        (1, 1)
-    } else {
-        let items_per_chunk = BYTES_PER_CHUNK / list[0].len();
-        let chunk_count = list.len() / items_per_chunk;
-        (items_per_chunk, chunk_count)
-    };
-
-    let mut chunkz = Vec::new();
-    if list.is_empty() {
-        // handle and empty list
-        chunkz.append(&mut vec![0; BYTES_PER_CHUNK * 2]);
-    } else if list[0].len() <= BYTES_PER_CHUNK {
-        // just create a blob here; we'll divide into
-        // chunked slices when we merklize
-        let mut chunk = Vec::with_capacity(BYTES_PER_CHUNK);
-        let mut item_count_in_chunk = 0;
-        chunkz.reserve(chunk_count * BYTES_PER_CHUNK);
-        for item in list.iter_mut() {
-            item_count_in_chunk += 1;
-            chunk.append(item);
-
-            // completed chunk?
-            if item_count_in_chunk == items_per_chunk {
-                zpad(&mut chunk, BYTES_PER_CHUNK);
-                chunkz.append(&mut chunk);
-                item_count_in_chunk = 0;
-            }
-        }
-
-        // left-over uncompleted chunk?
-        if item_count_in_chunk != 0 {
-            zpad(&mut chunk, BYTES_PER_CHUNK);
-            chunkz.append(&mut chunk);
-        }
-    }
-
-    // extend the number of chunks to a power of two if necessary
-    if !chunk_count.is_power_of_two() {
-        let zero_chunks_count = chunk_count.next_power_of_two() - chunk_count;
-        chunkz.append(&mut vec![0; zero_chunks_count * BYTES_PER_CHUNK]);
-    }
-
-    chunkz
-}
-
-/// right pads with zeros making 'bytes' 'size' in length
-fn zpad(bytes: &mut Vec<u8>, size: usize) {
-    if bytes.len() < size {
-        bytes.resize(size, 0);
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_merkle_hash() {
-        let data1 = vec![1; 32];
-        let data2 = vec![2; 32];
-        let data3 = vec![3; 32];
-        let mut list = vec![data1, data2, data3];
-        let result = merkle_hash(&mut list);
-
-        //note: should test againt a known test hash value
-        assert_eq!(HASHSIZE, result.len());
-    }
-}
diff --git a/eth2/utils/ssz_derive/src/lib.rs b/eth2/utils/ssz_derive/src/lib.rs
index ce2538785..f3475f5a7 100644
--- a/eth2/utils/ssz_derive/src/lib.rs
+++ b/eth2/utils/ssz_derive/src/lib.rs
@@ -188,157 +188,3 @@ pub fn ssz_decode_derive(input: TokenStream) -> TokenStream {
     };
     output.into()
 }
-
-/// Returns a Vec of `syn::Ident` for each named field in the struct, whilst filtering out fields
-/// that should not be tree hashed.
-///
-/// # Panics
-/// Any unnamed struct field (like in a tuple struct) will raise a panic at compile time.
-fn get_tree_hashable_named_field_idents<'a>(
-    struct_data: &'a syn::DataStruct,
-) -> Vec<&'a syn::Ident> {
-    struct_data
-        .fields
-        .iter()
-        .filter_map(|f| {
-            if should_skip_tree_hash(&f) {
-                None
-            } else {
-                Some(match &f.ident {
-                    Some(ref ident) => ident,
-                    _ => panic!("ssz_derive only supports named struct fields."),
-                })
-            }
-        })
-        .collect()
-}
-
-/// Returns true if some field has an attribute declaring it should not be tree-hashed.
-///
-/// The field attribute is: `#[tree_hash(skip_hashing)]`
-fn should_skip_tree_hash(field: &syn::Field) -> bool {
-    for attr in &field.attrs {
-        if attr.into_token_stream().to_string() == "# [ tree_hash ( skip_hashing ) ]" {
-            return true;
-        }
-    }
-    false
-}
-
-/// Implements `ssz::TreeHash` for some `struct`.
-///
-/// Fields are processed in the order they are defined.
-#[proc_macro_derive(TreeHash, attributes(tree_hash))]
-pub fn ssz_tree_hash_derive(input: TokenStream) -> TokenStream {
-    let item = parse_macro_input!(input as DeriveInput);
-
-    let name = &item.ident;
-
-    let struct_data = match &item.data {
-        syn::Data::Struct(s) => s,
-        _ => panic!("ssz_derive only supports structs."),
-    };
-
-    let field_idents = get_tree_hashable_named_field_idents(&struct_data);
-
-    let output = quote! {
-        impl ssz::TreeHash for #name {
-            fn hash_tree_root(&self) -> Vec<u8> {
-                let mut list: Vec<Vec<u8>> = Vec::new();
-                #(
-                    list.push(self.#field_idents.hash_tree_root());
-                )*
-
-                ssz::merkle_hash(&mut list)
-            }
-        }
-    };
-    output.into()
-}
-
-/// Returns `true` if some `Ident` should be considered to be a signature type.
-fn type_ident_is_signature(ident: &syn::Ident) -> bool {
-    match ident.to_string().as_ref() {
-        "Signature" => true,
-        "AggregateSignature" => true,
-        _ => false,
-    }
-}
-
-/// Takes a `Field` where the type (`ty`) portion is a path (e.g., `types::Signature`) and returns
-/// the final `Ident` in that path.
-///
-/// E.g., for `types::Signature` returns `Signature`.
-fn final_type_ident(field: &syn::Field) -> &syn::Ident {
-    match &field.ty {
-        syn::Type::Path(path) => &path.path.segments.last().unwrap().value().ident,
-        _ => panic!("ssz_derive only supports Path types."),
-    }
-}
-
-/// Implements `ssz::TreeHash` for some `struct`, whilst excluding any fields following and
-/// including a field that is of type "Signature" or "AggregateSignature".
-///
-/// See:
-/// https://github.com/ethereum/eth2.0-specs/blob/master/specs/simple-serialize.md#signed-roots
-///
-/// This is a rather horrendous macro, it will read the type of the object as a string and decide
-/// if it's a signature by matching that string against "Signature" or "AggregateSignature". So,
-/// it's important that you use those exact words as your type -- don't alias it to something else.
-///
-/// If you can think of a better way to do this, please make an issue!
-///
-/// Fields are processed in the order they are defined.
-#[proc_macro_derive(SignedRoot, attributes(signed_root))]
-pub fn ssz_signed_root_derive(input: TokenStream) -> TokenStream {
-    let item = parse_macro_input!(input as DeriveInput);
-
-    let name = &item.ident;
-
-    let struct_data = match &item.data {
-        syn::Data::Struct(s) => s,
-        _ => panic!("ssz_derive only supports structs."),
-    };
-
-    let mut field_idents: Vec<&syn::Ident> = vec![];
-
-    let field_idents = get_signed_root_named_field_idents(&struct_data);
-
-    let output = quote! {
-        impl ssz::SignedRoot for #name {
-            fn signed_root(&self) -> Vec<u8> {
-                let mut list: Vec<Vec<u8>> = Vec::new();
-                #(
-                    list.push(self.#field_idents.hash_tree_root());
-                )*
-
-                ssz::merkle_hash(&mut list)
-            }
-        }
-    };
-    output.into()
-}
-
-fn get_signed_root_named_field_idents(struct_data: &syn::DataStruct) -> Vec<&syn::Ident> {
-    struct_data
-        .fields
-        .iter()
-        .filter_map(|f| {
-            if should_skip_signed_root(&f) {
-                None
-            } else {
-                Some(match &f.ident {
-                    Some(ref ident) => ident,
-                    _ => panic!("ssz_derive only supports named struct fields"),
-                })
-            }
-        })
-        .collect()
-}
-
-fn should_skip_signed_root(field: &syn::Field) -> bool {
-    field
-        .attrs
-        .iter()
-        .any(|attr| attr.into_token_stream().to_string() == "# [ signed_root ( skip_hashing ) ]")
-}
diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs
index ac7e7633d..7c74c9f97 100644
--- a/eth2/utils/tree_hash/src/lib.rs
+++ b/eth2/utils/tree_hash/src/lib.rs
@@ -8,7 +8,7 @@ pub const MERKLE_HASH_CHUNCK: usize = 2 * BYTES_PER_CHUNK;
 
 pub use cached_tree_hash::{BTreeOverlay, CachedTreeHashSubTree, Error, TreeHashCache};
 pub use signed_root::SignedRoot;
-pub use standard_tree_hash::{efficient_merkleize, TreeHash};
+pub use standard_tree_hash::{merkle_root, TreeHash};
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum TreeHashType {
@@ -25,3 +25,26 @@ fn num_sanitized_leaves(num_bytes: usize) -> usize {
 fn num_nodes(num_leaves: usize) -> usize {
     2 * num_leaves - 1
 }
+
+#[macro_export]
+macro_rules! impl_tree_hash_for_ssz_bytes {
+    ($type: ident) => {
+        impl tree_hash::TreeHash for $type {
+            fn tree_hash_type() -> tree_hash::TreeHashType {
+                tree_hash::TreeHashType::List
+            }
+
+            fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+                panic!("bytesN should never be packed.")
+            }
+
+            fn tree_hash_packing_factor() -> usize {
+                panic!("bytesN should never be packed.")
+            }
+
+            fn tree_hash_root(&self) -> Vec<u8> {
+                tree_hash::merkle_root(&ssz::ssz_encode(self))
+            }
+        }
+    };
+}
diff --git a/eth2/utils/tree_hash/src/standard_tree_hash.rs b/eth2/utils/tree_hash/src/standard_tree_hash.rs
index e7f94560b..ea0677180 100644
--- a/eth2/utils/tree_hash/src/standard_tree_hash.rs
+++ b/eth2/utils/tree_hash/src/standard_tree_hash.rs
@@ -3,6 +3,8 @@ use hashing::hash;
 use int_to_bytes::int_to_bytes32;
 use ssz::ssz_encode;
 
+mod impls;
+
 pub trait TreeHash {
     fn tree_hash_type() -> TreeHashType;
 
@@ -13,70 +15,9 @@ pub trait TreeHash {
     fn tree_hash_root(&self) -> Vec<u8>;
 }
 
-impl TreeHash for u64 {
-    fn tree_hash_type() -> TreeHashType {
-        TreeHashType::Basic
-    }
-
-    fn tree_hash_packed_encoding(&self) -> Vec<u8> {
-        ssz_encode(self)
-    }
-
-    fn tree_hash_packing_factor() -> usize {
-        HASHSIZE / 8
-    }
-
-    fn tree_hash_root(&self) -> Vec<u8> {
-        int_to_bytes32(*self)
-    }
-}
-
-impl<T> TreeHash for Vec<T>
-where
-    T: TreeHash,
-{
-    fn tree_hash_type() -> TreeHashType {
-        TreeHashType::List
-    }
-
-    fn tree_hash_packed_encoding(&self) -> Vec<u8> {
-        unreachable!("List should never be packed.")
-    }
-
-    fn tree_hash_packing_factor() -> usize {
-        unreachable!("List should never be packed.")
-    }
-
-    fn tree_hash_root(&self) -> Vec<u8> {
-        let leaves = match T::tree_hash_type() {
-            TreeHashType::Basic => {
-                let mut leaves =
-                    Vec::with_capacity((HASHSIZE / T::tree_hash_packing_factor()) * self.len());
-
-                for item in self {
-                    leaves.append(&mut item.tree_hash_packed_encoding());
-                }
-
-                leaves
-            }
-            TreeHashType::Composite | TreeHashType::List => {
-                let mut leaves = Vec::with_capacity(self.len() * HASHSIZE);
-
-                for item in self {
-                    leaves.append(&mut item.tree_hash_root())
-                }
-
-                leaves
-            }
-        };
-
-        // Mix in the length
-        let mut root_and_len = Vec::with_capacity(HASHSIZE * 2);
-        root_and_len.append(&mut efficient_merkleize(&leaves)[0..32].to_vec());
-        root_and_len.append(&mut int_to_bytes32(self.len() as u64));
-
-        hash(&root_and_len)
-    }
+pub fn merkle_root(bytes: &[u8]) -> Vec<u8> {
+    // TODO: replace this with a _more_ efficient fn which is more memory efficient.
+    efficient_merkleize(&bytes)[0..32].to_vec()
 }
 
 pub fn efficient_merkleize(bytes: &[u8]) -> Vec<u8> {
diff --git a/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs b/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs
new file mode 100644
index 000000000..070e314b8
--- /dev/null
+++ b/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs
@@ -0,0 +1,97 @@
+use super::*;
+use ethereum_types::H256;
+
+macro_rules! impl_for_bitsize {
+    ($type: ident, $bit_size: expr) => {
+        impl TreeHash for $type {
+            fn tree_hash_type() -> TreeHashType {
+                TreeHashType::Basic
+            }
+
+            fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+                ssz_encode(self)
+            }
+
+            fn tree_hash_packing_factor() -> usize {
+                HASHSIZE / ($bit_size / 8)
+            }
+
+            fn tree_hash_root(&self) -> Vec<u8> {
+                int_to_bytes32(*self as u64)
+            }
+        }
+    };
+}
+
+impl_for_bitsize!(u8, 8);
+impl_for_bitsize!(u16, 16);
+impl_for_bitsize!(u32, 32);
+impl_for_bitsize!(u64, 64);
+impl_for_bitsize!(usize, 64);
+impl_for_bitsize!(bool, 8);
+
+impl TreeHash for H256 {
+    fn tree_hash_type() -> TreeHashType {
+        TreeHashType::Basic
+    }
+
+    fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+        ssz_encode(self)
+    }
+
+    fn tree_hash_packing_factor() -> usize {
+        1
+    }
+
+    fn tree_hash_root(&self) -> Vec<u8> {
+        ssz_encode(self)
+    }
+}
+
+impl<T> TreeHash for Vec<T>
+where
+    T: TreeHash,
+{
+    fn tree_hash_type() -> TreeHashType {
+        TreeHashType::List
+    }
+
+    fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+        unreachable!("List should never be packed.")
+    }
+
+    fn tree_hash_packing_factor() -> usize {
+        unreachable!("List should never be packed.")
+    }
+
+    fn tree_hash_root(&self) -> Vec<u8> {
+        let leaves = match T::tree_hash_type() {
+            TreeHashType::Basic => {
+                let mut leaves =
+                    Vec::with_capacity((HASHSIZE / T::tree_hash_packing_factor()) * self.len());
+
+                for item in self {
+                    leaves.append(&mut item.tree_hash_packed_encoding());
+                }
+
+                leaves
+            }
+            TreeHashType::Composite | TreeHashType::List => {
+                let mut leaves = Vec::with_capacity(self.len() * HASHSIZE);
+
+                for item in self {
+                    leaves.append(&mut item.tree_hash_root())
+                }
+
+                leaves
+            }
+        };
+
+        // Mix in the length
+        let mut root_and_len = Vec::with_capacity(HASHSIZE * 2);
+        root_and_len.append(&mut merkle_root(&leaves));
+        root_and_len.append(&mut int_to_bytes32(self.len() as u64));
+
+        hash(&root_and_len)
+    }
+}
diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs
index ff5bc0d47..dc3702c72 100644
--- a/eth2/utils/tree_hash_derive/src/lib.rs
+++ b/eth2/utils/tree_hash_derive/src/lib.rs
@@ -149,7 +149,7 @@ pub fn tree_hash_derive(input: TokenStream) -> TokenStream {
                     leaves.append(&mut self.#idents.tree_hash_root());
                 )*
 
-                tree_hash::efficient_merkleize(&leaves)[0..32].to_vec()
+                tree_hash::merkle_root(&leaves)
             }
         }
     };
@@ -191,7 +191,7 @@ pub fn tree_hash_signed_root_derive(input: TokenStream) -> TokenStream {
                     leaves.append(&mut self.#idents.tree_hash_root());
                 )*
 
-                tree_hash::efficient_merkleize(&leaves)[0..32].to_vec()
+                tree_hash::merkle_root(&leaves)
             }
         }
     };

From b8c4c3308a4affc2196bd56a1666965d20e2813c Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Tue, 16 Apr 2019 14:14:38 +1000
Subject: [PATCH 057/137] Update `types` to new tree_hash crate

---
 beacon_node/beacon_chain/src/initialise.rs    |  6 +-
 .../testing_beacon_chain_builder.rs           |  4 +-
 .../test_harness/src/beacon_chain_harness.rs  |  6 +-
 .../test_harness/src/test_case.rs             |  2 +-
 beacon_node/network/src/sync/import_queue.rs  | 10 +-
 beacon_node/network/src/sync/simple_sync.rs   |  4 +-
 eth2/attester/src/lib.rs                      |  5 +-
 eth2/block_proposer/src/lib.rs                |  2 +-
 .../benches/bench_block_processing.rs         |  4 +-
 .../benches/bench_epoch_processing.rs         |  4 +-
 .../state_processing/src/get_genesis_state.rs |  4 +-
 .../src/per_block_processing.rs               |  2 +-
 .../validate_attestation.rs                   |  6 +-
 .../src/per_block_processing/verify_exit.rs   |  2 +-
 .../verify_proposer_slashing.rs               |  2 +-
 .../verify_slashable_attestation.rs           |  6 +-
 .../per_block_processing/verify_transfer.rs   |  2 +-
 .../src/per_epoch_processing.rs               |  6 +-
 .../src/per_slot_processing.rs                |  6 +-
 eth2/types/Cargo.toml                         |  2 +
 eth2/types/src/attestation.rs                 |  5 +-
 eth2/types/src/attestation_data.rs            |  5 +-
 .../src/attestation_data_and_custody_bit.rs   |  3 +-
 eth2/types/src/attester_slashing.rs           |  3 +-
 eth2/types/src/beacon_block.rs                | 11 ++-
 eth2/types/src/beacon_block_body.rs           |  3 +-
 eth2/types/src/beacon_block_header.rs         |  9 +-
 eth2/types/src/beacon_state.rs                | 10 +-
 eth2/types/src/crosslink.rs                   |  3 +-
 eth2/types/src/crosslink_committee.rs         |  3 +-
 eth2/types/src/deposit.rs                     |  3 +-
 eth2/types/src/deposit_data.rs                |  3 +-
 eth2/types/src/deposit_input.rs               |  5 +-
 eth2/types/src/epoch_cache.rs                 |  0
 eth2/types/src/eth1_data.rs                   |  3 +-
 eth2/types/src/eth1_data_vote.rs              |  3 +-
 eth2/types/src/fork.rs                        |  3 +-
 eth2/types/src/historical_batch.rs            |  3 +-
 eth2/types/src/pending_attestation.rs         |  3 +-
 eth2/types/src/proposer_slashing.rs           |  3 +-
 eth2/types/src/slashable_attestation.rs       |  5 +-
 eth2/types/src/slot_epoch.rs                  |  2 +-
 eth2/types/src/slot_epoch_macros.rs           | 20 +++-
 eth2/types/src/slot_height.rs                 |  2 +-
 eth2/types/src/test_utils/macros.rs           |  6 +-
 .../test_utils/testing_attestation_builder.rs |  4 +-
 .../testing_attester_slashing_builder.rs      |  4 +-
 .../testing_beacon_block_builder.rs           |  4 +-
 .../testing_proposer_slashing_builder.rs      |  2 +-
 .../test_utils/testing_transfer_builder.rs    |  2 +-
 .../testing_voluntary_exit_builder.rs         |  2 +-
 eth2/types/src/transfer.rs                    |  5 +-
 eth2/types/src/validator.rs                   |  3 +-
 eth2/types/src/voluntary_exit.rs              |  5 +-
 eth2/utils/bls/Cargo.toml                     |  1 +
 eth2/utils/bls/src/aggregate_signature.rs     |  2 +-
 .../utils/bls/src/fake_aggregate_signature.rs |  9 +-
 eth2/utils/bls/src/fake_signature.rs          |  9 +-
 eth2/utils/bls/src/public_key.rs              |  9 +-
 eth2/utils/bls/src/secret_key.rs              |  9 +-
 eth2/utils/bls/src/signature.rs               |  2 +-
 eth2/utils/boolean-bitfield/Cargo.toml        |  1 +
 eth2/utils/boolean-bitfield/src/lib.rs        |  7 +-
 eth2/utils/ssz_derive/tests/test_derives.rs   | 94 -------------------
 .../utils/tree_hash/src/standard_tree_hash.rs |  7 +-
 .../tree_hash/src/standard_tree_hash/impls.rs | 35 +++++++
 eth2/utils/tree_hash_derive/src/lib.rs        | 10 +-
 eth2/utils/tree_hash_derive/tests/tests.rs    | 82 ++++++++++++++++
 .../src/attestation_producer/mod.rs           |  4 +-
 validator_client/src/block_producer/mod.rs    |  2 +-
 70 files changed, 284 insertions(+), 234 deletions(-)
 delete mode 100644 eth2/types/src/epoch_cache.rs
 delete mode 100644 eth2/utils/ssz_derive/tests/test_derives.rs

diff --git a/beacon_node/beacon_chain/src/initialise.rs b/beacon_node/beacon_chain/src/initialise.rs
index 0951e06fb..c66dd63b1 100644
--- a/beacon_node/beacon_chain/src/initialise.rs
+++ b/beacon_node/beacon_chain/src/initialise.rs
@@ -7,9 +7,9 @@ use db::stores::{BeaconBlockStore, BeaconStateStore};
 use db::{DiskDB, MemoryDB};
 use fork_choice::BitwiseLMDGhost;
 use slot_clock::SystemTimeSlotClock;
-use ssz::TreeHash;
 use std::path::PathBuf;
 use std::sync::Arc;
+use tree_hash::TreeHash;
 use types::test_utils::TestingBeaconStateBuilder;
 use types::{BeaconBlock, ChainSpec, Hash256};
 
@@ -32,7 +32,7 @@ pub fn initialise_beacon_chain(
     let (genesis_state, _keypairs) = state_builder.build();
 
     let mut genesis_block = BeaconBlock::empty(&spec);
-    genesis_block.state_root = Hash256::from_slice(&genesis_state.hash_tree_root());
+    genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root());
 
     // Slot clock
     let slot_clock = SystemTimeSlotClock::new(
@@ -73,7 +73,7 @@ pub fn initialise_test_beacon_chain(
     let (genesis_state, _keypairs) = state_builder.build();
 
     let mut genesis_block = BeaconBlock::empty(spec);
-    genesis_block.state_root = Hash256::from_slice(&genesis_state.hash_tree_root());
+    genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root());
 
     // Slot clock
     let slot_clock = SystemTimeSlotClock::new(
diff --git a/beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs b/beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs
index 5c5477e55..d174670c0 100644
--- a/beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs
+++ b/beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs
@@ -5,8 +5,8 @@ use db::{
 };
 use fork_choice::BitwiseLMDGhost;
 use slot_clock::TestingSlotClock;
-use ssz::TreeHash;
 use std::sync::Arc;
+use tree_hash::TreeHash;
 use types::test_utils::TestingBeaconStateBuilder;
 use types::*;
 
@@ -27,7 +27,7 @@ impl TestingBeaconChainBuilder {
         let (genesis_state, _keypairs) = self.state_builder.build();
 
         let mut genesis_block = BeaconBlock::empty(&spec);
-        genesis_block.state_root = Hash256::from_slice(&genesis_state.hash_tree_root());
+        genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root());
 
         // Create the Beacon Chain
         BeaconChain::from_genesis(
diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs
index aeb734a4e..34b559478 100644
--- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs
+++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs
@@ -9,8 +9,8 @@ use fork_choice::BitwiseLMDGhost;
 use log::debug;
 use rayon::prelude::*;
 use slot_clock::TestingSlotClock;
-use ssz::TreeHash;
 use std::sync::Arc;
+use tree_hash::TreeHash;
 use types::{test_utils::TestingBeaconStateBuilder, *};
 
 type TestingBeaconChain = BeaconChain<MemoryDB, TestingSlotClock, BitwiseLMDGhost<MemoryDB>>;
@@ -54,7 +54,7 @@ impl BeaconChainHarness {
         let (mut genesis_state, keypairs) = state_builder.build();
 
         let mut genesis_block = BeaconBlock::empty(&spec);
-        genesis_block.state_root = Hash256::from_slice(&genesis_state.hash_tree_root());
+        genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root());
 
         genesis_state
             .build_epoch_cache(RelativeEpoch::Previous, &spec)
@@ -163,7 +163,7 @@ impl BeaconChainHarness {
                         data: data.clone(),
                         custody_bit: false,
                     }
-                    .hash_tree_root();
+                    .tree_hash_root();
                     let domain = self.spec.get_domain(
                         state.slot.epoch(self.spec.slots_per_epoch),
                         Domain::Attestation,
diff --git a/beacon_node/beacon_chain/test_harness/src/test_case.rs b/beacon_node/beacon_chain/test_harness/src/test_case.rs
index f65b45505..28c7ae8a8 100644
--- a/beacon_node/beacon_chain/test_harness/src/test_case.rs
+++ b/beacon_node/beacon_chain/test_harness/src/test_case.rs
@@ -4,7 +4,7 @@
 use crate::beacon_chain_harness::BeaconChainHarness;
 use beacon_chain::CheckPoint;
 use log::{info, warn};
-use ssz::SignedRoot;
+use tree_hash::SignedRoot;
 use types::*;
 
 use types::test_utils::*;
diff --git a/beacon_node/network/src/sync/import_queue.rs b/beacon_node/network/src/sync/import_queue.rs
index 0026347eb..106e3eb66 100644
--- a/beacon_node/network/src/sync/import_queue.rs
+++ b/beacon_node/network/src/sync/import_queue.rs
@@ -2,9 +2,9 @@ use crate::beacon_chain::BeaconChain;
 use eth2_libp2p::rpc::methods::*;
 use eth2_libp2p::PeerId;
 use slog::{debug, error};
-use ssz::TreeHash;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
+use tree_hash::TreeHash;
 use types::{BeaconBlock, BeaconBlockBody, BeaconBlockHeader, Hash256, Slot};
 
 /// Provides a queue for fully and partially built `BeaconBlock`s.
@@ -15,7 +15,7 @@ use types::{BeaconBlock, BeaconBlockBody, BeaconBlockHeader, Hash256, Slot};
 ///
 /// - When we receive a `BeaconBlockBody`, the only way we can find it's matching
 /// `BeaconBlockHeader` is to find a header such that `header.beacon_block_body ==
-/// hash_tree_root(body)`. Therefore, if we used a `HashMap` we would need to use the root of
+/// tree_hash_root(body)`. Therefore, if we used a `HashMap` we would need to use the root of
 /// `BeaconBlockBody` as the key.
 /// - It is possible for multiple distinct blocks to have identical `BeaconBlockBodies`. Therefore
 /// we cannot use a `HashMap` keyed by the root of `BeaconBlockBody`.
@@ -166,7 +166,7 @@ impl ImportQueue {
         let mut required_bodies: Vec<Hash256> = vec![];
 
         for header in headers {
-            let block_root = Hash256::from_slice(&header.hash_tree_root()[..]);
+            let block_root = Hash256::from_slice(&header.tree_hash_root()[..]);
 
             if self.chain_has_not_seen_block(&block_root) {
                 self.insert_header(block_root, header, sender.clone());
@@ -230,7 +230,7 @@ impl ImportQueue {
     ///
     /// If the body already existed, the `inserted` time is set to `now`.
     fn insert_body(&mut self, body: BeaconBlockBody, sender: PeerId) {
-        let body_root = Hash256::from_slice(&body.hash_tree_root()[..]);
+        let body_root = Hash256::from_slice(&body.tree_hash_root()[..]);
 
         self.partials.iter_mut().for_each(|mut p| {
             if let Some(header) = &mut p.header {
@@ -250,7 +250,7 @@ impl ImportQueue {
     ///
     /// If the partial already existed, the `inserted` time is set to `now`.
     fn insert_full_block(&mut self, block: BeaconBlock, sender: PeerId) {
-        let block_root = Hash256::from_slice(&block.hash_tree_root()[..]);
+        let block_root = Hash256::from_slice(&block.tree_hash_root()[..]);
 
         let partial = PartialBeaconBlock {
             slot: block.slot,
diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs
index 824458b89..1b57fbc00 100644
--- a/beacon_node/network/src/sync/simple_sync.rs
+++ b/beacon_node/network/src/sync/simple_sync.rs
@@ -5,10 +5,10 @@ use eth2_libp2p::rpc::methods::*;
 use eth2_libp2p::rpc::{RPCRequest, RPCResponse, RequestId};
 use eth2_libp2p::PeerId;
 use slog::{debug, error, info, o, warn};
-use ssz::TreeHash;
 use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::Duration;
+use tree_hash::TreeHash;
 use types::{Attestation, BeaconBlock, Epoch, Hash256, Slot};
 
 /// The number of slots that we can import blocks ahead of us, before going into full Sync mode.
@@ -565,7 +565,7 @@ impl SimpleSync {
             return false;
         }
 
-        let block_root = Hash256::from_slice(&block.hash_tree_root());
+        let block_root = Hash256::from_slice(&block.tree_hash_root());
 
         // Ignore any block that the chain already knows about.
         if self.chain_has_seen_block(&block_root) {
diff --git a/eth2/attester/src/lib.rs b/eth2/attester/src/lib.rs
index a4295f005..a9e3091af 100644
--- a/eth2/attester/src/lib.rs
+++ b/eth2/attester/src/lib.rs
@@ -2,8 +2,8 @@ pub mod test_utils;
 mod traits;
 
 use slot_clock::SlotClock;
-use ssz::TreeHash;
 use std::sync::Arc;
+use tree_hash::TreeHash;
 use types::{AttestationData, AttestationDataAndCustodyBit, FreeAttestation, Signature, Slot};
 
 pub use self::traits::{
@@ -141,7 +141,8 @@ impl<T: SlotClock, U: BeaconNode, V: DutiesReader, W: Signer> Attester<T, U, V,
             data: attestation_data.clone(),
             custody_bit: PHASE_0_CUSTODY_BIT,
         }
-        .hash_tree_root();
+        .tree_hash_root
+            | update();
 
         self.signer
             .sign_attestation_message(&message[..], DOMAIN_ATTESTATION)
diff --git a/eth2/block_proposer/src/lib.rs b/eth2/block_proposer/src/lib.rs
index e62c4b71d..e98467b16 100644
--- a/eth2/block_proposer/src/lib.rs
+++ b/eth2/block_proposer/src/lib.rs
@@ -139,7 +139,7 @@ impl<T: SlotClock, U: BeaconNode, V: DutiesReader, W: Signer> BlockProducer<T, U
 
         let randao_reveal = {
             // TODO: add domain, etc to this message. Also ensure result matches `into_to_bytes32`.
-            let message = slot.epoch(self.spec.slots_per_epoch).hash_tree_root();
+            let message = slot.epoch(self.spec.slots_per_epoch).tree_hash_root();
 
             match self.signer.sign_randao_reveal(
                 &message,
diff --git a/eth2/state_processing/benches/bench_block_processing.rs b/eth2/state_processing/benches/bench_block_processing.rs
index 2ee08c96a..80be1828f 100644
--- a/eth2/state_processing/benches/bench_block_processing.rs
+++ b/eth2/state_processing/benches/bench_block_processing.rs
@@ -1,6 +1,5 @@
 use criterion::Criterion;
 use criterion::{black_box, Benchmark};
-use ssz::TreeHash;
 use state_processing::{
     per_block_processing,
     per_block_processing::{
@@ -9,6 +8,7 @@ use state_processing::{
         verify_block_signature,
     },
 };
+use tree_hash::TreeHash;
 use types::*;
 
 /// Run the detailed benchmarking suite on the given `BeaconState`.
@@ -263,7 +263,7 @@ pub fn bench_block_processing(
     c.bench(
         &format!("{}/block_processing", desc),
         Benchmark::new("tree_hash_block", move |b| {
-            b.iter(|| black_box(block.hash_tree_root()))
+            b.iter(|| black_box(block.tree_hash_root()))
         })
         .sample_size(10),
     );
diff --git a/eth2/state_processing/benches/bench_epoch_processing.rs b/eth2/state_processing/benches/bench_epoch_processing.rs
index cc7701296..9bff3a2e3 100644
--- a/eth2/state_processing/benches/bench_epoch_processing.rs
+++ b/eth2/state_processing/benches/bench_epoch_processing.rs
@@ -1,6 +1,5 @@
 use criterion::Criterion;
 use criterion::{black_box, Benchmark};
-use ssz::TreeHash;
 use state_processing::{
     per_epoch_processing,
     per_epoch_processing::{
@@ -9,6 +8,7 @@ use state_processing::{
         update_active_tree_index_roots, update_latest_slashed_balances,
     },
 };
+use tree_hash::TreeHash;
 use types::test_utils::TestingBeaconStateBuilder;
 use types::*;
 
@@ -256,7 +256,7 @@ fn bench_epoch_processing(c: &mut Criterion, state: &BeaconState, spec: &ChainSp
     c.bench(
         &format!("{}/epoch_processing", desc),
         Benchmark::new("tree_hash_state", move |b| {
-            b.iter(|| black_box(state_clone.hash_tree_root()))
+            b.iter(|| black_box(state_clone.tree_hash_root()))
         })
         .sample_size(SMALL_BENCHING_SAMPLE_SIZE),
     );
diff --git a/eth2/state_processing/src/get_genesis_state.rs b/eth2/state_processing/src/get_genesis_state.rs
index 7c4d4cafd..21cdafcf9 100644
--- a/eth2/state_processing/src/get_genesis_state.rs
+++ b/eth2/state_processing/src/get_genesis_state.rs
@@ -1,5 +1,5 @@
 use super::per_block_processing::{errors::BlockProcessingError, process_deposits};
-use ssz::TreeHash;
+use tree_hash::TreeHash;
 use types::*;
 
 pub enum GenesisError {
@@ -36,7 +36,7 @@ pub fn get_genesis_state(
     let active_validator_indices = state
         .get_cached_active_validator_indices(RelativeEpoch::Current, spec)?
         .to_vec();
-    let genesis_active_index_root = Hash256::from_slice(&active_validator_indices.hash_tree_root());
+    let genesis_active_index_root = Hash256::from_slice(&active_validator_indices.tree_hash_root());
     state.fill_active_index_roots_with(genesis_active_index_root, spec);
 
     // Generate the current shuffling seed.
diff --git a/eth2/state_processing/src/per_block_processing.rs b/eth2/state_processing/src/per_block_processing.rs
index 6c52a2676..32dc71a2e 100644
--- a/eth2/state_processing/src/per_block_processing.rs
+++ b/eth2/state_processing/src/per_block_processing.rs
@@ -162,7 +162,7 @@ pub fn process_randao(
     // Verify the RANDAO is a valid signature of the proposer.
     verify!(
         block.body.randao_reveal.verify(
-            &state.current_epoch(spec).hash_tree_root()[..],
+            &state.current_epoch(spec).tree_hash_root()[..],
             spec.get_domain(
                 block.slot.epoch(spec.slots_per_epoch),
                 Domain::Randao,
diff --git a/eth2/state_processing/src/per_block_processing/validate_attestation.rs b/eth2/state_processing/src/per_block_processing/validate_attestation.rs
index 3b89bec99..c9d0b38a4 100644
--- a/eth2/state_processing/src/per_block_processing/validate_attestation.rs
+++ b/eth2/state_processing/src/per_block_processing/validate_attestation.rs
@@ -1,6 +1,6 @@
 use super::errors::{AttestationInvalid as Invalid, AttestationValidationError as Error};
 use crate::common::verify_bitfield_length;
-use ssz::TreeHash;
+use tree_hash::TreeHash;
 use types::*;
 
 /// Indicates if an `Attestation` is valid to be included in a block in the current epoch of the
@@ -270,14 +270,14 @@ fn verify_attestation_signature(
         data: a.data.clone(),
         custody_bit: false,
     }
-    .hash_tree_root();
+    .tree_hash_root();
 
     // Message when custody bitfield is `true`
     let message_1 = AttestationDataAndCustodyBit {
         data: a.data.clone(),
         custody_bit: true,
     }
-    .hash_tree_root();
+    .tree_hash_root();
 
     let mut messages = vec![];
     let mut keys = vec![];
diff --git a/eth2/state_processing/src/per_block_processing/verify_exit.rs b/eth2/state_processing/src/per_block_processing/verify_exit.rs
index a3b694395..c5b8ebcb4 100644
--- a/eth2/state_processing/src/per_block_processing/verify_exit.rs
+++ b/eth2/state_processing/src/per_block_processing/verify_exit.rs
@@ -1,5 +1,5 @@
 use super::errors::{ExitInvalid as Invalid, ExitValidationError as Error};
-use ssz::SignedRoot;
+use tree_hash::SignedRoot;
 use types::*;
 
 /// Indicates if an `Exit` is valid to be included in a block in the current epoch of the given
diff --git a/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs b/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs
index dffb9d898..8e0a70f96 100644
--- a/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs
+++ b/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs
@@ -1,5 +1,5 @@
 use super::errors::{ProposerSlashingInvalid as Invalid, ProposerSlashingValidationError as Error};
-use ssz::SignedRoot;
+use tree_hash::SignedRoot;
 use types::*;
 
 /// Indicates if a `ProposerSlashing` is valid to be included in a block in the current epoch of the given
diff --git a/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs b/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs
index d3ab5e398..d39ac6759 100644
--- a/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs
+++ b/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs
@@ -2,7 +2,7 @@ use super::errors::{
     SlashableAttestationInvalid as Invalid, SlashableAttestationValidationError as Error,
 };
 use crate::common::verify_bitfield_length;
-use ssz::TreeHash;
+use tree_hash::TreeHash;
 use types::*;
 
 /// Indicates if a `SlashableAttestation` is valid to be included in a block in the current epoch of the given
@@ -77,12 +77,12 @@ pub fn verify_slashable_attestation(
         data: slashable_attestation.data.clone(),
         custody_bit: false,
     }
-    .hash_tree_root();
+    .tree_hash_root();
     let message_1 = AttestationDataAndCustodyBit {
         data: slashable_attestation.data.clone(),
         custody_bit: true,
     }
-    .hash_tree_root();
+    .tree_hash_root();
 
     let mut messages = vec![];
     let mut keys = vec![];
diff --git a/eth2/state_processing/src/per_block_processing/verify_transfer.rs b/eth2/state_processing/src/per_block_processing/verify_transfer.rs
index ac9e9aa09..978d0cfce 100644
--- a/eth2/state_processing/src/per_block_processing/verify_transfer.rs
+++ b/eth2/state_processing/src/per_block_processing/verify_transfer.rs
@@ -1,6 +1,6 @@
 use super::errors::{TransferInvalid as Invalid, TransferValidationError as Error};
 use bls::get_withdrawal_credentials;
-use ssz::SignedRoot;
+use tree_hash::SignedRoot;
 use types::*;
 
 /// Indicates if a `Transfer` is valid to be included in a block in the current epoch of the given
diff --git a/eth2/state_processing/src/per_epoch_processing.rs b/eth2/state_processing/src/per_epoch_processing.rs
index fcdc668f4..f4d6452a4 100644
--- a/eth2/state_processing/src/per_epoch_processing.rs
+++ b/eth2/state_processing/src/per_epoch_processing.rs
@@ -3,8 +3,8 @@ use errors::EpochProcessingError as Error;
 use process_ejections::process_ejections;
 use process_exit_queue::process_exit_queue;
 use process_slashings::process_slashings;
-use ssz::TreeHash;
 use std::collections::HashMap;
+use tree_hash::TreeHash;
 use types::*;
 use update_registry_and_shuffling_data::update_registry_and_shuffling_data;
 use validator_statuses::{TotalBalances, ValidatorStatuses};
@@ -236,7 +236,7 @@ pub fn finish_epoch_update(state: &mut BeaconState, spec: &ChainSpec) -> Result<
         let active_index_root = Hash256::from_slice(
             &state
                 .get_active_validator_indices(next_epoch + spec.activation_exit_delay)
-                .hash_tree_root()[..],
+                .tree_hash_root()[..],
         );
         state.set_active_index_root(next_epoch, active_index_root, spec)?;
 
@@ -261,7 +261,7 @@ pub fn finish_epoch_update(state: &mut BeaconState, spec: &ChainSpec) -> Result<
         let historical_batch: HistoricalBatch = state.historical_batch();
         state
             .historical_roots
-            .push(Hash256::from_slice(&historical_batch.hash_tree_root()[..]));
+            .push(Hash256::from_slice(&historical_batch.tree_hash_root()[..]));
     }
 
     state.previous_epoch_attestations = state.current_epoch_attestations.clone();
diff --git a/eth2/state_processing/src/per_slot_processing.rs b/eth2/state_processing/src/per_slot_processing.rs
index c6b5312c7..cd129a5f1 100644
--- a/eth2/state_processing/src/per_slot_processing.rs
+++ b/eth2/state_processing/src/per_slot_processing.rs
@@ -1,5 +1,5 @@
 use crate::*;
-use ssz::TreeHash;
+use tree_hash::TreeHash;
 use types::*;
 
 #[derive(Debug, PartialEq)]
@@ -32,7 +32,7 @@ fn cache_state(
     latest_block_header: &BeaconBlockHeader,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
-    let previous_slot_state_root = Hash256::from_slice(&state.hash_tree_root()[..]);
+    let previous_slot_state_root = Hash256::from_slice(&state.tree_hash_root()[..]);
 
     // Note: increment the state slot here to allow use of our `state_root` and `block_root`
     // getter/setter functions.
@@ -46,7 +46,7 @@ fn cache_state(
         state.latest_block_header.state_root = previous_slot_state_root
     }
 
-    let latest_block_root = Hash256::from_slice(&latest_block_header.hash_tree_root()[..]);
+    let latest_block_root = Hash256::from_slice(&latest_block_header.tree_hash_root()[..]);
     state.set_block_root(previous_slot, latest_block_root, spec)?;
 
     // Set the state slot back to what it should be.
diff --git a/eth2/types/Cargo.toml b/eth2/types/Cargo.toml
index 613eb7936..b88e1d4cf 100644
--- a/eth2/types/Cargo.toml
+++ b/eth2/types/Cargo.toml
@@ -26,6 +26,8 @@ ssz = { path = "../utils/ssz" }
 ssz_derive = { path = "../utils/ssz_derive" }
 swap_or_not_shuffle = { path = "../utils/swap_or_not_shuffle" }
 test_random_derive = { path = "../utils/test_random_derive" }
+tree_hash = { path = "../utils/tree_hash" }
+tree_hash_derive = { path = "../utils/tree_hash_derive" }
 libp2p =  { git = "https://github.com/SigP/rust-libp2p", rev = "b3c32d9a821ae6cc89079499cc6e8a6bab0bffc3" }
 
 [dev-dependencies]
diff --git a/eth2/types/src/attestation.rs b/eth2/types/src/attestation.rs
index a8eeea909..c43692a7b 100644
--- a/eth2/types/src/attestation.rs
+++ b/eth2/types/src/attestation.rs
@@ -2,9 +2,10 @@ use super::{AggregateSignature, AttestationData, Bitfield};
 use crate::test_utils::TestRandom;
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
-use ssz::TreeHash;
-use ssz_derive::{Decode, Encode, SignedRoot, TreeHash};
+use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
+use tree_hash::TreeHash;
+use tree_hash_derive::{SignedRoot, TreeHash};
 
 /// Details an attestation that can be slashable.
 ///
diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data.rs
index 4a6b57823..305ddafe0 100644
--- a/eth2/types/src/attestation_data.rs
+++ b/eth2/types/src/attestation_data.rs
@@ -2,9 +2,10 @@ use crate::test_utils::TestRandom;
 use crate::{Crosslink, Epoch, Hash256, Slot};
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
-use ssz::TreeHash;
-use ssz_derive::{Decode, Encode, SignedRoot, TreeHash};
+use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
+use tree_hash::TreeHash;
+use tree_hash_derive::{SignedRoot, TreeHash};
 
 /// The data upon which an attestation is based.
 ///
diff --git a/eth2/types/src/attestation_data_and_custody_bit.rs b/eth2/types/src/attestation_data_and_custody_bit.rs
index 2cc6bc80c..59a4eee77 100644
--- a/eth2/types/src/attestation_data_and_custody_bit.rs
+++ b/eth2/types/src/attestation_data_and_custody_bit.rs
@@ -2,7 +2,8 @@ use super::AttestationData;
 use crate::test_utils::TestRandom;
 use rand::RngCore;
 use serde_derive::Serialize;
-use ssz_derive::{Decode, Encode, TreeHash};
+use ssz_derive::{Decode, Encode};
+use tree_hash_derive::TreeHash;
 
 /// Used for pairing an attestation with a proof-of-custody.
 ///
diff --git a/eth2/types/src/attester_slashing.rs b/eth2/types/src/attester_slashing.rs
index 6fc404f42..0600e0ecc 100644
--- a/eth2/types/src/attester_slashing.rs
+++ b/eth2/types/src/attester_slashing.rs
@@ -1,8 +1,9 @@
 use crate::{test_utils::TestRandom, SlashableAttestation};
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
-use ssz_derive::{Decode, Encode, TreeHash};
+use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
+use tree_hash_derive::TreeHash;
 
 /// Two conflicting attestations.
 ///
diff --git a/eth2/types/src/beacon_block.rs b/eth2/types/src/beacon_block.rs
index 77c1620f3..bc6ccb0d5 100644
--- a/eth2/types/src/beacon_block.rs
+++ b/eth2/types/src/beacon_block.rs
@@ -3,9 +3,10 @@ use crate::*;
 use bls::Signature;
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
-use ssz::TreeHash;
-use ssz_derive::{Decode, Encode, SignedRoot, TreeHash};
+use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
+use tree_hash::TreeHash;
+use tree_hash_derive::{SignedRoot, TreeHash};
 
 /// A block of the `BeaconChain`.
 ///
@@ -57,11 +58,11 @@ impl BeaconBlock {
         }
     }
 
-    /// Returns the `hash_tree_root` of the block.
+    /// Returns the `tree_hash_root | update` of the block.
     ///
     /// Spec v0.5.0
     pub fn canonical_root(&self) -> Hash256 {
-        Hash256::from_slice(&self.hash_tree_root()[..])
+        Hash256::from_slice(&self.tree_hash_root()[..])
     }
 
     /// Returns a full `BeaconBlockHeader` of this block.
@@ -77,7 +78,7 @@ impl BeaconBlock {
             slot: self.slot,
             previous_block_root: self.previous_block_root,
             state_root: self.state_root,
-            block_body_root: Hash256::from_slice(&self.body.hash_tree_root()[..]),
+            block_body_root: Hash256::from_slice(&self.body.tree_hash_root()[..]),
             signature: self.signature.clone(),
         }
     }
diff --git a/eth2/types/src/beacon_block_body.rs b/eth2/types/src/beacon_block_body.rs
index 677e24cec..0414d0d72 100644
--- a/eth2/types/src/beacon_block_body.rs
+++ b/eth2/types/src/beacon_block_body.rs
@@ -2,8 +2,9 @@ use crate::test_utils::TestRandom;
 use crate::*;
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
-use ssz_derive::{Decode, Encode, TreeHash};
+use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
+use tree_hash_derive::TreeHash;
 
 /// The body of a `BeaconChain` block, containing operations.
 ///
diff --git a/eth2/types/src/beacon_block_header.rs b/eth2/types/src/beacon_block_header.rs
index 090d0a965..9076437c0 100644
--- a/eth2/types/src/beacon_block_header.rs
+++ b/eth2/types/src/beacon_block_header.rs
@@ -3,9 +3,10 @@ use crate::*;
 use bls::Signature;
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
-use ssz::TreeHash;
-use ssz_derive::{Decode, Encode, SignedRoot, TreeHash};
+use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
+use tree_hash::TreeHash;
+use tree_hash_derive::{SignedRoot, TreeHash};
 
 /// A header of a `BeaconBlock`.
 ///
@@ -32,11 +33,11 @@ pub struct BeaconBlockHeader {
 }
 
 impl BeaconBlockHeader {
-    /// Returns the `hash_tree_root` of the header.
+    /// Returns the `tree_hash_root` of the header.
     ///
     /// Spec v0.5.0
     pub fn canonical_root(&self) -> Hash256 {
-        Hash256::from_slice(&self.hash_tree_root()[..])
+        Hash256::from_slice(&self.tree_hash_root()[..])
     }
 
     /// Given a `body`, consumes `self` and returns a complete `BeaconBlock`.
diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs
index 774e8eb76..19c1b4c11 100644
--- a/eth2/types/src/beacon_state.rs
+++ b/eth2/types/src/beacon_state.rs
@@ -5,9 +5,11 @@ use int_to_bytes::int_to_bytes32;
 use pubkey_cache::PubkeyCache;
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
-use ssz::{hash, ssz_encode, TreeHash};
-use ssz_derive::{Decode, Encode, TreeHash};
+use ssz::{hash, ssz_encode};
+use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
+use tree_hash::TreeHash;
+use tree_hash_derive::TreeHash;
 
 mod epoch_cache;
 mod pubkey_cache;
@@ -186,11 +188,11 @@ impl BeaconState {
         }
     }
 
-    /// Returns the `hash_tree_root` of the state.
+    /// Returns the `tree_hash_root` of the state.
     ///
     /// Spec v0.5.0
     pub fn canonical_root(&self) -> Hash256 {
-        Hash256::from_slice(&self.hash_tree_root()[..])
+        Hash256::from_slice(&self.tree_hash_root()[..])
     }
 
     pub fn historical_batch(&self) -> HistoricalBatch {
diff --git a/eth2/types/src/crosslink.rs b/eth2/types/src/crosslink.rs
index f91680c75..a0fd7e0b3 100644
--- a/eth2/types/src/crosslink.rs
+++ b/eth2/types/src/crosslink.rs
@@ -2,8 +2,9 @@ use crate::test_utils::TestRandom;
 use crate::{Epoch, Hash256};
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
-use ssz_derive::{Decode, Encode, TreeHash};
+use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
+use tree_hash_derive::TreeHash;
 
 /// Specifies the block hash for a shard at an epoch.
 ///
diff --git a/eth2/types/src/crosslink_committee.rs b/eth2/types/src/crosslink_committee.rs
index af1778a1b..e8fc1b96d 100644
--- a/eth2/types/src/crosslink_committee.rs
+++ b/eth2/types/src/crosslink_committee.rs
@@ -1,6 +1,7 @@
 use crate::*;
 use serde_derive::{Deserialize, Serialize};
-use ssz_derive::{Decode, Encode, TreeHash};
+use ssz_derive::{Decode, Encode};
+use tree_hash_derive::TreeHash;
 
 #[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize, Decode, Encode, TreeHash)]
 pub struct CrosslinkCommittee {
diff --git a/eth2/types/src/deposit.rs b/eth2/types/src/deposit.rs
index ff8d83d77..5eb565c2b 100644
--- a/eth2/types/src/deposit.rs
+++ b/eth2/types/src/deposit.rs
@@ -2,8 +2,9 @@ use super::{DepositData, Hash256};
 use crate::test_utils::TestRandom;
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
-use ssz_derive::{Decode, Encode, TreeHash};
+use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
+use tree_hash_derive::TreeHash;
 
 /// A deposit to potentially become a beacon chain validator.
 ///
diff --git a/eth2/types/src/deposit_data.rs b/eth2/types/src/deposit_data.rs
index a1e30032f..f8726e95d 100644
--- a/eth2/types/src/deposit_data.rs
+++ b/eth2/types/src/deposit_data.rs
@@ -2,8 +2,9 @@ use super::DepositInput;
 use crate::test_utils::TestRandom;
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
-use ssz_derive::{Decode, Encode, TreeHash};
+use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
+use tree_hash_derive::TreeHash;
 
 /// Data generated by the deposit contract.
 ///
diff --git a/eth2/types/src/deposit_input.rs b/eth2/types/src/deposit_input.rs
index 380528dc0..828496293 100644
--- a/eth2/types/src/deposit_input.rs
+++ b/eth2/types/src/deposit_input.rs
@@ -3,9 +3,10 @@ use crate::*;
 use bls::{PublicKey, Signature};
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
-use ssz::{SignedRoot, TreeHash};
-use ssz_derive::{Decode, Encode, SignedRoot, TreeHash};
+use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
+use tree_hash::{SignedRoot, TreeHash};
+use tree_hash_derive::{SignedRoot, TreeHash};
 
 /// The data supplied by the user to the deposit contract.
 ///
diff --git a/eth2/types/src/epoch_cache.rs b/eth2/types/src/epoch_cache.rs
deleted file mode 100644
index e69de29bb..000000000
diff --git a/eth2/types/src/eth1_data.rs b/eth2/types/src/eth1_data.rs
index deced19fb..c1348cfba 100644
--- a/eth2/types/src/eth1_data.rs
+++ b/eth2/types/src/eth1_data.rs
@@ -2,8 +2,9 @@ use super::Hash256;
 use crate::test_utils::TestRandom;
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
-use ssz_derive::{Decode, Encode, TreeHash};
+use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
+use tree_hash_derive::TreeHash;
 
 /// Contains data obtained from the Eth1 chain.
 ///
diff --git a/eth2/types/src/eth1_data_vote.rs b/eth2/types/src/eth1_data_vote.rs
index 2f3a1ade1..a9741f065 100644
--- a/eth2/types/src/eth1_data_vote.rs
+++ b/eth2/types/src/eth1_data_vote.rs
@@ -2,8 +2,9 @@ use super::Eth1Data;
 use crate::test_utils::TestRandom;
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
-use ssz_derive::{Decode, Encode, TreeHash};
+use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
+use tree_hash_derive::TreeHash;
 
 /// A summation of votes for some `Eth1Data`.
 ///
diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs
index b9d16c333..99908e9ed 100644
--- a/eth2/types/src/fork.rs
+++ b/eth2/types/src/fork.rs
@@ -5,8 +5,9 @@ use crate::{
 use int_to_bytes::int_to_bytes4;
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
-use ssz_derive::{Decode, Encode, TreeHash};
+use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
+use tree_hash_derive::TreeHash;
 
 /// Specifies a fork of the `BeaconChain`, to prevent replay attacks.
 ///
diff --git a/eth2/types/src/historical_batch.rs b/eth2/types/src/historical_batch.rs
index 77859ed1a..33dc9c450 100644
--- a/eth2/types/src/historical_batch.rs
+++ b/eth2/types/src/historical_batch.rs
@@ -2,8 +2,9 @@ use crate::test_utils::TestRandom;
 use crate::Hash256;
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
-use ssz_derive::{Decode, Encode, TreeHash};
+use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
+use tree_hash_derive::TreeHash;
 
 /// Historical block and state roots.
 ///
diff --git a/eth2/types/src/pending_attestation.rs b/eth2/types/src/pending_attestation.rs
index 938e59bef..5cbe1edeb 100644
--- a/eth2/types/src/pending_attestation.rs
+++ b/eth2/types/src/pending_attestation.rs
@@ -2,8 +2,9 @@ use crate::test_utils::TestRandom;
 use crate::{Attestation, AttestationData, Bitfield, Slot};
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
-use ssz_derive::{Decode, Encode, TreeHash};
+use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
+use tree_hash_derive::TreeHash;
 
 /// An attestation that has been included in the state but not yet fully processed.
 ///
diff --git a/eth2/types/src/proposer_slashing.rs b/eth2/types/src/proposer_slashing.rs
index 02216a2fc..901f02388 100644
--- a/eth2/types/src/proposer_slashing.rs
+++ b/eth2/types/src/proposer_slashing.rs
@@ -2,8 +2,9 @@ use super::BeaconBlockHeader;
 use crate::test_utils::TestRandom;
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
-use ssz_derive::{Decode, Encode, TreeHash};
+use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
+use tree_hash_derive::TreeHash;
 
 /// Two conflicting proposals from the same proposer (validator).
 ///
diff --git a/eth2/types/src/slashable_attestation.rs b/eth2/types/src/slashable_attestation.rs
index e557285b8..37462f006 100644
--- a/eth2/types/src/slashable_attestation.rs
+++ b/eth2/types/src/slashable_attestation.rs
@@ -1,9 +1,10 @@
 use crate::{test_utils::TestRandom, AggregateSignature, AttestationData, Bitfield, ChainSpec};
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
-use ssz::TreeHash;
-use ssz_derive::{Decode, Encode, SignedRoot, TreeHash};
+use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
+use tree_hash::TreeHash;
+use tree_hash_derive::{SignedRoot, TreeHash};
 
 /// Details an attestation that can be slashable.
 ///
diff --git a/eth2/types/src/slot_epoch.rs b/eth2/types/src/slot_epoch.rs
index d334177e5..6c6a92ecb 100644
--- a/eth2/types/src/slot_epoch.rs
+++ b/eth2/types/src/slot_epoch.rs
@@ -14,7 +14,7 @@ use crate::test_utils::TestRandom;
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
 use slog;
-use ssz::{hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash};
+use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream};
 use std::cmp::{Ord, Ordering};
 use std::fmt;
 use std::hash::{Hash, Hasher};
diff --git a/eth2/types/src/slot_epoch_macros.rs b/eth2/types/src/slot_epoch_macros.rs
index 300ad3f6f..b3ca5c4bc 100644
--- a/eth2/types/src/slot_epoch_macros.rs
+++ b/eth2/types/src/slot_epoch_macros.rs
@@ -206,11 +206,21 @@ macro_rules! impl_ssz {
             }
         }
 
-        impl TreeHash for $type {
-            fn hash_tree_root(&self) -> Vec<u8> {
-                let mut result: Vec<u8> = vec![];
-                result.append(&mut self.0.hash_tree_root());
-                hash(&result)
+        impl tree_hash::TreeHash for $type {
+            fn tree_hash_type() -> tree_hash::TreeHashType {
+                tree_hash::TreeHashType::Basic
+            }
+
+            fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+                ssz_encode(self)
+            }
+
+            fn tree_hash_packing_factor() -> usize {
+                32 / 8
+            }
+
+            fn tree_hash_root(&self) -> Vec<u8> {
+                int_to_bytes::int_to_bytes32(self.0)
             }
         }
 
diff --git a/eth2/types/src/slot_height.rs b/eth2/types/src/slot_height.rs
index 4a783d4a0..f7a34cbba 100644
--- a/eth2/types/src/slot_height.rs
+++ b/eth2/types/src/slot_height.rs
@@ -2,7 +2,7 @@ use crate::slot_epoch::{Epoch, Slot};
 use crate::test_utils::TestRandom;
 use rand::RngCore;
 use serde_derive::Serialize;
-use ssz::{hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash};
+use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream};
 use std::cmp::{Ord, Ordering};
 use std::fmt;
 use std::hash::{Hash, Hasher};
diff --git a/eth2/types/src/test_utils/macros.rs b/eth2/types/src/test_utils/macros.rs
index d580fd818..d5711e96e 100644
--- a/eth2/types/src/test_utils/macros.rs
+++ b/eth2/types/src/test_utils/macros.rs
@@ -17,14 +17,14 @@ macro_rules! ssz_tests {
         }
 
         #[test]
-        pub fn test_hash_tree_root() {
+        pub fn test_tree_hash_root() {
             use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
-            use ssz::TreeHash;
+            use tree_hash::TreeHash;
 
             let mut rng = XorShiftRng::from_seed([42; 16]);
             let original = $type::random_for_test(&mut rng);
 
-            let result = original.hash_tree_root();
+            let result = original.tree_hash_root();
 
             assert_eq!(result.len(), 32);
             // TODO: Add further tests
diff --git a/eth2/types/src/test_utils/testing_attestation_builder.rs b/eth2/types/src/test_utils/testing_attestation_builder.rs
index 60624b48d..162facc8e 100644
--- a/eth2/types/src/test_utils/testing_attestation_builder.rs
+++ b/eth2/types/src/test_utils/testing_attestation_builder.rs
@@ -1,6 +1,6 @@
 use crate::test_utils::TestingAttestationDataBuilder;
 use crate::*;
-use ssz::TreeHash;
+use tree_hash::TreeHash;
 
 /// Builds an attestation to be used for testing purposes.
 ///
@@ -74,7 +74,7 @@ impl TestingAttestationBuilder {
                 data: self.attestation.data.clone(),
                 custody_bit: false,
             }
-            .hash_tree_root();
+            .tree_hash_root();
 
             let domain = spec.get_domain(
                 self.attestation.data.slot.epoch(spec.slots_per_epoch),
diff --git a/eth2/types/src/test_utils/testing_attester_slashing_builder.rs b/eth2/types/src/test_utils/testing_attester_slashing_builder.rs
index fcaa3285b..dc01f7fb0 100644
--- a/eth2/types/src/test_utils/testing_attester_slashing_builder.rs
+++ b/eth2/types/src/test_utils/testing_attester_slashing_builder.rs
@@ -1,5 +1,5 @@
 use crate::*;
-use ssz::TreeHash;
+use tree_hash::TreeHash;
 
 /// Builds an `AttesterSlashing`.
 ///
@@ -66,7 +66,7 @@ impl TestingAttesterSlashingBuilder {
                 data: attestation.data.clone(),
                 custody_bit: false,
             };
-            let message = attestation_data_and_custody_bit.hash_tree_root();
+            let message = attestation_data_and_custody_bit.tree_hash_root();
 
             for (i, validator_index) in validator_indices.iter().enumerate() {
                 attestation.custody_bitfield.set(i, false);
diff --git a/eth2/types/src/test_utils/testing_beacon_block_builder.rs b/eth2/types/src/test_utils/testing_beacon_block_builder.rs
index c5cd22ed4..549c00ac0 100644
--- a/eth2/types/src/test_utils/testing_beacon_block_builder.rs
+++ b/eth2/types/src/test_utils/testing_beacon_block_builder.rs
@@ -6,7 +6,7 @@ use crate::{
     *,
 };
 use rayon::prelude::*;
-use ssz::{SignedRoot, TreeHash};
+use tree_hash::{SignedRoot, TreeHash};
 
 /// Builds a beacon block to be used for testing purposes.
 ///
@@ -43,7 +43,7 @@ impl TestingBeaconBlockBuilder {
     /// Modifying the block's slot after signing may invalidate the signature.
     pub fn set_randao_reveal(&mut self, sk: &SecretKey, fork: &Fork, spec: &ChainSpec) {
         let epoch = self.block.slot.epoch(spec.slots_per_epoch);
-        let message = epoch.hash_tree_root();
+        let message = epoch.tree_hash_root();
         let domain = spec.get_domain(epoch, Domain::Randao, fork);
         self.block.body.randao_reveal = Signature::new(&message, domain, sk);
     }
diff --git a/eth2/types/src/test_utils/testing_proposer_slashing_builder.rs b/eth2/types/src/test_utils/testing_proposer_slashing_builder.rs
index 2cfebd915..03c257b2d 100644
--- a/eth2/types/src/test_utils/testing_proposer_slashing_builder.rs
+++ b/eth2/types/src/test_utils/testing_proposer_slashing_builder.rs
@@ -1,5 +1,5 @@
 use crate::*;
-use ssz::SignedRoot;
+use tree_hash::SignedRoot;
 
 /// Builds a `ProposerSlashing`.
 ///
diff --git a/eth2/types/src/test_utils/testing_transfer_builder.rs b/eth2/types/src/test_utils/testing_transfer_builder.rs
index 354e29aa5..2680f7b66 100644
--- a/eth2/types/src/test_utils/testing_transfer_builder.rs
+++ b/eth2/types/src/test_utils/testing_transfer_builder.rs
@@ -1,5 +1,5 @@
 use crate::*;
-use ssz::SignedRoot;
+use tree_hash::SignedRoot;
 
 /// Builds a transfer to be used for testing purposes.
 ///
diff --git a/eth2/types/src/test_utils/testing_voluntary_exit_builder.rs b/eth2/types/src/test_utils/testing_voluntary_exit_builder.rs
index fe5c8325a..8583bc451 100644
--- a/eth2/types/src/test_utils/testing_voluntary_exit_builder.rs
+++ b/eth2/types/src/test_utils/testing_voluntary_exit_builder.rs
@@ -1,5 +1,5 @@
 use crate::*;
-use ssz::SignedRoot;
+use tree_hash::SignedRoot;
 
 /// Builds an exit to be used for testing purposes.
 ///
diff --git a/eth2/types/src/transfer.rs b/eth2/types/src/transfer.rs
index f291190b2..f40050bc4 100644
--- a/eth2/types/src/transfer.rs
+++ b/eth2/types/src/transfer.rs
@@ -4,9 +4,10 @@ use bls::{PublicKey, Signature};
 use derivative::Derivative;
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
-use ssz::TreeHash;
-use ssz_derive::{Decode, Encode, SignedRoot, TreeHash};
+use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
+use tree_hash::TreeHash;
+use tree_hash_derive::{SignedRoot, TreeHash};
 
 /// The data submitted to the deposit contract.
 ///
diff --git a/eth2/types/src/validator.rs b/eth2/types/src/validator.rs
index f57261175..67b4e85df 100644
--- a/eth2/types/src/validator.rs
+++ b/eth2/types/src/validator.rs
@@ -1,8 +1,9 @@
 use crate::{test_utils::TestRandom, Epoch, Hash256, PublicKey};
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
-use ssz_derive::{Decode, Encode, TreeHash};
+use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
+use tree_hash_derive::TreeHash;
 
 /// Information about a `BeaconChain` validator.
 ///
diff --git a/eth2/types/src/voluntary_exit.rs b/eth2/types/src/voluntary_exit.rs
index 0cdc63149..16d22c544 100644
--- a/eth2/types/src/voluntary_exit.rs
+++ b/eth2/types/src/voluntary_exit.rs
@@ -2,9 +2,10 @@ use crate::{test_utils::TestRandom, Epoch};
 use bls::Signature;
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
-use ssz::TreeHash;
-use ssz_derive::{Decode, Encode, SignedRoot, TreeHash};
+use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
+use tree_hash::TreeHash;
+use tree_hash_derive::{SignedRoot, TreeHash};
 
 /// An exit voluntarily submitted a validator who wishes to withdraw.
 ///
diff --git a/eth2/utils/bls/Cargo.toml b/eth2/utils/bls/Cargo.toml
index 4230a06ea..439debdcb 100644
--- a/eth2/utils/bls/Cargo.toml
+++ b/eth2/utils/bls/Cargo.toml
@@ -12,3 +12,4 @@ serde = "1.0"
 serde_derive = "1.0"
 serde_hex = { path = "../serde_hex" }
 ssz = { path = "../ssz" }
+tree_hash = { path = "../tree_hash" }
diff --git a/eth2/utils/bls/src/aggregate_signature.rs b/eth2/utils/bls/src/aggregate_signature.rs
index 8c7ae5222..156e362e2 100644
--- a/eth2/utils/bls/src/aggregate_signature.rs
+++ b/eth2/utils/bls/src/aggregate_signature.rs
@@ -166,7 +166,7 @@ impl<'de> Deserialize<'de> for AggregateSignature {
 }
 
 impl TreeHash for AggregateSignature {
-    fn hash_tree_root(&self) -> Vec<u8> {
+    fn tree_hash_root(&self) -> Vec<u8> {
         hash(&self.as_bytes())
     }
 }
diff --git a/eth2/utils/bls/src/fake_aggregate_signature.rs b/eth2/utils/bls/src/fake_aggregate_signature.rs
index 3f0ec0d6d..602639b6b 100644
--- a/eth2/utils/bls/src/fake_aggregate_signature.rs
+++ b/eth2/utils/bls/src/fake_aggregate_signature.rs
@@ -2,7 +2,8 @@ use super::{fake_signature::FakeSignature, AggregatePublicKey, BLS_AGG_SIG_BYTE_
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::{encode as hex_encode, PrefixedHexVisitor};
-use ssz::{hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash};
+use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream};
+use tree_hash::impl_tree_hash_for_ssz_bytes;
 
 /// A BLS aggregate signature.
 ///
@@ -98,11 +99,7 @@ impl<'de> Deserialize<'de> for FakeAggregateSignature {
     }
 }
 
-impl TreeHash for FakeAggregateSignature {
-    fn hash_tree_root(&self) -> Vec<u8> {
-        hash(&self.bytes)
-    }
-}
+impl_tree_hash_for_ssz_bytes!(FakeAggregateSignature);
 
 #[cfg(test)]
 mod tests {
diff --git a/eth2/utils/bls/src/fake_signature.rs b/eth2/utils/bls/src/fake_signature.rs
index 3c9f3a9f4..b07dd66a5 100644
--- a/eth2/utils/bls/src/fake_signature.rs
+++ b/eth2/utils/bls/src/fake_signature.rs
@@ -3,7 +3,8 @@ use hex::encode as hex_encode;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::HexVisitor;
-use ssz::{hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash};
+use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream};
+use tree_hash::impl_tree_hash_for_ssz_bytes;
 
 /// A single BLS signature.
 ///
@@ -73,11 +74,7 @@ impl Decodable for FakeSignature {
     }
 }
 
-impl TreeHash for FakeSignature {
-    fn hash_tree_root(&self) -> Vec<u8> {
-        hash(&self.bytes)
-    }
-}
+impl_tree_hash_for_ssz_bytes!(FakeSignature);
 
 impl Serialize for FakeSignature {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
diff --git a/eth2/utils/bls/src/public_key.rs b/eth2/utils/bls/src/public_key.rs
index 177a735c4..a553ee888 100644
--- a/eth2/utils/bls/src/public_key.rs
+++ b/eth2/utils/bls/src/public_key.rs
@@ -3,10 +3,11 @@ use bls_aggregates::PublicKey as RawPublicKey;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::{encode as hex_encode, HexVisitor};
-use ssz::{decode, hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash};
+use ssz::{decode, ssz_encode, Decodable, DecodeError, Encodable, SszStream};
 use std::default;
 use std::fmt;
 use std::hash::{Hash, Hasher};
+use tree_hash::impl_tree_hash_for_ssz_bytes;
 
 /// A single BLS signature.
 ///
@@ -104,11 +105,7 @@ impl<'de> Deserialize<'de> for PublicKey {
     }
 }
 
-impl TreeHash for PublicKey {
-    fn hash_tree_root(&self) -> Vec<u8> {
-        hash(&self.0.as_bytes())
-    }
-}
+impl_tree_hash_for_ssz_bytes!(PublicKey);
 
 impl PartialEq for PublicKey {
     fn eq(&self, other: &PublicKey) -> bool {
diff --git a/eth2/utils/bls/src/secret_key.rs b/eth2/utils/bls/src/secret_key.rs
index 40c469513..38fd2d379 100644
--- a/eth2/utils/bls/src/secret_key.rs
+++ b/eth2/utils/bls/src/secret_key.rs
@@ -4,7 +4,8 @@ use hex::encode as hex_encode;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::HexVisitor;
-use ssz::{decode, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash};
+use ssz::{decode, ssz_encode, Decodable, DecodeError, Encodable, SszStream};
+use tree_hash::impl_tree_hash_for_ssz_bytes;
 
 /// A single BLS signature.
 ///
@@ -69,11 +70,7 @@ impl<'de> Deserialize<'de> for SecretKey {
     }
 }
 
-impl TreeHash for SecretKey {
-    fn hash_tree_root(&self) -> Vec<u8> {
-        self.0.as_bytes().clone()
-    }
-}
+impl_tree_hash_for_ssz_bytes!(SecretKey);
 
 #[cfg(test)]
 mod tests {
diff --git a/eth2/utils/bls/src/signature.rs b/eth2/utils/bls/src/signature.rs
index d19af545f..30b55a787 100644
--- a/eth2/utils/bls/src/signature.rs
+++ b/eth2/utils/bls/src/signature.rs
@@ -115,7 +115,7 @@ impl Decodable for Signature {
 }
 
 impl TreeHash for Signature {
-    fn hash_tree_root(&self) -> Vec<u8> {
+    fn tree_hash_root(&self) -> Vec<u8> {
         hash(&self.as_bytes())
     }
 }
diff --git a/eth2/utils/boolean-bitfield/Cargo.toml b/eth2/utils/boolean-bitfield/Cargo.toml
index cf037c5d7..f08695bd1 100644
--- a/eth2/utils/boolean-bitfield/Cargo.toml
+++ b/eth2/utils/boolean-bitfield/Cargo.toml
@@ -10,3 +10,4 @@ ssz = { path = "../ssz" }
 bit-vec = "0.5.0"
 serde = "1.0"
 serde_derive = "1.0"
+tree_hash = { path = "../tree_hash" }
diff --git a/eth2/utils/boolean-bitfield/src/lib.rs b/eth2/utils/boolean-bitfield/src/lib.rs
index d04516dba..fbd0e2ecd 100644
--- a/eth2/utils/boolean-bitfield/src/lib.rs
+++ b/eth2/utils/boolean-bitfield/src/lib.rs
@@ -9,6 +9,7 @@ use serde_hex::{encode, PrefixedHexVisitor};
 use ssz::{Decodable, Encodable};
 use std::cmp;
 use std::default;
+use tree_hash::impl_tree_hash_for_ssz_bytes;
 
 /// A BooleanBitfield represents a set of booleans compactly stored as a vector of bits.
 /// The BooleanBitfield is given a fixed size during construction. Reads outside of the current size return an out-of-bounds error. Writes outside of the current size expand the size of the set.
@@ -256,11 +257,7 @@ impl<'de> Deserialize<'de> for BooleanBitfield {
     }
 }
 
-impl ssz::TreeHash for BooleanBitfield {
-    fn hash_tree_root(&self) -> Vec<u8> {
-        self.to_bytes().hash_tree_root()
-    }
-}
+impl_tree_hash_for_ssz_bytes!(BooleanBitfield);
 
 #[cfg(test)]
 mod tests {
diff --git a/eth2/utils/ssz_derive/tests/test_derives.rs b/eth2/utils/ssz_derive/tests/test_derives.rs
deleted file mode 100644
index e025dc3a5..000000000
--- a/eth2/utils/ssz_derive/tests/test_derives.rs
+++ /dev/null
@@ -1,94 +0,0 @@
-use ssz::{SignedRoot, TreeHash};
-use ssz_derive::{SignedRoot, TreeHash};
-
-#[derive(TreeHash, SignedRoot)]
-struct CryptoKitties {
-    best_kitty: u64,
-    worst_kitty: u8,
-    kitties: Vec<u32>,
-}
-
-impl CryptoKitties {
-    fn new() -> Self {
-        CryptoKitties {
-            best_kitty: 9999,
-            worst_kitty: 1,
-            kitties: vec![2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43],
-        }
-    }
-
-    fn hash(&self) -> Vec<u8> {
-        let mut list: Vec<Vec<u8>> = Vec::new();
-        list.push(self.best_kitty.hash_tree_root());
-        list.push(self.worst_kitty.hash_tree_root());
-        list.push(self.kitties.hash_tree_root());
-        ssz::merkle_hash(&mut list)
-    }
-}
-
-#[test]
-fn test_cryptokitties_hash() {
-    let kitties = CryptoKitties::new();
-    let expected_hash = vec![
-        201, 9, 139, 14, 24, 247, 21, 55, 132, 211, 51, 125, 183, 186, 177, 33, 147, 210, 42, 108,
-        174, 162, 221, 227, 157, 179, 15, 7, 97, 239, 82, 220,
-    ];
-    assert_eq!(kitties.hash(), expected_hash);
-}
-
-#[test]
-fn test_simple_tree_hash_derive() {
-    let kitties = CryptoKitties::new();
-    assert_eq!(kitties.hash_tree_root(), kitties.hash());
-}
-
-#[test]
-fn test_simple_signed_root_derive() {
-    let kitties = CryptoKitties::new();
-    assert_eq!(kitties.signed_root(), kitties.hash());
-}
-
-#[derive(TreeHash, SignedRoot)]
-struct Casper {
-    friendly: bool,
-    #[tree_hash(skip_hashing)]
-    friends: Vec<u32>,
-    #[signed_root(skip_hashing)]
-    dead: bool,
-}
-
-impl Casper {
-    fn new() -> Self {
-        Casper {
-            friendly: true,
-            friends: vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
-            dead: true,
-        }
-    }
-
-    fn expected_signed_hash(&self) -> Vec<u8> {
-        let mut list = Vec::new();
-        list.push(self.friendly.hash_tree_root());
-        list.push(self.friends.hash_tree_root());
-        ssz::merkle_hash(&mut list)
-    }
-
-    fn expected_tree_hash(&self) -> Vec<u8> {
-        let mut list = Vec::new();
-        list.push(self.friendly.hash_tree_root());
-        list.push(self.dead.hash_tree_root());
-        ssz::merkle_hash(&mut list)
-    }
-}
-
-#[test]
-fn test_annotated_tree_hash_derive() {
-    let casper = Casper::new();
-    assert_eq!(casper.hash_tree_root(), casper.expected_tree_hash());
-}
-
-#[test]
-fn test_annotated_signed_root_derive() {
-    let casper = Casper::new();
-    assert_eq!(casper.signed_root(), casper.expected_signed_hash());
-}
diff --git a/eth2/utils/tree_hash/src/standard_tree_hash.rs b/eth2/utils/tree_hash/src/standard_tree_hash.rs
index ea0677180..473d2a5f0 100644
--- a/eth2/utils/tree_hash/src/standard_tree_hash.rs
+++ b/eth2/utils/tree_hash/src/standard_tree_hash.rs
@@ -25,9 +25,14 @@ pub fn efficient_merkleize(bytes: &[u8]) -> Vec<u8> {
     let nodes = num_nodes(leaves);
     let internal_nodes = nodes - leaves;
 
-    let num_bytes = internal_nodes * HASHSIZE + bytes.len();
+    let num_bytes = std::cmp::max(internal_nodes, 1) * HASHSIZE + bytes.len();
 
     let mut o: Vec<u8> = vec![0; internal_nodes * HASHSIZE];
+
+    if o.len() < HASHSIZE {
+        o.resize(HASHSIZE, 0);
+    }
+
     o.append(&mut bytes.to_vec());
 
     assert_eq!(o.len(), num_bytes);
diff --git a/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs b/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs
index 070e314b8..749d5b3bb 100644
--- a/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs
+++ b/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs
@@ -30,6 +30,24 @@ impl_for_bitsize!(u64, 64);
 impl_for_bitsize!(usize, 64);
 impl_for_bitsize!(bool, 8);
 
+impl TreeHash for [u8; 4] {
+    fn tree_hash_type() -> TreeHashType {
+        TreeHashType::List
+    }
+
+    fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+        panic!("bytesN should never be packed.")
+    }
+
+    fn tree_hash_packing_factor() -> usize {
+        panic!("bytesN should never be packed.")
+    }
+
+    fn tree_hash_root(&self) -> Vec<u8> {
+        merkle_root(&ssz::ssz_encode(self))
+    }
+}
+
 impl TreeHash for H256 {
     fn tree_hash_type() -> TreeHashType {
         TreeHashType::Basic
@@ -95,3 +113,20 @@ where
         hash(&root_and_len)
     }
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn bool() {
+        let mut true_bytes: Vec<u8> = vec![1];
+        true_bytes.append(&mut vec![0; 31]);
+
+        let false_bytes: Vec<u8> = vec![0; 32];
+
+        assert_eq!(true.tree_hash_root(), true_bytes);
+        assert_eq!(false.tree_hash_root(), false_bytes);
+    }
+
+}
diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs
index dc3702c72..e3a7b4aaa 100644
--- a/eth2/utils/tree_hash_derive/src/lib.rs
+++ b/eth2/utils/tree_hash_derive/src/lib.rs
@@ -31,12 +31,10 @@ fn get_hashable_named_field_idents<'a>(struct_data: &'a syn::DataStruct) -> Vec<
 ///
 /// The field attribute is: `#[tree_hash(skip_hashing)]`
 fn should_skip_hashing(field: &syn::Field) -> bool {
-    for attr in &field.attrs {
-        if attr.tts.to_string() == "( skip_hashing )" {
-            return true;
-        }
-    }
-    false
+    field
+        .attrs
+        .iter()
+        .any(|attr| attr.into_token_stream().to_string() == "# [ tree_hash ( skip_hashing ) ]")
 }
 
 /// Implements `tree_hash::CachedTreeHashSubTree` for some `struct`.
diff --git a/eth2/utils/tree_hash_derive/tests/tests.rs b/eth2/utils/tree_hash_derive/tests/tests.rs
index 721e77715..a7c74b23e 100644
--- a/eth2/utils/tree_hash_derive/tests/tests.rs
+++ b/eth2/utils/tree_hash_derive/tests/tests.rs
@@ -98,3 +98,85 @@ fn signed_root() {
 
     assert_eq!(unsigned.tree_hash_root(), signed.signed_root());
 }
+
+#[derive(TreeHash, SignedRoot)]
+struct CryptoKitties {
+    best_kitty: u64,
+    worst_kitty: u8,
+    kitties: Vec<u32>,
+}
+
+impl CryptoKitties {
+    fn new() -> Self {
+        CryptoKitties {
+            best_kitty: 9999,
+            worst_kitty: 1,
+            kitties: vec![2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43],
+        }
+    }
+
+    fn hash(&self) -> Vec<u8> {
+        let mut leaves = vec![];
+        leaves.append(&mut self.best_kitty.tree_hash_root());
+        leaves.append(&mut self.worst_kitty.tree_hash_root());
+        leaves.append(&mut self.kitties.tree_hash_root());
+        tree_hash::merkle_root(&leaves)
+    }
+}
+
+#[test]
+fn test_simple_tree_hash_derive() {
+    let kitties = CryptoKitties::new();
+    assert_eq!(kitties.tree_hash_root(), kitties.hash());
+}
+
+#[test]
+fn test_simple_signed_root_derive() {
+    let kitties = CryptoKitties::new();
+    assert_eq!(kitties.signed_root(), kitties.hash());
+}
+
+#[derive(TreeHash, SignedRoot)]
+struct Casper {
+    friendly: bool,
+    #[tree_hash(skip_hashing)]
+    friends: Vec<u32>,
+    #[signed_root(skip_hashing)]
+    dead: bool,
+}
+
+impl Casper {
+    fn new() -> Self {
+        Casper {
+            friendly: true,
+            friends: vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+            dead: true,
+        }
+    }
+
+    fn expected_signed_hash(&self) -> Vec<u8> {
+        let mut list = Vec::new();
+        list.append(&mut self.friendly.tree_hash_root());
+        list.append(&mut self.friends.tree_hash_root());
+        tree_hash::merkle_root(&list)
+    }
+
+    fn expected_tree_hash(&self) -> Vec<u8> {
+        let mut list = Vec::new();
+        list.append(&mut self.friendly.tree_hash_root());
+        list.append(&mut self.dead.tree_hash_root());
+        tree_hash::merkle_root(&list)
+    }
+}
+
+#[test]
+fn test_annotated_tree_hash_derive() {
+    let casper = Casper::new();
+    assert_eq!(casper.tree_hash_root(), casper.expected_tree_hash());
+}
+
+#[test]
+fn test_annotated_signed_root_derive() {
+    let casper = Casper::new();
+    assert_eq!(casper.signed_root(), casper.expected_signed_hash());
+}
diff --git a/validator_client/src/attestation_producer/mod.rs b/validator_client/src/attestation_producer/mod.rs
index 0fbc7bcba..d2dbdf2e2 100644
--- a/validator_client/src/attestation_producer/mod.rs
+++ b/validator_client/src/attestation_producer/mod.rs
@@ -8,7 +8,7 @@ use super::block_producer::{BeaconNodeError, PublishOutcome, ValidatorEvent};
 use crate::signer::Signer;
 use beacon_node_attestation::BeaconNodeAttestation;
 use slog::{error, info, warn};
-use ssz::TreeHash;
+use tree_hash::TreeHash;
 use types::{
     AggregateSignature, Attestation, AttestationData, AttestationDataAndCustodyBit,
     AttestationDuty, Bitfield,
@@ -123,7 +123,7 @@ impl<'a, B: BeaconNodeAttestation, S: Signer> AttestationProducer<'a, B, S> {
                 data: attestation.clone(),
                 custody_bit: false,
             }
-            .hash_tree_root();
+            .tree_hash_root();
 
             let sig = self.signer.sign_message(&message, domain)?;
 
diff --git a/validator_client/src/block_producer/mod.rs b/validator_client/src/block_producer/mod.rs
index 8b4f5abda..9cc0460c7 100644
--- a/validator_client/src/block_producer/mod.rs
+++ b/validator_client/src/block_producer/mod.rs
@@ -86,7 +86,7 @@ impl<'a, B: BeaconNodeBlock, S: Signer> BlockProducer<'a, B, S> {
     pub fn produce_block(&mut self) -> Result<ValidatorEvent, Error> {
         let epoch = self.slot.epoch(self.spec.slots_per_epoch);
 
-        let message = epoch.hash_tree_root();
+        let message = epoch.tree_hash_root();
         let randao_reveal = match self.signer.sign_message(
             &message,
             self.spec.get_domain(epoch, Domain::Randao, &self.fork),

From f69b56ad6032d906b79aa086e55ee9097ca685df Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Tue, 16 Apr 2019 14:25:43 +1000
Subject: [PATCH 058/137] Add new `tree_hash` crate project wide.

---
 beacon_node/beacon_chain/Cargo.toml               | 1 +
 beacon_node/beacon_chain/test_harness/Cargo.toml  | 1 +
 beacon_node/network/Cargo.toml                    | 1 +
 eth2/attester/Cargo.toml                          | 1 +
 eth2/attester/src/lib.rs                          | 3 +--
 eth2/block_proposer/Cargo.toml                    | 1 +
 eth2/block_proposer/src/lib.rs                    | 2 +-
 eth2/state_processing/Cargo.toml                  | 2 ++
 eth2/state_processing/src/per_block_processing.rs | 2 +-
 eth2/utils/bls/src/aggregate_signature.rs         | 9 +++------
 eth2/utils/bls/src/signature.rs                   | 9 +++------
 validator_client/Cargo.toml                       | 1 +
 validator_client/src/block_producer/mod.rs        | 2 +-
 13 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml
index 55d4bacfd..e2a4527a9 100644
--- a/beacon_node/beacon_chain/Cargo.toml
+++ b/beacon_node/beacon_chain/Cargo.toml
@@ -23,4 +23,5 @@ serde_json = "1.0"
 slot_clock = { path = "../../eth2/utils/slot_clock" }
 ssz = { path = "../../eth2/utils/ssz" }
 state_processing = { path = "../../eth2/state_processing" }
+tree_hash = { path = "../../eth2/utils/tree_hash" }
 types = { path = "../../eth2/types" }
diff --git a/beacon_node/beacon_chain/test_harness/Cargo.toml b/beacon_node/beacon_chain/test_harness/Cargo.toml
index 50d154732..a2abf6c5a 100644
--- a/beacon_node/beacon_chain/test_harness/Cargo.toml
+++ b/beacon_node/beacon_chain/test_harness/Cargo.toml
@@ -38,5 +38,6 @@ serde_json = "1.0"
 serde_yaml = "0.8"
 slot_clock = { path = "../../../eth2/utils/slot_clock" }
 ssz = { path = "../../../eth2/utils/ssz" }
+tree_hash = { path = "../../../eth2/utils/tree_hash" }
 types = { path = "../../../eth2/types" }
 yaml-rust = "0.4.2"
diff --git a/beacon_node/network/Cargo.toml b/beacon_node/network/Cargo.toml
index cd2c2269a..36bf1f141 100644
--- a/beacon_node/network/Cargo.toml
+++ b/beacon_node/network/Cargo.toml
@@ -15,6 +15,7 @@ version = { path = "../version" }
 types = { path = "../../eth2/types" }
 slog = { version = "^2.2.3" , features = ["max_level_trace", "release_max_level_debug"] }
 ssz = { path = "../../eth2/utils/ssz" }
+tree_hash = { path = "../../eth2/utils/tree_hash" }
 futures = "0.1.25"
 error-chain = "0.12.0"
 crossbeam-channel = "0.3.8"
diff --git a/eth2/attester/Cargo.toml b/eth2/attester/Cargo.toml
index 956ecf565..41824274d 100644
--- a/eth2/attester/Cargo.toml
+++ b/eth2/attester/Cargo.toml
@@ -7,4 +7,5 @@ edition = "2018"
 [dependencies]
 slot_clock = { path = "../../eth2/utils/slot_clock" }
 ssz = { path = "../../eth2/utils/ssz" }
+tree_hash = { path = "../../eth2/utils/tree_hash" }
 types = { path = "../../eth2/types" }
diff --git a/eth2/attester/src/lib.rs b/eth2/attester/src/lib.rs
index a9e3091af..1bbbd6b43 100644
--- a/eth2/attester/src/lib.rs
+++ b/eth2/attester/src/lib.rs
@@ -141,8 +141,7 @@ impl<T: SlotClock, U: BeaconNode, V: DutiesReader, W: Signer> Attester<T, U, V,
             data: attestation_data.clone(),
             custody_bit: PHASE_0_CUSTODY_BIT,
         }
-        .tree_hash_root
-            | update();
+        .tree_hash_root();
 
         self.signer
             .sign_attestation_message(&message[..], DOMAIN_ATTESTATION)
diff --git a/eth2/block_proposer/Cargo.toml b/eth2/block_proposer/Cargo.toml
index 81f1ccc28..b5e60d383 100644
--- a/eth2/block_proposer/Cargo.toml
+++ b/eth2/block_proposer/Cargo.toml
@@ -8,4 +8,5 @@ edition = "2018"
 int_to_bytes = { path = "../utils/int_to_bytes" }
 slot_clock = { path = "../utils/slot_clock" }
 ssz = { path = "../utils/ssz" }
+tree_hash = { path = "../../eth2/utils/tree_hash" }
 types = { path = "../types" }
diff --git a/eth2/block_proposer/src/lib.rs b/eth2/block_proposer/src/lib.rs
index e98467b16..f38278e53 100644
--- a/eth2/block_proposer/src/lib.rs
+++ b/eth2/block_proposer/src/lib.rs
@@ -2,8 +2,8 @@ pub mod test_utils;
 mod traits;
 
 use slot_clock::SlotClock;
-use ssz::{SignedRoot, TreeHash};
 use std::sync::Arc;
+use tree_hash::{SignedRoot, TreeHash};
 use types::{BeaconBlock, ChainSpec, Domain, Slot};
 
 pub use self::traits::{
diff --git a/eth2/state_processing/Cargo.toml b/eth2/state_processing/Cargo.toml
index 62722aac8..1bc7a6c45 100644
--- a/eth2/state_processing/Cargo.toml
+++ b/eth2/state_processing/Cargo.toml
@@ -26,5 +26,7 @@ log = "0.4"
 merkle_proof = { path = "../utils/merkle_proof" }
 ssz = { path = "../utils/ssz" }
 ssz_derive = { path = "../utils/ssz_derive" }
+tree_hash = { path = "../utils/tree_hash" }
+tree_hash_derive = { path = "../utils/tree_hash_derive" }
 types = { path = "../types" }
 rayon = "1.0"
diff --git a/eth2/state_processing/src/per_block_processing.rs b/eth2/state_processing/src/per_block_processing.rs
index 32dc71a2e..5afddc74e 100644
--- a/eth2/state_processing/src/per_block_processing.rs
+++ b/eth2/state_processing/src/per_block_processing.rs
@@ -1,7 +1,7 @@
 use crate::common::slash_validator;
 use errors::{BlockInvalid as Invalid, BlockProcessingError as Error, IntoWithIndex};
 use rayon::prelude::*;
-use ssz::{SignedRoot, TreeHash};
+use tree_hash::{SignedRoot, TreeHash};
 use types::*;
 
 pub use self::verify_attester_slashing::{
diff --git a/eth2/utils/bls/src/aggregate_signature.rs b/eth2/utils/bls/src/aggregate_signature.rs
index 156e362e2..f26bd2db6 100644
--- a/eth2/utils/bls/src/aggregate_signature.rs
+++ b/eth2/utils/bls/src/aggregate_signature.rs
@@ -5,7 +5,8 @@ use bls_aggregates::{
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::{encode as hex_encode, HexVisitor};
-use ssz::{decode, hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
+use ssz::{decode, Decodable, DecodeError, Encodable, SszStream};
+use tree_hash::impl_tree_hash_for_ssz_bytes;
 
 /// A BLS aggregate signature.
 ///
@@ -165,11 +166,7 @@ impl<'de> Deserialize<'de> for AggregateSignature {
     }
 }
 
-impl TreeHash for AggregateSignature {
-    fn tree_hash_root(&self) -> Vec<u8> {
-        hash(&self.as_bytes())
-    }
-}
+impl_tree_hash_for_ssz_bytes!(AggregateSignature);
 
 #[cfg(test)]
 mod tests {
diff --git a/eth2/utils/bls/src/signature.rs b/eth2/utils/bls/src/signature.rs
index 30b55a787..cf6c8fe5a 100644
--- a/eth2/utils/bls/src/signature.rs
+++ b/eth2/utils/bls/src/signature.rs
@@ -4,7 +4,8 @@ use hex::encode as hex_encode;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::HexVisitor;
-use ssz::{decode, hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash};
+use ssz::{decode, ssz_encode, Decodable, DecodeError, Encodable, SszStream};
+use tree_hash::impl_tree_hash_for_ssz_bytes;
 
 /// A single BLS signature.
 ///
@@ -114,11 +115,7 @@ impl Decodable for Signature {
     }
 }
 
-impl TreeHash for Signature {
-    fn tree_hash_root(&self) -> Vec<u8> {
-        hash(&self.as_bytes())
-    }
-}
+impl_tree_hash_for_ssz_bytes!(Signature);
 
 impl Serialize for Signature {
     /// Serde serialization is compliant the Ethereum YAML test format.
diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml
index 80477c8ea..7f6b0cee9 100644
--- a/validator_client/Cargo.toml
+++ b/validator_client/Cargo.toml
@@ -17,6 +17,7 @@ block_proposer = { path = "../eth2/block_proposer" }
 attester = { path = "../eth2/attester" }
 bls = { path = "../eth2/utils/bls" }
 ssz = { path = "../eth2/utils/ssz" }
+tree_hash = { path = "../eth2/utils/tree_hash" }
 clap = "2.32.0"
 dirs = "1.0.3"
 grpcio = { version = "0.4", default-features = false, features = ["protobuf-codec"] }
diff --git a/validator_client/src/block_producer/mod.rs b/validator_client/src/block_producer/mod.rs
index 9cc0460c7..2689b302d 100644
--- a/validator_client/src/block_producer/mod.rs
+++ b/validator_client/src/block_producer/mod.rs
@@ -6,8 +6,8 @@ pub use self::beacon_node_block::{BeaconNodeError, PublishOutcome};
 pub use self::grpc::BeaconBlockGrpcClient;
 use crate::signer::Signer;
 use slog::{error, info, warn};
-use ssz::{SignedRoot, TreeHash};
 use std::sync::Arc;
+use tree_hash::{SignedRoot, TreeHash};
 use types::{BeaconBlock, ChainSpec, Domain, Fork, Slot};
 
 #[derive(Debug, PartialEq)]

From 49d066015b08901f5729148c4a27fb8872d42e8f Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 17 Apr 2019 10:33:31 +1000
Subject: [PATCH 059/137] Make genesis beacon state return a beacon state

---
 eth2/state_processing/src/get_genesis_state.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eth2/state_processing/src/get_genesis_state.rs b/eth2/state_processing/src/get_genesis_state.rs
index 21cdafcf9..2bde8ce0c 100644
--- a/eth2/state_processing/src/get_genesis_state.rs
+++ b/eth2/state_processing/src/get_genesis_state.rs
@@ -15,7 +15,7 @@ pub fn get_genesis_state(
     genesis_time: u64,
     genesis_eth1_data: Eth1Data,
     spec: &ChainSpec,
-) -> Result<(), BlockProcessingError> {
+) -> Result<BeaconState, BlockProcessingError> {
     // Get the genesis `BeaconState`
     let mut state = BeaconState::genesis(genesis_time, genesis_eth1_data, spec);
 
@@ -42,7 +42,7 @@ pub fn get_genesis_state(
     // Generate the current shuffling seed.
     state.current_shuffling_seed = state.generate_seed(spec.genesis_epoch, spec)?;
 
-    Ok(())
+    Ok(state)
 }
 
 impl From<BlockProcessingError> for GenesisError {

From af39f096e7a635f39acf9851f84f96cda6af8750 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 17 Apr 2019 10:57:36 +1000
Subject: [PATCH 060/137] Add vector type to tree hashing

---
 eth2/types/src/beacon_state.rs                | 28 ++++---
 eth2/types/src/historical_batch.rs            |  6 +-
 eth2/types/src/lib.rs                         |  2 +
 eth2/types/src/test_utils/mod.rs              |  5 +-
 eth2/types/src/tree_hash_vector.rs            | 82 +++++++++++++++++++
 .../src/cached_tree_hash/impls/vec.rs         |  6 +-
 eth2/utils/tree_hash/src/lib.rs               | 32 +++++++-
 .../utils/tree_hash/src/standard_tree_hash.rs | 15 ++--
 .../tree_hash/src/standard_tree_hash/impls.rs | 58 +++++++------
 eth2/utils/tree_hash/tests/tests.rs           | 34 +++++++-
 eth2/utils/tree_hash_derive/src/lib.rs        | 15 +---
 11 files changed, 211 insertions(+), 72 deletions(-)
 create mode 100644 eth2/types/src/tree_hash_vector.rs

diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs
index 19c1b4c11..c068c4e03 100644
--- a/eth2/types/src/beacon_state.rs
+++ b/eth2/types/src/beacon_state.rs
@@ -60,7 +60,7 @@ pub struct BeaconState {
     pub validator_registry_update_epoch: Epoch,
 
     // Randomness and committees
-    pub latest_randao_mixes: Vec<Hash256>,
+    pub latest_randao_mixes: TreeHashVector<Hash256>,
     pub previous_shuffling_start_shard: u64,
     pub current_shuffling_start_shard: u64,
     pub previous_shuffling_epoch: Epoch,
@@ -80,11 +80,11 @@ pub struct BeaconState {
     pub finalized_root: Hash256,
 
     // Recent state
-    pub latest_crosslinks: Vec<Crosslink>,
-    latest_block_roots: Vec<Hash256>,
-    latest_state_roots: Vec<Hash256>,
-    latest_active_index_roots: Vec<Hash256>,
-    latest_slashed_balances: Vec<u64>,
+    pub latest_crosslinks: TreeHashVector<Crosslink>,
+    latest_block_roots: TreeHashVector<Hash256>,
+    latest_state_roots: TreeHashVector<Hash256>,
+    latest_active_index_roots: TreeHashVector<Hash256>,
+    latest_slashed_balances: TreeHashVector<u64>,
     pub latest_block_header: BeaconBlockHeader,
     pub historical_roots: Vec<Hash256>,
 
@@ -139,7 +139,8 @@ impl BeaconState {
             validator_registry_update_epoch: spec.genesis_epoch,
 
             // Randomness and committees
-            latest_randao_mixes: vec![spec.zero_hash; spec.latest_randao_mixes_length as usize],
+            latest_randao_mixes: vec![spec.zero_hash; spec.latest_randao_mixes_length as usize]
+                .into(),
             previous_shuffling_start_shard: spec.genesis_start_shard,
             current_shuffling_start_shard: spec.genesis_start_shard,
             previous_shuffling_epoch: spec.genesis_epoch,
@@ -159,11 +160,12 @@ impl BeaconState {
             finalized_root: spec.zero_hash,
 
             // Recent state
-            latest_crosslinks: vec![initial_crosslink; spec.shard_count as usize],
-            latest_block_roots: vec![spec.zero_hash; spec.slots_per_historical_root],
-            latest_state_roots: vec![spec.zero_hash; spec.slots_per_historical_root],
-            latest_active_index_roots: vec![spec.zero_hash; spec.latest_active_index_roots_length],
-            latest_slashed_balances: vec![0; spec.latest_slashed_exit_length],
+            latest_crosslinks: vec![initial_crosslink; spec.shard_count as usize].into(),
+            latest_block_roots: vec![spec.zero_hash; spec.slots_per_historical_root].into(),
+            latest_state_roots: vec![spec.zero_hash; spec.slots_per_historical_root].into(),
+            latest_active_index_roots: vec![spec.zero_hash; spec.latest_active_index_roots_length]
+                .into(),
+            latest_slashed_balances: vec![0; spec.latest_slashed_exit_length].into(),
             latest_block_header: BeaconBlock::empty(spec).temporary_block_header(spec),
             historical_roots: vec![],
 
@@ -505,7 +507,7 @@ impl BeaconState {
     /// Spec v0.5.0
     pub fn fill_active_index_roots_with(&mut self, index_root: Hash256, spec: &ChainSpec) {
         self.latest_active_index_roots =
-            vec![index_root; spec.latest_active_index_roots_length as usize]
+            vec![index_root; spec.latest_active_index_roots_length as usize].into()
     }
 
     /// Safely obtains the index for latest state roots, given some `slot`.
diff --git a/eth2/types/src/historical_batch.rs b/eth2/types/src/historical_batch.rs
index 33dc9c450..23c26901e 100644
--- a/eth2/types/src/historical_batch.rs
+++ b/eth2/types/src/historical_batch.rs
@@ -1,5 +1,5 @@
 use crate::test_utils::TestRandom;
-use crate::Hash256;
+use crate::{Hash256, TreeHashVector};
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
@@ -11,8 +11,8 @@ use tree_hash_derive::TreeHash;
 /// Spec v0.5.0
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
 pub struct HistoricalBatch {
-    pub block_roots: Vec<Hash256>,
-    pub state_roots: Vec<Hash256>,
+    pub block_roots: TreeHashVector<Hash256>,
+    pub state_roots: TreeHashVector<Hash256>,
 }
 
 #[cfg(test)]
diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs
index 118e862e8..070ed6745 100644
--- a/eth2/types/src/lib.rs
+++ b/eth2/types/src/lib.rs
@@ -27,6 +27,7 @@ pub mod pending_attestation;
 pub mod proposer_slashing;
 pub mod slashable_attestation;
 pub mod transfer;
+pub mod tree_hash_vector;
 pub mod voluntary_exit;
 #[macro_use]
 pub mod slot_epoch_macros;
@@ -65,6 +66,7 @@ pub use crate::slashable_attestation::SlashableAttestation;
 pub use crate::slot_epoch::{Epoch, Slot};
 pub use crate::slot_height::SlotHeight;
 pub use crate::transfer::Transfer;
+pub use crate::tree_hash_vector::TreeHashVector;
 pub use crate::validator::Validator;
 pub use crate::voluntary_exit::VoluntaryExit;
 
diff --git a/eth2/types/src/test_utils/mod.rs b/eth2/types/src/test_utils/mod.rs
index 018b70d15..9d69a48f6 100644
--- a/eth2/types/src/test_utils/mod.rs
+++ b/eth2/types/src/test_utils/mod.rs
@@ -17,7 +17,10 @@ mod testing_voluntary_exit_builder;
 
 pub use generate_deterministic_keypairs::generate_deterministic_keypairs;
 pub use keypairs_file::KeypairsFile;
-pub use rand::{prng::XorShiftRng, SeedableRng};
+pub use rand::{
+    RngCore,
+    {prng::XorShiftRng, SeedableRng},
+};
 pub use serde_utils::{fork_from_hex_str, u8_from_hex_str};
 pub use test_random::TestRandom;
 pub use testing_attestation_builder::TestingAttestationBuilder;
diff --git a/eth2/types/src/tree_hash_vector.rs b/eth2/types/src/tree_hash_vector.rs
new file mode 100644
index 000000000..1cc8e40a5
--- /dev/null
+++ b/eth2/types/src/tree_hash_vector.rs
@@ -0,0 +1,82 @@
+use crate::test_utils::{RngCore, TestRandom};
+use serde_derive::{Deserialize, Serialize};
+use ssz::{Decodable, DecodeError, Encodable, SszStream};
+use std::ops::{Deref, DerefMut};
+use tree_hash::TreeHash;
+
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
+pub struct TreeHashVector<T>(Vec<T>);
+
+impl<T> From<Vec<T>> for TreeHashVector<T> {
+    fn from(vec: Vec<T>) -> TreeHashVector<T> {
+        TreeHashVector(vec)
+    }
+}
+
+impl<T> Into<Vec<T>> for TreeHashVector<T> {
+    fn into(self) -> Vec<T> {
+        self.0
+    }
+}
+
+impl<T> Deref for TreeHashVector<T> {
+    type Target = Vec<T>;
+
+    fn deref(&self) -> &Vec<T> {
+        &self.0
+    }
+}
+
+impl<T> DerefMut for TreeHashVector<T> {
+    fn deref_mut(&mut self) -> &mut Vec<T> {
+        &mut self.0
+    }
+}
+
+impl<T> tree_hash::TreeHash for TreeHashVector<T>
+where
+    T: TreeHash,
+{
+    fn tree_hash_type() -> tree_hash::TreeHashType {
+        tree_hash::TreeHashType::Vector
+    }
+
+    fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+        unreachable!("Vector should never be packed.")
+    }
+
+    fn tree_hash_packing_factor() -> usize {
+        unreachable!("Vector should never be packed.")
+    }
+
+    fn tree_hash_root(&self) -> Vec<u8> {
+        tree_hash::standard_tree_hash::vec_tree_hash_root(self)
+    }
+}
+
+impl<T> Encodable for TreeHashVector<T>
+where
+    T: Encodable,
+{
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append_vec(self)
+    }
+}
+
+impl<T> Decodable for TreeHashVector<T>
+where
+    T: Decodable,
+{
+    fn ssz_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), DecodeError> {
+        ssz::decode_ssz_list(bytes, index).and_then(|(vec, i)| Ok((vec.into(), i)))
+    }
+}
+
+impl<T: RngCore, U> TestRandom<T> for TreeHashVector<U>
+where
+    U: TestRandom<T>,
+{
+    fn random_for_test(rng: &mut T) -> Self {
+        Vec::random_for_test(rng).into()
+    }
+}
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
index 6c0970cef..1cd7eb902 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
@@ -9,7 +9,7 @@ where
             TreeHashType::Basic => {
                 TreeHashCache::from_bytes(merkleize(get_packed_leaves(self)?), false)
             }
-            TreeHashType::Composite | TreeHashType::List => {
+            TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
                 let subtrees = self
                     .iter()
                     .map(|item| TreeHashCache::new(item))
@@ -23,7 +23,7 @@ where
     fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
         let lengths = match T::tree_hash_type() {
             TreeHashType::Basic => vec![1; self.len() / T::tree_hash_packing_factor()],
-            TreeHashType::Composite | TreeHashType::List => {
+            TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
                 let mut lengths = vec![];
 
                 for item in self {
@@ -97,7 +97,7 @@ where
                     TreeHashCache::from_bytes(leaves, true)?,
                 );
             }
-            TreeHashType::Composite | TreeHashType::List => {
+            TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
                 let mut i = offset_handler.num_leaf_nodes;
                 for &start_chunk in offset_handler.iter_leaf_nodes().rev() {
                     i -= 1;
diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs
index 7c74c9f97..fe2001002 100644
--- a/eth2/utils/tree_hash/src/lib.rs
+++ b/eth2/utils/tree_hash/src/lib.rs
@@ -13,8 +13,9 @@ pub use standard_tree_hash::{merkle_root, TreeHash};
 #[derive(Debug, PartialEq, Clone)]
 pub enum TreeHashType {
     Basic,
+    Vector,
     List,
-    Composite,
+    Container,
 }
 
 fn num_sanitized_leaves(num_bytes: usize) -> usize {
@@ -31,15 +32,15 @@ macro_rules! impl_tree_hash_for_ssz_bytes {
     ($type: ident) => {
         impl tree_hash::TreeHash for $type {
             fn tree_hash_type() -> tree_hash::TreeHashType {
-                tree_hash::TreeHashType::List
+                tree_hash::TreeHashType::Vector
             }
 
             fn tree_hash_packed_encoding(&self) -> Vec<u8> {
-                panic!("bytesN should never be packed.")
+                unreachable!("Vector should never be packed.")
             }
 
             fn tree_hash_packing_factor() -> usize {
-                panic!("bytesN should never be packed.")
+                unreachable!("Vector should never be packed.")
             }
 
             fn tree_hash_root(&self) -> Vec<u8> {
@@ -48,3 +49,26 @@ macro_rules! impl_tree_hash_for_ssz_bytes {
         }
     };
 }
+
+#[macro_export]
+macro_rules! impl_vec_as_fixed_len {
+    ($type: ty) => {
+        impl tree_hash::TreeHash for $type {
+            fn tree_hash_type() -> tree_hash::TreeHashType {
+                tree_hash::TreeHashType::Vector
+            }
+
+            fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+                unreachable!("Vector should never be packed.")
+            }
+
+            fn tree_hash_packing_factor() -> usize {
+                unreachable!("Vector should never be packed.")
+            }
+
+            fn tree_hash_root(&self) -> Vec<u8> {
+                tree_hash::standard_tree_hash::vec_tree_hash_root(self)
+            }
+        }
+    };
+}
diff --git a/eth2/utils/tree_hash/src/standard_tree_hash.rs b/eth2/utils/tree_hash/src/standard_tree_hash.rs
index 473d2a5f0..130c360ed 100644
--- a/eth2/utils/tree_hash/src/standard_tree_hash.rs
+++ b/eth2/utils/tree_hash/src/standard_tree_hash.rs
@@ -3,6 +3,8 @@ use hashing::hash;
 use int_to_bytes::int_to_bytes32;
 use ssz::ssz_encode;
 
+pub use impls::vec_tree_hash_root;
+
 mod impls;
 
 pub trait TreeHash {
@@ -16,11 +18,18 @@ pub trait TreeHash {
 }
 
 pub fn merkle_root(bytes: &[u8]) -> Vec<u8> {
-    // TODO: replace this with a _more_ efficient fn which is more memory efficient.
+    // TODO: replace this with a more memory efficient method.
     efficient_merkleize(&bytes)[0..32].to_vec()
 }
 
 pub fn efficient_merkleize(bytes: &[u8]) -> Vec<u8> {
+    // If the bytes are just one chunk (or less than one chunk) just return them.
+    if bytes.len() <= HASHSIZE {
+        let mut o = bytes.to_vec();
+        o.resize(HASHSIZE, 0);
+        return o;
+    }
+
     let leaves = num_sanitized_leaves(bytes.len());
     let nodes = num_nodes(leaves);
     let internal_nodes = nodes - leaves;
@@ -29,10 +38,6 @@ pub fn efficient_merkleize(bytes: &[u8]) -> Vec<u8> {
 
     let mut o: Vec<u8> = vec![0; internal_nodes * HASHSIZE];
 
-    if o.len() < HASHSIZE {
-        o.resize(HASHSIZE, 0);
-    }
-
     o.append(&mut bytes.to_vec());
 
     assert_eq!(o.len(), num_bytes);
diff --git a/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs b/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs
index 749d5b3bb..c3be8d55b 100644
--- a/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs
+++ b/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs
@@ -50,7 +50,7 @@ impl TreeHash for [u8; 4] {
 
 impl TreeHash for H256 {
     fn tree_hash_type() -> TreeHashType {
-        TreeHashType::Basic
+        TreeHashType::Vector
     }
 
     fn tree_hash_packed_encoding(&self) -> Vec<u8> {
@@ -62,7 +62,7 @@ impl TreeHash for H256 {
     }
 
     fn tree_hash_root(&self) -> Vec<u8> {
-        ssz_encode(self)
+        merkle_root(&ssz::ssz_encode(self))
     }
 }
 
@@ -83,37 +83,43 @@ where
     }
 
     fn tree_hash_root(&self) -> Vec<u8> {
-        let leaves = match T::tree_hash_type() {
-            TreeHashType::Basic => {
-                let mut leaves =
-                    Vec::with_capacity((HASHSIZE / T::tree_hash_packing_factor()) * self.len());
-
-                for item in self {
-                    leaves.append(&mut item.tree_hash_packed_encoding());
-                }
-
-                leaves
-            }
-            TreeHashType::Composite | TreeHashType::List => {
-                let mut leaves = Vec::with_capacity(self.len() * HASHSIZE);
-
-                for item in self {
-                    leaves.append(&mut item.tree_hash_root())
-                }
-
-                leaves
-            }
-        };
-
-        // Mix in the length
         let mut root_and_len = Vec::with_capacity(HASHSIZE * 2);
-        root_and_len.append(&mut merkle_root(&leaves));
+        root_and_len.append(&mut vec_tree_hash_root(self));
         root_and_len.append(&mut int_to_bytes32(self.len() as u64));
 
         hash(&root_and_len)
     }
 }
 
+pub fn vec_tree_hash_root<T>(vec: &[T]) -> Vec<u8>
+where
+    T: TreeHash,
+{
+    let leaves = match T::tree_hash_type() {
+        TreeHashType::Basic => {
+            let mut leaves =
+                Vec::with_capacity((HASHSIZE / T::tree_hash_packing_factor()) * vec.len());
+
+            for item in vec {
+                leaves.append(&mut item.tree_hash_packed_encoding());
+            }
+
+            leaves
+        }
+        TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
+            let mut leaves = Vec::with_capacity(vec.len() * HASHSIZE);
+
+            for item in vec {
+                leaves.append(&mut item.tree_hash_root())
+            }
+
+            leaves
+        }
+    };
+
+    merkle_root(&leaves)
+}
+
 #[cfg(test)]
 mod test {
     use super::*;
diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
index db33709ac..4d2c6f282 100644
--- a/eth2/utils/tree_hash/tests/tests.rs
+++ b/eth2/utils/tree_hash/tests/tests.rs
@@ -13,7 +13,7 @@ pub struct InternalCache {
 
 impl TreeHash for InternalCache {
     fn tree_hash_type() -> TreeHashType {
-        TreeHashType::Composite
+        TreeHashType::Container
     }
 
     fn tree_hash_packed_encoding(&self) -> Vec<u8> {
@@ -146,7 +146,7 @@ pub struct Inner {
 
 impl TreeHash for Inner {
     fn tree_hash_type() -> TreeHashType {
-        TreeHashType::Composite
+        TreeHashType::Container
     }
 
     fn tree_hash_packed_encoding(&self) -> Vec<u8> {
@@ -231,7 +231,7 @@ pub struct Outer {
 
 impl TreeHash for Outer {
     fn tree_hash_type() -> TreeHashType {
-        TreeHashType::Composite
+        TreeHashType::Container
     }
 
     fn tree_hash_packed_encoding(&self) -> Vec<u8> {
@@ -894,11 +894,39 @@ fn vec_of_u64_builds() {
 
     let my_vec = vec![1, 2, 3, 4, 5];
 
+    //
+    // Note: the length is not mixed-in in this example. The user must ensure the length is
+    // mixed-in.
+    //
+
     let cache: Vec<u8> = TreeHashCache::new(&my_vec).unwrap().into();
 
     assert_eq!(expected, cache);
 }
 
+#[test]
+fn vec_does_mix_in_len() {
+    let data = join(vec![
+        int_to_bytes8(1),
+        int_to_bytes8(2),
+        int_to_bytes8(3),
+        int_to_bytes8(4),
+        int_to_bytes8(5),
+        vec![0; 32 - 8], // padding
+    ]);
+
+    let tree = merkleize(data);
+
+    let my_vec: Vec<u64> = vec![1, 2, 3, 4, 5];
+
+    let mut expected = vec![0; 32];
+    expected.copy_from_slice(&tree[0..HASHSIZE]);
+    expected.append(&mut int_to_bytes32(my_vec.len() as u64));
+    let expected = hash(&expected);
+
+    assert_eq!(&expected[0..HASHSIZE], &my_vec.tree_hash_root()[..]);
+}
+
 #[test]
 fn merkleize_odd() {
     let data = join(vec![
diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs
index e3a7b4aaa..4b7761f91 100644
--- a/eth2/utils/tree_hash_derive/src/lib.rs
+++ b/eth2/utils/tree_hash_derive/src/lib.rs
@@ -129,7 +129,7 @@ pub fn tree_hash_derive(input: TokenStream) -> TokenStream {
     let output = quote! {
         impl tree_hash::TreeHash for #name {
             fn tree_hash_type() -> tree_hash::TreeHashType {
-                tree_hash::TreeHashType::Composite
+                tree_hash::TreeHashType::Container
             }
 
             fn tree_hash_packed_encoding(&self) -> Vec<u8> {
@@ -154,19 +154,6 @@ pub fn tree_hash_derive(input: TokenStream) -> TokenStream {
     output.into()
 }
 
-/// Implements `tree_hash::TreeHash` for some `struct`, whilst excluding any fields following and
-/// including a field that is of type "Signature" or "AggregateSignature".
-///
-/// See:
-/// https://github.com/ethereum/eth2.0-specs/blob/master/specs/simple-serialize.md#signed-roots
-///
-/// This is a rather horrendous macro, it will read the type of the object as a string and decide
-/// if it's a signature by matching that string against "Signature" or "AggregateSignature". So,
-/// it's important that you use those exact words as your type -- don't alias it to something else.
-///
-/// If you can think of a better way to do this, please make an issue!
-///
-/// Fields are processed in the order they are defined.
 #[proc_macro_derive(SignedRoot, attributes(signed_root))]
 pub fn tree_hash_signed_root_derive(input: TokenStream) -> TokenStream {
     let item = parse_macro_input!(input as DeriveInput);

From 10eeced227fe1be279e2ef92dea1ee8a02d86f5c Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 17 Apr 2019 11:18:00 +1000
Subject: [PATCH 061/137] Remove SSZ dep from `tree_hash`

---
 eth2/utils/tree_hash/Cargo.toml               |  1 -
 .../tree_hash/src/cached_tree_hash/impls.rs   |  5 ++--
 eth2/utils/tree_hash/src/lib.rs               | 23 ----------------
 .../utils/tree_hash/src/standard_tree_hash.rs |  1 -
 .../tree_hash/src/standard_tree_hash/impls.rs | 27 +++++++++++++++----
 5 files changed, 24 insertions(+), 33 deletions(-)

diff --git a/eth2/utils/tree_hash/Cargo.toml b/eth2/utils/tree_hash/Cargo.toml
index 243a49446..328d91577 100644
--- a/eth2/utils/tree_hash/Cargo.toml
+++ b/eth2/utils/tree_hash/Cargo.toml
@@ -8,4 +8,3 @@ edition = "2018"
 ethereum-types = "0.5"
 hashing = { path = "../hashing" }
 int_to_bytes = { path = "../int_to_bytes" }
-ssz = { path = "../ssz" }
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
index 62d013881..6500e4eff 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
@@ -1,13 +1,12 @@
 use super::resize::{grow_merkle_cache, shrink_merkle_cache};
 use super::*;
-use ssz::ssz_encode;
 
 mod vec;
 
 impl CachedTreeHashSubTree<u64> for u64 {
     fn new_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
         Ok(TreeHashCache::from_bytes(
-            merkleize(ssz_encode(self)),
+            merkleize(self.to_le_bytes().to_vec()),
             false,
         )?)
     }
@@ -23,7 +22,7 @@ impl CachedTreeHashSubTree<u64> for u64 {
         chunk: usize,
     ) -> Result<usize, Error> {
         if self != other {
-            let leaf = merkleize(ssz_encode(self));
+            let leaf = merkleize(self.to_le_bytes().to_vec());
             cache.modify_chunk(chunk, &leaf)?;
         }
 
diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs
index fe2001002..fd1708a2d 100644
--- a/eth2/utils/tree_hash/src/lib.rs
+++ b/eth2/utils/tree_hash/src/lib.rs
@@ -49,26 +49,3 @@ macro_rules! impl_tree_hash_for_ssz_bytes {
         }
     };
 }
-
-#[macro_export]
-macro_rules! impl_vec_as_fixed_len {
-    ($type: ty) => {
-        impl tree_hash::TreeHash for $type {
-            fn tree_hash_type() -> tree_hash::TreeHashType {
-                tree_hash::TreeHashType::Vector
-            }
-
-            fn tree_hash_packed_encoding(&self) -> Vec<u8> {
-                unreachable!("Vector should never be packed.")
-            }
-
-            fn tree_hash_packing_factor() -> usize {
-                unreachable!("Vector should never be packed.")
-            }
-
-            fn tree_hash_root(&self) -> Vec<u8> {
-                tree_hash::standard_tree_hash::vec_tree_hash_root(self)
-            }
-        }
-    };
-}
diff --git a/eth2/utils/tree_hash/src/standard_tree_hash.rs b/eth2/utils/tree_hash/src/standard_tree_hash.rs
index 130c360ed..812a2c352 100644
--- a/eth2/utils/tree_hash/src/standard_tree_hash.rs
+++ b/eth2/utils/tree_hash/src/standard_tree_hash.rs
@@ -1,7 +1,6 @@
 use super::*;
 use hashing::hash;
 use int_to_bytes::int_to_bytes32;
-use ssz::ssz_encode;
 
 pub use impls::vec_tree_hash_root;
 
diff --git a/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs b/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs
index c3be8d55b..be6b4ba07 100644
--- a/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs
+++ b/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs
@@ -9,7 +9,7 @@ macro_rules! impl_for_bitsize {
             }
 
             fn tree_hash_packed_encoding(&self) -> Vec<u8> {
-                ssz_encode(self)
+                self.to_le_bytes().to_vec()
             }
 
             fn tree_hash_packing_factor() -> usize {
@@ -28,7 +28,24 @@ impl_for_bitsize!(u16, 16);
 impl_for_bitsize!(u32, 32);
 impl_for_bitsize!(u64, 64);
 impl_for_bitsize!(usize, 64);
-impl_for_bitsize!(bool, 8);
+
+impl TreeHash for bool {
+    fn tree_hash_type() -> TreeHashType {
+        TreeHashType::Basic
+    }
+
+    fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+        (*self as u8).tree_hash_packed_encoding()
+    }
+
+    fn tree_hash_packing_factor() -> usize {
+        u8::tree_hash_packing_factor()
+    }
+
+    fn tree_hash_root(&self) -> Vec<u8> {
+        int_to_bytes32(*self as u64)
+    }
+}
 
 impl TreeHash for [u8; 4] {
     fn tree_hash_type() -> TreeHashType {
@@ -44,7 +61,7 @@ impl TreeHash for [u8; 4] {
     }
 
     fn tree_hash_root(&self) -> Vec<u8> {
-        merkle_root(&ssz::ssz_encode(self))
+        merkle_root(&self[..])
     }
 }
 
@@ -54,7 +71,7 @@ impl TreeHash for H256 {
     }
 
     fn tree_hash_packed_encoding(&self) -> Vec<u8> {
-        ssz_encode(self)
+        self.as_bytes().to_vec()
     }
 
     fn tree_hash_packing_factor() -> usize {
@@ -62,7 +79,7 @@ impl TreeHash for H256 {
     }
 
     fn tree_hash_root(&self) -> Vec<u8> {
-        merkle_root(&ssz::ssz_encode(self))
+        merkle_root(&self.as_bytes().to_vec())
     }
 }
 

From ea8d5a3db9856eb54f5f8926253448e79834faff Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 17 Apr 2019 11:57:34 +1000
Subject: [PATCH 062/137] Ensure deposit uses correct list type

---
 eth2/types/src/deposit.rs                            | 4 ++--
 eth2/types/src/test_utils/testing_deposit_builder.rs | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/eth2/types/src/deposit.rs b/eth2/types/src/deposit.rs
index 5eb565c2b..bd3355a3f 100644
--- a/eth2/types/src/deposit.rs
+++ b/eth2/types/src/deposit.rs
@@ -1,4 +1,4 @@
-use super::{DepositData, Hash256};
+use super::{DepositData, Hash256, TreeHashVector};
 use crate::test_utils::TestRandom;
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
@@ -11,7 +11,7 @@ use tree_hash_derive::TreeHash;
 /// Spec v0.5.0
 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
 pub struct Deposit {
-    pub proof: Vec<Hash256>,
+    pub proof: TreeHashVector<Hash256>,
     pub index: u64,
     pub deposit_data: DepositData,
 }
diff --git a/eth2/types/src/test_utils/testing_deposit_builder.rs b/eth2/types/src/test_utils/testing_deposit_builder.rs
index 326858c31..080ed5cfb 100644
--- a/eth2/types/src/test_utils/testing_deposit_builder.rs
+++ b/eth2/types/src/test_utils/testing_deposit_builder.rs
@@ -12,7 +12,7 @@ impl TestingDepositBuilder {
     /// Instantiates a new builder.
     pub fn new(pubkey: PublicKey, amount: u64) -> Self {
         let deposit = Deposit {
-            proof: vec![],
+            proof: vec![].into(),
             index: 0,
             deposit_data: DepositData {
                 amount,

From 10a5d2657caf402347d257c8b4429716c27355d8 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 17 Apr 2019 11:57:57 +1000
Subject: [PATCH 063/137] Encode bitfield as list not vector

---
 eth2/utils/bls/src/aggregate_signature.rs     |  4 ++--
 .../utils/bls/src/fake_aggregate_signature.rs |  4 ++--
 eth2/utils/bls/src/fake_signature.rs          |  4 ++--
 eth2/utils/bls/src/public_key.rs              |  4 ++--
 eth2/utils/bls/src/secret_key.rs              |  4 ++--
 eth2/utils/bls/src/signature.rs               |  4 ++--
 eth2/utils/boolean-bitfield/src/lib.rs        |  4 ++--
 eth2/utils/tree_hash/src/lib.rs               | 24 ++++++++++++++++++-
 8 files changed, 37 insertions(+), 15 deletions(-)

diff --git a/eth2/utils/bls/src/aggregate_signature.rs b/eth2/utils/bls/src/aggregate_signature.rs
index f26bd2db6..0fbcc3493 100644
--- a/eth2/utils/bls/src/aggregate_signature.rs
+++ b/eth2/utils/bls/src/aggregate_signature.rs
@@ -6,7 +6,7 @@ use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::{encode as hex_encode, HexVisitor};
 use ssz::{decode, Decodable, DecodeError, Encodable, SszStream};
-use tree_hash::impl_tree_hash_for_ssz_bytes;
+use tree_hash::tree_hash_ssz_encoding_as_vector;
 
 /// A BLS aggregate signature.
 ///
@@ -166,7 +166,7 @@ impl<'de> Deserialize<'de> for AggregateSignature {
     }
 }
 
-impl_tree_hash_for_ssz_bytes!(AggregateSignature);
+tree_hash_ssz_encoding_as_vector!(AggregateSignature);
 
 #[cfg(test)]
 mod tests {
diff --git a/eth2/utils/bls/src/fake_aggregate_signature.rs b/eth2/utils/bls/src/fake_aggregate_signature.rs
index 602639b6b..f201eba3e 100644
--- a/eth2/utils/bls/src/fake_aggregate_signature.rs
+++ b/eth2/utils/bls/src/fake_aggregate_signature.rs
@@ -3,7 +3,7 @@ use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::{encode as hex_encode, PrefixedHexVisitor};
 use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream};
-use tree_hash::impl_tree_hash_for_ssz_bytes;
+use tree_hash::tree_hash_ssz_encoding_as_vector;
 
 /// A BLS aggregate signature.
 ///
@@ -99,7 +99,7 @@ impl<'de> Deserialize<'de> for FakeAggregateSignature {
     }
 }
 
-impl_tree_hash_for_ssz_bytes!(FakeAggregateSignature);
+tree_hash_ssz_encoding_as_vector!(FakeAggregateSignature);
 
 #[cfg(test)]
 mod tests {
diff --git a/eth2/utils/bls/src/fake_signature.rs b/eth2/utils/bls/src/fake_signature.rs
index b07dd66a5..3208ed992 100644
--- a/eth2/utils/bls/src/fake_signature.rs
+++ b/eth2/utils/bls/src/fake_signature.rs
@@ -4,7 +4,7 @@ use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::HexVisitor;
 use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream};
-use tree_hash::impl_tree_hash_for_ssz_bytes;
+use tree_hash::tree_hash_ssz_encoding_as_vector;
 
 /// A single BLS signature.
 ///
@@ -74,7 +74,7 @@ impl Decodable for FakeSignature {
     }
 }
 
-impl_tree_hash_for_ssz_bytes!(FakeSignature);
+tree_hash_ssz_encoding_as_vector!(FakeSignature);
 
 impl Serialize for FakeSignature {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
diff --git a/eth2/utils/bls/src/public_key.rs b/eth2/utils/bls/src/public_key.rs
index a553ee888..dcbbc622a 100644
--- a/eth2/utils/bls/src/public_key.rs
+++ b/eth2/utils/bls/src/public_key.rs
@@ -7,7 +7,7 @@ use ssz::{decode, ssz_encode, Decodable, DecodeError, Encodable, SszStream};
 use std::default;
 use std::fmt;
 use std::hash::{Hash, Hasher};
-use tree_hash::impl_tree_hash_for_ssz_bytes;
+use tree_hash::tree_hash_ssz_encoding_as_vector;
 
 /// A single BLS signature.
 ///
@@ -105,7 +105,7 @@ impl<'de> Deserialize<'de> for PublicKey {
     }
 }
 
-impl_tree_hash_for_ssz_bytes!(PublicKey);
+tree_hash_ssz_encoding_as_vector!(PublicKey);
 
 impl PartialEq for PublicKey {
     fn eq(&self, other: &PublicKey) -> bool {
diff --git a/eth2/utils/bls/src/secret_key.rs b/eth2/utils/bls/src/secret_key.rs
index 38fd2d379..d1aaa96da 100644
--- a/eth2/utils/bls/src/secret_key.rs
+++ b/eth2/utils/bls/src/secret_key.rs
@@ -5,7 +5,7 @@ use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::HexVisitor;
 use ssz::{decode, ssz_encode, Decodable, DecodeError, Encodable, SszStream};
-use tree_hash::impl_tree_hash_for_ssz_bytes;
+use tree_hash::tree_hash_ssz_encoding_as_vector;
 
 /// A single BLS signature.
 ///
@@ -70,7 +70,7 @@ impl<'de> Deserialize<'de> for SecretKey {
     }
 }
 
-impl_tree_hash_for_ssz_bytes!(SecretKey);
+tree_hash_ssz_encoding_as_vector!(SecretKey);
 
 #[cfg(test)]
 mod tests {
diff --git a/eth2/utils/bls/src/signature.rs b/eth2/utils/bls/src/signature.rs
index cf6c8fe5a..3fb68dc53 100644
--- a/eth2/utils/bls/src/signature.rs
+++ b/eth2/utils/bls/src/signature.rs
@@ -5,7 +5,7 @@ use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::HexVisitor;
 use ssz::{decode, ssz_encode, Decodable, DecodeError, Encodable, SszStream};
-use tree_hash::impl_tree_hash_for_ssz_bytes;
+use tree_hash::tree_hash_ssz_encoding_as_vector;
 
 /// A single BLS signature.
 ///
@@ -115,7 +115,7 @@ impl Decodable for Signature {
     }
 }
 
-impl_tree_hash_for_ssz_bytes!(Signature);
+tree_hash_ssz_encoding_as_vector!(Signature);
 
 impl Serialize for Signature {
     /// Serde serialization is compliant the Ethereum YAML test format.
diff --git a/eth2/utils/boolean-bitfield/src/lib.rs b/eth2/utils/boolean-bitfield/src/lib.rs
index fbd0e2ecd..d35d87c5c 100644
--- a/eth2/utils/boolean-bitfield/src/lib.rs
+++ b/eth2/utils/boolean-bitfield/src/lib.rs
@@ -9,7 +9,7 @@ use serde_hex::{encode, PrefixedHexVisitor};
 use ssz::{Decodable, Encodable};
 use std::cmp;
 use std::default;
-use tree_hash::impl_tree_hash_for_ssz_bytes;
+use tree_hash::tree_hash_ssz_encoding_as_list;
 
 /// A BooleanBitfield represents a set of booleans compactly stored as a vector of bits.
 /// The BooleanBitfield is given a fixed size during construction. Reads outside of the current size return an out-of-bounds error. Writes outside of the current size expand the size of the set.
@@ -257,7 +257,7 @@ impl<'de> Deserialize<'de> for BooleanBitfield {
     }
 }
 
-impl_tree_hash_for_ssz_bytes!(BooleanBitfield);
+tree_hash_ssz_encoding_as_list!(BooleanBitfield);
 
 #[cfg(test)]
 mod tests {
diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs
index fd1708a2d..ed60079c8 100644
--- a/eth2/utils/tree_hash/src/lib.rs
+++ b/eth2/utils/tree_hash/src/lib.rs
@@ -28,7 +28,7 @@ fn num_nodes(num_leaves: usize) -> usize {
 }
 
 #[macro_export]
-macro_rules! impl_tree_hash_for_ssz_bytes {
+macro_rules! tree_hash_ssz_encoding_as_vector {
     ($type: ident) => {
         impl tree_hash::TreeHash for $type {
             fn tree_hash_type() -> tree_hash::TreeHashType {
@@ -49,3 +49,25 @@ macro_rules! impl_tree_hash_for_ssz_bytes {
         }
     };
 }
+#[macro_export]
+macro_rules! tree_hash_ssz_encoding_as_list {
+    ($type: ident) => {
+        impl tree_hash::TreeHash for $type {
+            fn tree_hash_type() -> tree_hash::TreeHashType {
+                tree_hash::TreeHashType::List
+            }
+
+            fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+                unreachable!("List should never be packed.")
+            }
+
+            fn tree_hash_packing_factor() -> usize {
+                unreachable!("List should never be packed.")
+            }
+
+            fn tree_hash_root(&self) -> Vec<u8> {
+                ssz::ssz_encode(self).tree_hash_root()
+            }
+        }
+    };
+}

From 8da8730dca4fa43edf0cca37963f475c108fdbf3 Mon Sep 17 00:00:00 2001
From: Michael Sproul <michael@sigmaprime.io>
Date: Tue, 2 Apr 2019 10:22:19 +1100
Subject: [PATCH 064/137] spec: check ProposalSlashing epochs, not slots

As per v0.5.{0,1} of the spec, we only need to check that the epochs of two proposal slashings are
equal, not their slots.
---
 eth2/state_processing/src/per_block_processing/errors.rs     | 4 ++--
 .../src/per_block_processing/verify_proposer_slashing.rs     | 5 +++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/eth2/state_processing/src/per_block_processing/errors.rs b/eth2/state_processing/src/per_block_processing/errors.rs
index 6614f6f60..9c36e0238 100644
--- a/eth2/state_processing/src/per_block_processing/errors.rs
+++ b/eth2/state_processing/src/per_block_processing/errors.rs
@@ -271,10 +271,10 @@ pub enum ProposerSlashingValidationError {
 pub enum ProposerSlashingInvalid {
     /// The proposer index is not a known validator.
     ProposerUnknown(u64),
-    /// The two proposal have different slots.
+    /// The two proposal have different epochs.
     ///
     /// (proposal_1_slot, proposal_2_slot)
-    ProposalSlotMismatch(Slot, Slot),
+    ProposalEpochMismatch(Slot, Slot),
     /// The proposals are identical and therefore not slashable.
     ProposalsIdentical,
     /// The specified proposer has already been slashed.
diff --git a/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs b/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs
index 8e0a70f96..b5113863e 100644
--- a/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs
+++ b/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs
@@ -21,8 +21,9 @@ pub fn verify_proposer_slashing(
         })?;
 
     verify!(
-        proposer_slashing.header_1.slot == proposer_slashing.header_2.slot,
-        Invalid::ProposalSlotMismatch(
+        proposer_slashing.header_1.slot.epoch(spec.slots_per_epoch)
+            == proposer_slashing.header_2.slot.epoch(spec.slots_per_epoch),
+        Invalid::ProposalEpochMismatch(
             proposer_slashing.header_1.slot,
             proposer_slashing.header_2.slot
         )

From 0a02567440f1e77dda40e0904b63077fb238d4e0 Mon Sep 17 00:00:00 2001
From: Michael Sproul <michael@sigmaprime.io>
Date: Tue, 2 Apr 2019 14:14:57 +1100
Subject: [PATCH 065/137] bitfield: fix bit ordering issue with YAML parsing

---
 .../src/common/verify_bitfield.rs             | 13 ++++++
 eth2/utils/boolean-bitfield/Cargo.toml        |  4 ++
 eth2/utils/boolean-bitfield/src/lib.rs        | 44 ++++++++++++++++---
 3 files changed, 56 insertions(+), 5 deletions(-)

diff --git a/eth2/state_processing/src/common/verify_bitfield.rs b/eth2/state_processing/src/common/verify_bitfield.rs
index 03fcdbb67..71c9f9c3e 100644
--- a/eth2/state_processing/src/common/verify_bitfield.rs
+++ b/eth2/state_processing/src/common/verify_bitfield.rs
@@ -18,3 +18,16 @@ pub fn verify_bitfield_length(bitfield: &Bitfield, committee_size: usize) -> boo
 
     true
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn bitfield_length() {
+        assert!(verify_bitfield_length(
+            &Bitfield::from_bytes(&[0b10000000]),
+            4
+        ));
+    }
+}
diff --git a/eth2/utils/boolean-bitfield/Cargo.toml b/eth2/utils/boolean-bitfield/Cargo.toml
index f08695bd1..61bbc60a8 100644
--- a/eth2/utils/boolean-bitfield/Cargo.toml
+++ b/eth2/utils/boolean-bitfield/Cargo.toml
@@ -8,6 +8,10 @@ edition = "2018"
 serde_hex = { path = "../serde_hex" }
 ssz = { path = "../ssz" }
 bit-vec = "0.5.0"
+bit_reverse = "0.1"
 serde = "1.0"
 serde_derive = "1.0"
 tree_hash = { path = "../tree_hash" }
+
+[dev-dependencies]
+serde_yaml = "0.8"
diff --git a/eth2/utils/boolean-bitfield/src/lib.rs b/eth2/utils/boolean-bitfield/src/lib.rs
index d35d87c5c..c19702ec9 100644
--- a/eth2/utils/boolean-bitfield/src/lib.rs
+++ b/eth2/utils/boolean-bitfield/src/lib.rs
@@ -1,8 +1,8 @@
 extern crate bit_vec;
 extern crate ssz;
 
+use bit_reverse::LookupReverse;
 use bit_vec::BitVec;
-
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::{encode, PrefixedHexVisitor};
@@ -236,24 +236,36 @@ impl Decodable for BooleanBitfield {
     }
 }
 
+// Reverse the bit order of a whole byte vec, so that the ith bit
+// of the input vec is placed in the (N - i)th bit of the output vec.
+// This function is necessary for converting bitfields to and from YAML,
+// as the BitVec library and the hex-parser use opposing bit orders.
+fn reverse_bit_order(mut bytes: Vec<u8>) -> Vec<u8> {
+    bytes.reverse();
+    bytes.into_iter().map(|b| b.swap_bits()).collect()
+}
+
 impl Serialize for BooleanBitfield {
-    /// Serde serialization is compliant the Ethereum YAML test format.
+    /// Serde serialization is compliant with the Ethereum YAML test format.
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
     where
         S: Serializer,
     {
-        serializer.serialize_str(&encode(&self.to_bytes()))
+        serializer.serialize_str(&encode(&reverse_bit_order(self.to_bytes())))
     }
 }
 
 impl<'de> Deserialize<'de> for BooleanBitfield {
-    /// Serde serialization is compliant the Ethereum YAML test format.
+    /// Serde serialization is compliant with the Ethereum YAML test format.
     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
         D: Deserializer<'de>,
     {
+        // We reverse the bit-order so that the BitVec library can read its 0th
+        // bit from the end of the hex string, e.g.
+        // "0xef01" => [0xef, 0x01] => [0b1000_0000, 0b1111_1110]
         let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?;
-        Ok(BooleanBitfield::from_bytes(&bytes))
+        Ok(BooleanBitfield::from_bytes(&reverse_bit_order(bytes)))
     }
 }
 
@@ -262,6 +274,7 @@ tree_hash_ssz_encoding_as_list!(BooleanBitfield);
 #[cfg(test)]
 mod tests {
     use super::*;
+    use serde_yaml;
     use ssz::{decode, ssz_encode, SszStream};
 
     #[test]
@@ -462,6 +475,27 @@ mod tests {
         assert_eq!(field, expected);
     }
 
+    #[test]
+    fn test_serialize_deserialize() {
+        use serde_yaml::Value;
+
+        let data: &[(_, &[_])] = &[
+            ("0x01", &[0b10000000]),
+            ("0xf301", &[0b10000000, 0b11001111]),
+        ];
+        for (hex_data, bytes) in data {
+            let bitfield = BooleanBitfield::from_bytes(bytes);
+            assert_eq!(
+                serde_yaml::from_str::<BooleanBitfield>(hex_data).unwrap(),
+                bitfield
+            );
+            assert_eq!(
+                serde_yaml::to_value(&bitfield).unwrap(),
+                Value::String(hex_data.to_string())
+            );
+        }
+    }
+
     #[test]
     fn test_ssz_round_trip() {
         let original = BooleanBitfield::from_bytes(&vec![18; 12][..]);

From 300fcd6ec3587ab6377acc210693c2f31bc94395 Mon Sep 17 00:00:00 2001
From: Michael Sproul <michael@sigmaprime.io>
Date: Tue, 2 Apr 2019 14:17:41 +1100
Subject: [PATCH 066/137] state transition test progress

---
 eth2/state_processing/tests/tests.rs        | 74 +++++++++------------
 eth2/state_processing/yaml_utils/Cargo.toml |  1 -
 eth2/state_processing/yaml_utils/build.rs   |  1 -
 3 files changed, 32 insertions(+), 44 deletions(-)

diff --git a/eth2/state_processing/tests/tests.rs b/eth2/state_processing/tests/tests.rs
index 1359508dc..03401b2c7 100644
--- a/eth2/state_processing/tests/tests.rs
+++ b/eth2/state_processing/tests/tests.rs
@@ -27,61 +27,28 @@ pub struct TestDoc {
     pub test_cases: Vec<TestCase>,
 }
 
-#[test]
-fn test_read_yaml() {
-    // Test sanity-check_small-config_32-vals.yaml
+fn load_test_case(test_name: &str) -> TestDoc {
     let mut file = {
         let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-        file_path_buf.push("yaml_utils/specs/sanity-check_small-config_32-vals.yaml");
+        file_path_buf.push(format!("yaml_utils/specs/{}", test_name));
 
         File::open(file_path_buf).unwrap()
     };
-
     let mut yaml_str = String::new();
-
     file.read_to_string(&mut yaml_str).unwrap();
-
     yaml_str = yaml_str.to_lowercase();
 
-    let _doc: TestDoc = serde_yaml::from_str(&yaml_str.as_str()).unwrap();
-
-    // Test sanity-check_default-config_100-vals.yaml
-    file = {
-        let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-        file_path_buf.push("yaml_utils/specs/sanity-check_default-config_100-vals.yaml");
-
-        File::open(file_path_buf).unwrap()
-    };
-
-    yaml_str = String::new();
-
-    file.read_to_string(&mut yaml_str).unwrap();
-
-    yaml_str = yaml_str.to_lowercase();
-
-    let _doc: TestDoc = serde_yaml::from_str(&yaml_str.as_str()).unwrap();
+    serde_yaml::from_str(&yaml_str.as_str()).unwrap()
 }
 
-#[test]
-#[cfg(not(debug_assertions))]
-fn run_state_transition_tests_small() {
-    // Test sanity-check_small-config_32-vals.yaml
-    let mut file = {
-        let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-        file_path_buf.push("yaml_utils/specs/sanity-check_small-config_32-vals.yaml");
-
-        File::open(file_path_buf).unwrap()
-    };
-    let mut yaml_str = String::new();
-    file.read_to_string(&mut yaml_str).unwrap();
-    yaml_str = yaml_str.to_lowercase();
-
-    let doc: TestDoc = serde_yaml::from_str(&yaml_str.as_str()).unwrap();
+fn run_state_transition_test(test_name: &str) {
+    let doc = load_test_case(test_name);
 
     // Run Tests
+    let mut ok = true;
     for (i, test_case) in doc.test_cases.iter().enumerate() {
         let mut state = test_case.initial_state.clone();
-        for block in test_case.blocks.iter() {
+        for (j, block) in test_case.blocks.iter().enumerate() {
             while block.slot > state.slot {
                 let latest_block_header = state.latest_block_header.clone();
                 per_slot_processing(&mut state, &latest_block_header, &test_case.config).unwrap();
@@ -89,8 +56,9 @@ fn run_state_transition_tests_small() {
             if test_case.verify_signatures {
                 let res = per_block_processing(&mut state, &block, &test_case.config);
                 if res.is_err() {
-                    println!("{:?}", i);
+                    println!("Error in {} (#{}), on block {}", test_case.name, i, j);
                     println!("{:?}", res);
+                    ok = false;
                 };
             } else {
                 let res = per_block_processing_without_verifying_block_signature(
@@ -99,10 +67,32 @@ fn run_state_transition_tests_small() {
                     &test_case.config,
                 );
                 if res.is_err() {
-                    println!("{:?}", i);
+                    println!("Error in {} (#{}), on block {}", test_case.name, i, j);
                     println!("{:?}", res);
+                    ok = false;
                 }
             }
         }
     }
+
+    assert!(ok, "one or more tests failed, see above");
+}
+
+#[test]
+#[cfg(not(debug_assertions))]
+fn test_read_yaml() {
+    load_test_case("sanity-check_small-config_32-vals.yaml");
+    load_test_case("sanity-check_default-config_100-vals.yaml");
+}
+
+#[test]
+#[cfg(not(debug_assertions))]
+fn run_state_transition_tests_small() {
+    run_state_transition_test("sanity-check_small-config_32-vals.yaml");
+}
+
+#[test]
+#[cfg(not(debug_assertions))]
+fn run_state_transition_tests_large() {
+    run_state_transition_test("sanity-check_default-config_100-vals.yaml");
 }
diff --git a/eth2/state_processing/yaml_utils/Cargo.toml b/eth2/state_processing/yaml_utils/Cargo.toml
index 4a7ae5b89..5f216fe1a 100644
--- a/eth2/state_processing/yaml_utils/Cargo.toml
+++ b/eth2/state_processing/yaml_utils/Cargo.toml
@@ -6,7 +6,6 @@ edition = "2018"
 
 [build-dependencies]
 reqwest = "0.9"
-tempdir = "0.3"
 
 [dependencies]
 
diff --git a/eth2/state_processing/yaml_utils/build.rs b/eth2/state_processing/yaml_utils/build.rs
index 3b7f31471..7fb652cc1 100644
--- a/eth2/state_processing/yaml_utils/build.rs
+++ b/eth2/state_processing/yaml_utils/build.rs
@@ -1,5 +1,4 @@
 extern crate reqwest;
-extern crate tempdir;
 
 use std::fs::File;
 use std::io::copy;

From 71a0fed8eb08eba866f553a658e142c8321594a0 Mon Sep 17 00:00:00 2001
From: Michael Sproul <michael@sigmaprime.io>
Date: Tue, 2 Apr 2019 17:51:12 +1100
Subject: [PATCH 067/137] testing: add a `fake_crypto` feature

---
 eth2/state_processing/Cargo.toml     |  3 +++
 eth2/state_processing/build.rs       |  1 +
 eth2/state_processing/tests/tests.rs | 35 +++++++++++++---------------
 eth2/utils/bls/Cargo.toml            |  3 +++
 eth2/utils/bls/build.rs              | 19 +++++++++++++++
 eth2/utils/bls/src/lib.rs            | 16 ++++++-------
 6 files changed, 50 insertions(+), 27 deletions(-)
 create mode 120000 eth2/state_processing/build.rs
 create mode 100644 eth2/utils/bls/build.rs

diff --git a/eth2/state_processing/Cargo.toml b/eth2/state_processing/Cargo.toml
index 1bc7a6c45..a2ae11aa8 100644
--- a/eth2/state_processing/Cargo.toml
+++ b/eth2/state_processing/Cargo.toml
@@ -30,3 +30,6 @@ tree_hash = { path = "../utils/tree_hash" }
 tree_hash_derive = { path = "../utils/tree_hash_derive" }
 types = { path = "../types" }
 rayon = "1.0"
+
+[features]
+fake_crypto = ["bls/fake_crypto"]
diff --git a/eth2/state_processing/build.rs b/eth2/state_processing/build.rs
new file mode 120000
index 000000000..70d6c75b9
--- /dev/null
+++ b/eth2/state_processing/build.rs
@@ -0,0 +1 @@
+../utils/bls/build.rs
\ No newline at end of file
diff --git a/eth2/state_processing/tests/tests.rs b/eth2/state_processing/tests/tests.rs
index 03401b2c7..6ea8863b8 100644
--- a/eth2/state_processing/tests/tests.rs
+++ b/eth2/state_processing/tests/tests.rs
@@ -47,31 +47,28 @@ fn run_state_transition_test(test_name: &str) {
     // Run Tests
     let mut ok = true;
     for (i, test_case) in doc.test_cases.iter().enumerate() {
+        let fake_crypto = cfg!(feature = "fake_crypto");
+        if !test_case.verify_signatures == fake_crypto {
+            println!("Running {}", test_case.name);
+        } else {
+            println!(
+                "Skipping {} (fake_crypto: {}, need fake: {})",
+                test_case.name, fake_crypto, !test_case.verify_signatures
+            );
+            continue;
+        }
         let mut state = test_case.initial_state.clone();
         for (j, block) in test_case.blocks.iter().enumerate() {
             while block.slot > state.slot {
                 let latest_block_header = state.latest_block_header.clone();
                 per_slot_processing(&mut state, &latest_block_header, &test_case.config).unwrap();
             }
-            if test_case.verify_signatures {
-                let res = per_block_processing(&mut state, &block, &test_case.config);
-                if res.is_err() {
-                    println!("Error in {} (#{}), on block {}", test_case.name, i, j);
-                    println!("{:?}", res);
-                    ok = false;
-                };
-            } else {
-                let res = per_block_processing_without_verifying_block_signature(
-                    &mut state,
-                    &block,
-                    &test_case.config,
-                );
-                if res.is_err() {
-                    println!("Error in {} (#{}), on block {}", test_case.name, i, j);
-                    println!("{:?}", res);
-                    ok = false;
-                }
-            }
+            let res = per_block_processing(&mut state, &block, &test_case.config);
+            if res.is_err() {
+                println!("Error in {} (#{}), on block {}", test_case.name, i, j);
+                println!("{:?}", res);
+                ok = false;
+            };
         }
     }
 
diff --git a/eth2/utils/bls/Cargo.toml b/eth2/utils/bls/Cargo.toml
index 439debdcb..4ce499580 100644
--- a/eth2/utils/bls/Cargo.toml
+++ b/eth2/utils/bls/Cargo.toml
@@ -13,3 +13,6 @@ serde_derive = "1.0"
 serde_hex = { path = "../serde_hex" }
 ssz = { path = "../ssz" }
 tree_hash = { path = "../tree_hash" }
+
+[features]
+fake_crypto = []
diff --git a/eth2/utils/bls/build.rs b/eth2/utils/bls/build.rs
new file mode 100644
index 000000000..7f08a1ed5
--- /dev/null
+++ b/eth2/utils/bls/build.rs
@@ -0,0 +1,19 @@
+// This build script is symlinked from each project that requires BLS's "fake crypto",
+// so that the `fake_crypto` feature of every sub-crate can be turned on by running
+// with FAKE_CRYPTO=1 from the top-level workspace.
+// At some point in the future it might be possible to do:
+// $ cargo test --all --release --features fake_crypto
+// but at the present time this doesn't work.
+// Related: https://github.com/rust-lang/cargo/issues/5364
+fn main() {
+    if let Ok(fake_crypto) = std::env::var("FAKE_CRYPTO") {
+        if fake_crypto == "1" {
+            println!("cargo:rustc-cfg=feature=\"fake_crypto\"");
+            println!("cargo:rerun-if-env-changed=FAKE_CRYPTO");
+            println!(
+                "cargo:warning=[{}]: Compiled with fake BLS cryptography. DO NOT USE, TESTING ONLY",
+                std::env::var("CARGO_PKG_NAME").unwrap()
+            );
+        }
+    }
+}
diff --git a/eth2/utils/bls/src/lib.rs b/eth2/utils/bls/src/lib.rs
index b9a4d5c1d..fae41aeed 100644
--- a/eth2/utils/bls/src/lib.rs
+++ b/eth2/utils/bls/src/lib.rs
@@ -6,22 +6,22 @@ mod keypair;
 mod public_key;
 mod secret_key;
 
-#[cfg(not(debug_assertions))]
+#[cfg(not(feature = "fake_crypto"))]
 mod aggregate_signature;
-#[cfg(not(debug_assertions))]
+#[cfg(not(feature = "fake_crypto"))]
 mod signature;
-#[cfg(not(debug_assertions))]
+#[cfg(not(feature = "fake_crypto"))]
 pub use crate::aggregate_signature::AggregateSignature;
-#[cfg(not(debug_assertions))]
+#[cfg(not(feature = "fake_crypto"))]
 pub use crate::signature::Signature;
 
-#[cfg(debug_assertions)]
+#[cfg(feature = "fake_crypto")]
 mod fake_aggregate_signature;
-#[cfg(debug_assertions)]
+#[cfg(feature = "fake_crypto")]
 mod fake_signature;
-#[cfg(debug_assertions)]
+#[cfg(feature = "fake_crypto")]
 pub use crate::fake_aggregate_signature::FakeAggregateSignature as AggregateSignature;
-#[cfg(debug_assertions)]
+#[cfg(feature = "fake_crypto")]
 pub use crate::fake_signature::FakeSignature as Signature;
 
 pub use crate::aggregate_public_key::AggregatePublicKey;

From b21cc64949e5abeb6f9d9805daa4e5449477367e Mon Sep 17 00:00:00 2001
From: Michael Sproul <michael@sigmaprime.io>
Date: Tue, 2 Apr 2019 18:46:08 +1100
Subject: [PATCH 068/137] state transition tests: check expected state

---
 eth2/state_processing/tests/tests.rs | 52 ++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/eth2/state_processing/tests/tests.rs b/eth2/state_processing/tests/tests.rs
index 6ea8863b8..193511852 100644
--- a/eth2/state_processing/tests/tests.rs
+++ b/eth2/state_processing/tests/tests.rs
@@ -10,6 +10,45 @@ use types::*;
 #[allow(unused_imports)]
 use yaml_utils;
 
+#[derive(Debug, Deserialize)]
+pub struct ExpectedState {
+    pub slot: Option<Slot>,
+    pub genesis_time: Option<u64>,
+    pub fork: Option<Fork>,
+    pub validator_registry: Option<Vec<Validator>>,
+    pub validator_balances: Option<Vec<u64>>,
+}
+
+impl ExpectedState {
+    // Return a list of fields that differ, and a string representation of the beacon state's field.
+    fn check(&self, state: &BeaconState) -> Vec<(&str, String)> {
+        // Check field equality
+        macro_rules! cfe {
+            ($field_name:ident) => {
+                if self.$field_name.as_ref().map_or(true, |$field_name| {
+                    println!("  > Checking {}", stringify!($field_name));
+                    $field_name == &state.$field_name
+                }) {
+                    vec![]
+                } else {
+                    vec![(stringify!($field_name), format!("{:#?}", state.$field_name))]
+                }
+            };
+        }
+
+        vec![
+            cfe!(slot),
+            cfe!(genesis_time),
+            cfe!(fork),
+            cfe!(validator_registry),
+            cfe!(validator_balances),
+        ]
+        .into_iter()
+        .flat_map(|x| x)
+        .collect()
+    }
+}
+
 #[derive(Debug, Deserialize)]
 pub struct TestCase {
     pub name: String,
@@ -17,6 +56,7 @@ pub struct TestCase {
     pub verify_signatures: bool,
     pub initial_state: BeaconState,
     pub blocks: Vec<BeaconBlock>,
+    pub expected_state: ExpectedState,
 }
 
 #[derive(Debug, Deserialize)]
@@ -70,6 +110,18 @@ fn run_state_transition_test(test_name: &str) {
                 ok = false;
             };
         }
+
+        let mismatched_fields = test_case.expected_state.check(&state);
+        if !mismatched_fields.is_empty() {
+            println!(
+                "Error in expected state, these fields didn't match: {:?}",
+                mismatched_fields.iter().map(|(f, _)| f).collect::<Vec<_>>()
+            );
+            for (field_name, state_val) in mismatched_fields {
+                println!("state.{} was: {}", field_name, state_val);
+            }
+            ok = false;
+        }
     }
 
     assert!(ok, "one or more tests failed, see above");

From 19fad1012f414de90f261bbfcc4beb7805af3b92 Mon Sep 17 00:00:00 2001
From: Michael Sproul <michael@sigmaprime.io>
Date: Wed, 3 Apr 2019 17:14:11 +1100
Subject: [PATCH 069/137] state transitions tests: check more fields

---
 eth2/state_processing/tests/tests.rs          | 19 ++++++++++++-------
 .../yaml_utils/expected_state_fields.py       | 15 +++++++++++++++
 eth2/types/src/beacon_state.rs                |  2 +-
 3 files changed, 28 insertions(+), 8 deletions(-)
 create mode 100755 eth2/state_processing/yaml_utils/expected_state_fields.py

diff --git a/eth2/state_processing/tests/tests.rs b/eth2/state_processing/tests/tests.rs
index 193511852..54fd6bf8d 100644
--- a/eth2/state_processing/tests/tests.rs
+++ b/eth2/state_processing/tests/tests.rs
@@ -1,14 +1,9 @@
 use serde_derive::Deserialize;
 use serde_yaml;
 #[cfg(not(debug_assertions))]
-use state_processing::{
-    per_block_processing, per_block_processing_without_verifying_block_signature,
-    per_slot_processing,
-};
+use state_processing::{per_block_processing, per_slot_processing};
 use std::{fs::File, io::prelude::*, path::PathBuf};
 use types::*;
-#[allow(unused_imports)]
-use yaml_utils;
 
 #[derive(Debug, Deserialize)]
 pub struct ExpectedState {
@@ -17,6 +12,11 @@ pub struct ExpectedState {
     pub fork: Option<Fork>,
     pub validator_registry: Option<Vec<Validator>>,
     pub validator_balances: Option<Vec<u64>>,
+    pub previous_epoch_attestations: Option<Vec<PendingAttestation>>,
+    pub current_epoch_attestations: Option<Vec<PendingAttestation>>,
+    pub historical_roots: Option<Vec<Hash256>>,
+    pub finalized_epoch: Option<Epoch>,
+    pub latest_block_roots: Option<Vec<Hash256>>,
 }
 
 impl ExpectedState {
@@ -42,6 +42,11 @@ impl ExpectedState {
             cfe!(fork),
             cfe!(validator_registry),
             cfe!(validator_balances),
+            cfe!(previous_epoch_attestations),
+            cfe!(current_epoch_attestations),
+            cfe!(historical_roots),
+            cfe!(finalized_epoch),
+            cfe!(latest_block_roots),
         ]
         .into_iter()
         .flat_map(|x| x)
@@ -108,7 +113,7 @@ fn run_state_transition_test(test_name: &str) {
                 println!("Error in {} (#{}), on block {}", test_case.name, i, j);
                 println!("{:?}", res);
                 ok = false;
-            };
+            }
         }
 
         let mismatched_fields = test_case.expected_state.check(&state);
diff --git a/eth2/state_processing/yaml_utils/expected_state_fields.py b/eth2/state_processing/yaml_utils/expected_state_fields.py
new file mode 100755
index 000000000..df4cb83f7
--- /dev/null
+++ b/eth2/state_processing/yaml_utils/expected_state_fields.py
@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+
+# Script to extract all the fields of the state mentioned in `expected_state` fields of tests
+# in the `spec` directory. These fields can then be added to the `ExpectedState` struct.
+# Might take a while to run.
+
+import os, yaml
+
+if __name__ == "__main__":
+    yaml_files = (filename for filename in os.listdir("specs") if filename.endswith(".yaml"))
+    parsed_yaml = (yaml.load(open("specs/" + filename, "r")) for filename in yaml_files)
+    all_fields = set()
+    for y in parsed_yaml:
+        all_fields.update(*({key for key in case["expected_state"]} for case in y["test_cases"]))
+    print(all_fields)
diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs
index c068c4e03..0461e947b 100644
--- a/eth2/types/src/beacon_state.rs
+++ b/eth2/types/src/beacon_state.rs
@@ -81,7 +81,7 @@ pub struct BeaconState {
 
     // Recent state
     pub latest_crosslinks: TreeHashVector<Crosslink>,
-    latest_block_roots: TreeHashVector<Hash256>,
+    pub latest_block_roots: TreeHashVector<Hash256>,
     latest_state_roots: TreeHashVector<Hash256>,
     latest_active_index_roots: TreeHashVector<Hash256>,
     latest_slashed_balances: TreeHashVector<u64>,

From b801303374bd49f0985a7d63eb965ffc26e6c57d Mon Sep 17 00:00:00 2001
From: Michael Sproul <michael@sigmaprime.io>
Date: Wed, 3 Apr 2019 17:15:07 +1100
Subject: [PATCH 070/137] spec: fix shuffle direction in
 get_crosslink_committees_at_slot

---
 eth2/types/src/beacon_state/epoch_cache.rs         | 2 +-
 eth2/types/src/beacon_state/epoch_cache/tests.rs   | 2 +-
 eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs | 2 ++
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/eth2/types/src/beacon_state/epoch_cache.rs b/eth2/types/src/beacon_state/epoch_cache.rs
index 62df90271..dd9ae3403 100644
--- a/eth2/types/src/beacon_state/epoch_cache.rs
+++ b/eth2/types/src/beacon_state/epoch_cache.rs
@@ -288,7 +288,7 @@ impl EpochCrosslinkCommitteesBuilder {
                 self.active_validator_indices,
                 spec.shuffle_round_count,
                 &self.shuffling_seed[..],
-                true,
+                false,
             )
             .ok_or_else(|| Error::UnableToShuffle)?
         };
diff --git a/eth2/types/src/beacon_state/epoch_cache/tests.rs b/eth2/types/src/beacon_state/epoch_cache/tests.rs
index 5643776e2..5b1e53338 100644
--- a/eth2/types/src/beacon_state/epoch_cache/tests.rs
+++ b/eth2/types/src/beacon_state/epoch_cache/tests.rs
@@ -27,7 +27,7 @@ fn do_sane_cache_test(
         active_indices,
         spec.shuffle_round_count,
         &expected_seed[..],
-        true,
+        false,
     )
     .unwrap();
 
diff --git a/eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs b/eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs
index e7e1e18e6..f60d793f2 100644
--- a/eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs
+++ b/eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs
@@ -18,6 +18,8 @@ const TOTAL_SIZE: usize = SEED_SIZE + ROUND_SIZE + POSITION_WINDOW_SIZE;
 /// Credits to [@protolambda](https://github.com/protolambda) for defining this algorithm.
 ///
 /// Shuffles if `forwards == true`, otherwise un-shuffles.
+/// It holds that: shuffle_list(shuffle_list(l, r, s, true), r, s, false) == l
+///           and: shuffle_list(shuffle_list(l, r, s, false), r, s, true) == l
 ///
 /// Returns `None` under any of the following conditions:
 ///  - `list_size == 0`

From 32547373e528e8ba584f349f99f7523e8d14a175 Mon Sep 17 00:00:00 2001
From: Michael Sproul <michael@sigmaprime.io>
Date: Mon, 8 Apr 2019 14:37:01 +1000
Subject: [PATCH 071/137] spec: simplify `cache_state`

The `latest_block_root` input argument was unnecessary as we were always setting it to something
almost equivalent to `state.latest_block_root` anyway, and more importantly, it was messing up the
caching of the state root. Previously it was possible for the function to update the state's latest
block root, and then hash the outdated block root that was passed in as an argument.
---
 beacon_node/beacon_chain/src/beacon_chain.rs     | 11 +++--------
 eth2/state_processing/src/per_slot_processing.rs | 16 ++++------------
 eth2/state_processing/tests/tests.rs             |  3 +--
 3 files changed, 8 insertions(+), 22 deletions(-)

diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs
index a22f4179e..41a718655 100644
--- a/beacon_node/beacon_chain/src/beacon_chain.rs
+++ b/beacon_node/beacon_chain/src/beacon_chain.rs
@@ -303,8 +303,6 @@ where
     /// then having it iteratively updated -- in such a case it's possible for another thread to
     /// find the state at an old slot.
     pub fn update_state(&self, mut state: BeaconState) -> Result<(), Error> {
-        let latest_block_header = self.head().beacon_block.block_header();
-
         let present_slot = match self.slot_clock.present_slot() {
             Ok(Some(slot)) => slot,
             _ => return Err(Error::UnableToReadSlot),
@@ -312,7 +310,7 @@ where
 
         // If required, transition the new state to the present slot.
         for _ in state.slot.as_u64()..present_slot.as_u64() {
-            per_slot_processing(&mut state, &latest_block_header, &self.spec)?;
+            per_slot_processing(&mut state, &self.spec)?;
         }
 
         state.build_all_caches(&self.spec)?;
@@ -324,8 +322,6 @@ where
 
     /// Ensures the current canonical `BeaconState` has been transitioned to match the `slot_clock`.
     pub fn catchup_state(&self) -> Result<(), Error> {
-        let latest_block_header = self.head().beacon_block.block_header();
-
         let present_slot = match self.slot_clock.present_slot() {
             Ok(Some(slot)) => slot,
             _ => return Err(Error::UnableToReadSlot),
@@ -339,7 +335,7 @@ where
             state.build_epoch_cache(RelativeEpoch::NextWithoutRegistryChange, &self.spec)?;
             state.build_epoch_cache(RelativeEpoch::NextWithRegistryChange, &self.spec)?;
 
-            per_slot_processing(&mut *state, &latest_block_header, &self.spec)?;
+            per_slot_processing(&mut *state, &self.spec)?;
         }
 
         state.build_all_caches(&self.spec)?;
@@ -617,9 +613,8 @@ where
 
         // Transition the parent state to the block slot.
         let mut state = parent_state;
-        let previous_block_header = parent_block.block_header();
         for _ in state.slot.as_u64()..block.slot.as_u64() {
-            if let Err(e) = per_slot_processing(&mut state, &previous_block_header, &self.spec) {
+            if let Err(e) = per_slot_processing(&mut state, &self.spec) {
                 return Ok(BlockProcessingOutcome::InvalidBlock(
                     InvalidBlock::SlotProcessingError(e),
                 ));
diff --git a/eth2/state_processing/src/per_slot_processing.rs b/eth2/state_processing/src/per_slot_processing.rs
index cd129a5f1..7d2bb468f 100644
--- a/eth2/state_processing/src/per_slot_processing.rs
+++ b/eth2/state_processing/src/per_slot_processing.rs
@@ -11,12 +11,8 @@ pub enum Error {
 /// Advances a state forward by one slot, performing per-epoch processing if required.
 ///
 /// Spec v0.5.0
-pub fn per_slot_processing(
-    state: &mut BeaconState,
-    latest_block_header: &BeaconBlockHeader,
-    spec: &ChainSpec,
-) -> Result<(), Error> {
-    cache_state(state, latest_block_header, spec)?;
+pub fn per_slot_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> {
+    cache_state(state, spec)?;
 
     if (state.slot + 1) % spec.slots_per_epoch == 0 {
         per_epoch_processing(state, spec)?;
@@ -27,11 +23,7 @@ pub fn per_slot_processing(
     Ok(())
 }
 
-fn cache_state(
-    state: &mut BeaconState,
-    latest_block_header: &BeaconBlockHeader,
-    spec: &ChainSpec,
-) -> Result<(), Error> {
+fn cache_state(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> {
     let previous_slot_state_root = Hash256::from_slice(&state.tree_hash_root()[..]);
 
     // Note: increment the state slot here to allow use of our `state_root` and `block_root`
@@ -46,7 +38,7 @@ fn cache_state(
         state.latest_block_header.state_root = previous_slot_state_root
     }
 
-    let latest_block_root = Hash256::from_slice(&latest_block_header.tree_hash_root()[..]);
+    let latest_block_root = Hash256::from_slice(&state.latest_block_header.tree_hash_root()[..]);
     state.set_block_root(previous_slot, latest_block_root, spec)?;
 
     // Set the state slot back to what it should be.
diff --git a/eth2/state_processing/tests/tests.rs b/eth2/state_processing/tests/tests.rs
index 54fd6bf8d..d305b2d3c 100644
--- a/eth2/state_processing/tests/tests.rs
+++ b/eth2/state_processing/tests/tests.rs
@@ -105,8 +105,7 @@ fn run_state_transition_test(test_name: &str) {
         let mut state = test_case.initial_state.clone();
         for (j, block) in test_case.blocks.iter().enumerate() {
             while block.slot > state.slot {
-                let latest_block_header = state.latest_block_header.clone();
-                per_slot_processing(&mut state, &latest_block_header, &test_case.config).unwrap();
+                per_slot_processing(&mut state, &test_case.config).unwrap();
             }
             let res = per_block_processing(&mut state, &block, &test_case.config);
             if res.is_err() {

From a19f8580f51aca0d1e608342b7a89f94c35dda02 Mon Sep 17 00:00:00 2001
From: Michael Sproul <michael@sigmaprime.io>
Date: Mon, 15 Apr 2019 09:33:54 +1000
Subject: [PATCH 072/137] travis: state transition tests

---
 .travis.yml                          | 1 +
 eth2/state_processing/tests/tests.rs | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index e725aa0ba..6233ea68b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,6 +11,7 @@ script:
   - cargo build --verbose --release --all
   - cargo test --verbose --all
   - cargo test --verbose --release --all
+  - cargo test --manifest-path eth2/state_processing/Cargo.toml --verbose --release --features fake_crypto
   - cargo fmt --all -- --check
   # No clippy until later...
   #- cargo clippy
diff --git a/eth2/state_processing/tests/tests.rs b/eth2/state_processing/tests/tests.rs
index d305b2d3c..cdad99062 100644
--- a/eth2/state_processing/tests/tests.rs
+++ b/eth2/state_processing/tests/tests.rs
@@ -144,8 +144,9 @@ fn run_state_transition_tests_small() {
     run_state_transition_test("sanity-check_small-config_32-vals.yaml");
 }
 
+// Run with --ignored to run this test
 #[test]
-#[cfg(not(debug_assertions))]
+#[ignored]
 fn run_state_transition_tests_large() {
     run_state_transition_test("sanity-check_default-config_100-vals.yaml");
 }

From 4f63c89bb649209b9228512533eaf0365cee544a Mon Sep 17 00:00:00 2001
From: Michael Sproul <michael@sigmaprime.io>
Date: Mon, 15 Apr 2019 09:38:04 +1000
Subject: [PATCH 073/137] jenkins: run all state tests

---
 Jenkinsfile                          | 3 +++
 eth2/state_processing/tests/tests.rs | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index d12189941..48a07e1e7 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -23,6 +23,9 @@ pipeline {
 			steps {
 				sh 'cargo test --verbose --all'
 				sh 'cargo test --verbose --all --release'
+                sh 'cargo test --manifest-path eth2/state_processing/Cargo.toml --verbose \
+                               --release --features fake_crypto --ignored'
+
 			}
 		}
 	}
diff --git a/eth2/state_processing/tests/tests.rs b/eth2/state_processing/tests/tests.rs
index cdad99062..6491e255a 100644
--- a/eth2/state_processing/tests/tests.rs
+++ b/eth2/state_processing/tests/tests.rs
@@ -146,7 +146,7 @@ fn run_state_transition_tests_small() {
 
 // Run with --ignored to run this test
 #[test]
-#[ignored]
+#[ignore]
 fn run_state_transition_tests_large() {
     run_state_transition_test("sanity-check_default-config_100-vals.yaml");
 }

From 2914d77cd38d19abc88249955cb8b0c1ee3392b2 Mon Sep 17 00:00:00 2001
From: Michael Sproul <michael@sigmaprime.io>
Date: Tue, 2 Apr 2019 14:18:07 +1100
Subject: [PATCH 074/137] spec: update to v0.5.1

---
 eth2/state_processing/src/per_block_processing.rs | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/eth2/state_processing/src/per_block_processing.rs b/eth2/state_processing/src/per_block_processing.rs
index 5afddc74e..6eafcb937 100644
--- a/eth2/state_processing/src/per_block_processing.rs
+++ b/eth2/state_processing/src/per_block_processing.rs
@@ -99,7 +99,7 @@ fn per_block_processing_signature_optional(
 
 /// Processes the block header.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn process_block_header(
     state: &mut BeaconState,
     block: &BeaconBlock,
@@ -107,11 +107,8 @@ pub fn process_block_header(
 ) -> Result<(), Error> {
     verify!(block.slot == state.slot, Invalid::StateSlotMismatch);
 
-    // NOTE: this is not to spec. I think spec is broken. See:
-    //
-    // https://github.com/ethereum/eth2.0-specs/issues/797
     verify!(
-        block.previous_block_root == *state.get_block_root(state.slot - 1, spec)?,
+        block.previous_block_root == Hash256::from_slice(&state.latest_block_header.signed_root()),
         Invalid::ParentBlockRootMismatch
     );
 

From d95ae95ce8a882b8a33f676262f5549765be9bf3 Mon Sep 17 00:00:00 2001
From: Michael Sproul <michael@sigmaprime.io>
Date: Mon, 15 Apr 2019 10:51:20 +1000
Subject: [PATCH 075/137] spec: update tags to v0.5.1

---
 .../src/common/exit_validator.rs              |  2 +-
 .../src/common/slash_validator.rs             |  2 +-
 .../src/common/verify_bitfield.rs             |  2 +-
 .../state_processing/src/get_genesis_state.rs |  2 +-
 .../src/per_block_processing.rs               | 24 +++----
 .../validate_attestation.rs                   | 10 +--
 .../verify_attester_slashing.rs               |  4 +-
 .../per_block_processing/verify_deposit.rs    | 10 +--
 .../src/per_block_processing/verify_exit.rs   |  2 +-
 .../verify_proposer_slashing.rs               |  4 +-
 .../verify_slashable_attestation.rs           |  2 +-
 .../per_block_processing/verify_transfer.rs   |  4 +-
 .../src/per_epoch_processing.rs               | 10 +--
 .../src/per_epoch_processing/apply_rewards.rs | 18 ++---
 .../get_attestation_participants.rs           |  2 +-
 .../inclusion_distance.rs                     |  6 +-
 .../per_epoch_processing/process_ejections.rs |  2 +-
 .../process_exit_queue.rs                     |  4 +-
 .../per_epoch_processing/process_slashings.rs |  2 +-
 .../update_registry_and_shuffling_data.rs     |  8 +--
 .../validator_statuses.rs                     | 14 ++--
 .../src/per_epoch_processing/winning_root.rs  |  8 +--
 .../src/per_slot_processing.rs                |  2 +-
 eth2/types/src/attestation.rs                 |  2 +-
 eth2/types/src/attestation_data.rs            |  2 +-
 .../src/attestation_data_and_custody_bit.rs   |  2 +-
 eth2/types/src/attester_slashing.rs           |  2 +-
 eth2/types/src/beacon_block.rs                | 10 +--
 eth2/types/src/beacon_block_body.rs           |  2 +-
 eth2/types/src/beacon_block_header.rs         |  6 +-
 eth2/types/src/beacon_state.rs                | 68 +++++++++----------
 eth2/types/src/beacon_state/epoch_cache.rs    |  2 +-
 eth2/types/src/chain_spec.rs                  | 10 +--
 eth2/types/src/crosslink.rs                   |  2 +-
 eth2/types/src/deposit.rs                     |  2 +-
 eth2/types/src/deposit_data.rs                |  2 +-
 eth2/types/src/deposit_input.rs               |  6 +-
 eth2/types/src/eth1_data.rs                   |  2 +-
 eth2/types/src/eth1_data_vote.rs              |  2 +-
 eth2/types/src/fork.rs                        |  6 +-
 eth2/types/src/historical_batch.rs            |  2 +-
 eth2/types/src/pending_attestation.rs         |  2 +-
 eth2/types/src/proposer_slashing.rs           |  2 +-
 eth2/types/src/relative_epoch.rs              |  6 +-
 eth2/types/src/slashable_attestation.rs       |  6 +-
 eth2/types/src/transfer.rs                    |  2 +-
 eth2/types/src/validator.rs                   |  2 +-
 eth2/types/src/voluntary_exit.rs              |  2 +-
 48 files changed, 148 insertions(+), 148 deletions(-)

diff --git a/eth2/state_processing/src/common/exit_validator.rs b/eth2/state_processing/src/common/exit_validator.rs
index 8ab530b18..a6cfb395e 100644
--- a/eth2/state_processing/src/common/exit_validator.rs
+++ b/eth2/state_processing/src/common/exit_validator.rs
@@ -2,7 +2,7 @@ use types::{BeaconStateError as Error, *};
 
 /// Exit the validator of the given `index`.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn exit_validator(
     state: &mut BeaconState,
     validator_index: usize,
diff --git a/eth2/state_processing/src/common/slash_validator.rs b/eth2/state_processing/src/common/slash_validator.rs
index 9be87b978..c1aad7da1 100644
--- a/eth2/state_processing/src/common/slash_validator.rs
+++ b/eth2/state_processing/src/common/slash_validator.rs
@@ -3,7 +3,7 @@ use types::{BeaconStateError as Error, *};
 
 /// Slash the validator with index ``index``.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn slash_validator(
     state: &mut BeaconState,
     validator_index: usize,
diff --git a/eth2/state_processing/src/common/verify_bitfield.rs b/eth2/state_processing/src/common/verify_bitfield.rs
index 71c9f9c3e..7b3c07086 100644
--- a/eth2/state_processing/src/common/verify_bitfield.rs
+++ b/eth2/state_processing/src/common/verify_bitfield.rs
@@ -4,7 +4,7 @@ use types::*;
 ///
 /// Is title `verify_bitfield` in spec.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn verify_bitfield_length(bitfield: &Bitfield, committee_size: usize) -> bool {
     if bitfield.num_bytes() != ((committee_size + 7) / 8) {
         return false;
diff --git a/eth2/state_processing/src/get_genesis_state.rs b/eth2/state_processing/src/get_genesis_state.rs
index 2bde8ce0c..4e9fb6caf 100644
--- a/eth2/state_processing/src/get_genesis_state.rs
+++ b/eth2/state_processing/src/get_genesis_state.rs
@@ -9,7 +9,7 @@ pub enum GenesisError {
 
 /// Returns the genesis `BeaconState`
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn get_genesis_state(
     genesis_validator_deposits: &[Deposit],
     genesis_time: u64,
diff --git a/eth2/state_processing/src/per_block_processing.rs b/eth2/state_processing/src/per_block_processing.rs
index 6eafcb937..257d92acf 100644
--- a/eth2/state_processing/src/per_block_processing.rs
+++ b/eth2/state_processing/src/per_block_processing.rs
@@ -39,7 +39,7 @@ const VERIFY_DEPOSIT_MERKLE_PROOFS: bool = false;
 /// Returns `Ok(())` if the block is valid and the state was successfully updated. Otherwise
 /// returns an error describing why the block was invalid or how the function failed to execute.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn per_block_processing(
     state: &mut BeaconState,
     block: &BeaconBlock,
@@ -54,7 +54,7 @@ pub fn per_block_processing(
 /// Returns `Ok(())` if the block is valid and the state was successfully updated. Otherwise
 /// returns an error describing why the block was invalid or how the function failed to execute.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn per_block_processing_without_verifying_block_signature(
     state: &mut BeaconState,
     block: &BeaconBlock,
@@ -69,7 +69,7 @@ pub fn per_block_processing_without_verifying_block_signature(
 /// Returns `Ok(())` if the block is valid and the state was successfully updated. Otherwise
 /// returns an error describing why the block was invalid or how the function failed to execute.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn per_block_processing_signature_optional(
     mut state: &mut BeaconState,
     block: &BeaconBlock,
@@ -119,7 +119,7 @@ pub fn process_block_header(
 
 /// Verifies the signature of a block.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn verify_block_signature(
     state: &BeaconState,
     block: &BeaconBlock,
@@ -147,7 +147,7 @@ pub fn verify_block_signature(
 /// Verifies the `randao_reveal` against the block's proposer pubkey and updates
 /// `state.latest_randao_mixes`.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn process_randao(
     state: &mut BeaconState,
     block: &BeaconBlock,
@@ -178,7 +178,7 @@ pub fn process_randao(
 
 /// Update the `state.eth1_data_votes` based upon the `eth1_data` provided.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn process_eth1_data(state: &mut BeaconState, eth1_data: &Eth1Data) -> Result<(), Error> {
     // Attempt to find a `Eth1DataVote` with matching `Eth1Data`.
     let matching_eth1_vote_index = state
@@ -204,7 +204,7 @@ pub fn process_eth1_data(state: &mut BeaconState, eth1_data: &Eth1Data) -> Resul
 /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns
 /// an `Err` describing the invalid object or cause of failure.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn process_proposer_slashings(
     state: &mut BeaconState,
     proposer_slashings: &[ProposerSlashing],
@@ -237,7 +237,7 @@ pub fn process_proposer_slashings(
 /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns
 /// an `Err` describing the invalid object or cause of failure.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn process_attester_slashings(
     state: &mut BeaconState,
     attester_slashings: &[AttesterSlashing],
@@ -295,7 +295,7 @@ pub fn process_attester_slashings(
 /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns
 /// an `Err` describing the invalid object or cause of failure.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn process_attestations(
     state: &mut BeaconState,
     attestations: &[Attestation],
@@ -337,7 +337,7 @@ pub fn process_attestations(
 /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns
 /// an `Err` describing the invalid object or cause of failure.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn process_deposits(
     state: &mut BeaconState,
     deposits: &[Deposit],
@@ -407,7 +407,7 @@ pub fn process_deposits(
 /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns
 /// an `Err` describing the invalid object or cause of failure.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn process_exits(
     state: &mut BeaconState,
     voluntary_exits: &[VoluntaryExit],
@@ -439,7 +439,7 @@ pub fn process_exits(
 /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns
 /// an `Err` describing the invalid object or cause of failure.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn process_transfers(
     state: &mut BeaconState,
     transfers: &[Transfer],
diff --git a/eth2/state_processing/src/per_block_processing/validate_attestation.rs b/eth2/state_processing/src/per_block_processing/validate_attestation.rs
index c9d0b38a4..438a75c94 100644
--- a/eth2/state_processing/src/per_block_processing/validate_attestation.rs
+++ b/eth2/state_processing/src/per_block_processing/validate_attestation.rs
@@ -8,7 +8,7 @@ use types::*;
 ///
 /// Returns `Ok(())` if the `Attestation` is valid, otherwise indicates the reason for invalidity.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn validate_attestation(
     state: &BeaconState,
     attestation: &Attestation,
@@ -31,7 +31,7 @@ pub fn validate_attestation_time_independent_only(
 ///
 /// Returns `Ok(())` if the `Attestation` is valid, otherwise indicates the reason for invalidity.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn validate_attestation_without_signature(
     state: &BeaconState,
     attestation: &Attestation,
@@ -44,7 +44,7 @@ pub fn validate_attestation_without_signature(
 /// given state, optionally validating the aggregate signature.
 ///
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn validate_attestation_parametric(
     state: &BeaconState,
     attestation: &Attestation,
@@ -167,7 +167,7 @@ fn validate_attestation_parametric(
 /// Verify that the `source_epoch` and `source_root` of an `Attestation` correctly
 /// match the current (or previous) justified epoch and root from the state.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn verify_justified_epoch_and_root(
     attestation: &Attestation,
     state: &BeaconState,
@@ -222,7 +222,7 @@ fn verify_justified_epoch_and_root(
 ///  - `custody_bitfield` does not have a bit for each index of `committee`.
 ///  - A `validator_index` in `committee` is not in `state.validator_registry`.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn verify_attestation_signature(
     state: &BeaconState,
     committee: &[usize],
diff --git a/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs b/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs
index abf99da64..3527b62e3 100644
--- a/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs
+++ b/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs
@@ -7,7 +7,7 @@ use types::*;
 ///
 /// Returns `Ok(())` if the `AttesterSlashing` is valid, otherwise indicates the reason for invalidity.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn verify_attester_slashing(
     state: &BeaconState,
     attester_slashing: &AttesterSlashing,
@@ -41,7 +41,7 @@ pub fn verify_attester_slashing(
 ///
 /// Returns Ok(indices) if `indices.len() > 0`.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn gather_attester_slashing_indices(
     state: &BeaconState,
     attester_slashing: &AttesterSlashing,
diff --git a/eth2/state_processing/src/per_block_processing/verify_deposit.rs b/eth2/state_processing/src/per_block_processing/verify_deposit.rs
index a3a0f5734..22a62a321 100644
--- a/eth2/state_processing/src/per_block_processing/verify_deposit.rs
+++ b/eth2/state_processing/src/per_block_processing/verify_deposit.rs
@@ -15,7 +15,7 @@ use types::*;
 ///
 /// Note: this function is incomplete.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn verify_deposit(
     state: &BeaconState,
     deposit: &Deposit,
@@ -46,7 +46,7 @@ pub fn verify_deposit(
 
 /// Verify that the `Deposit` index is correct.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn verify_deposit_index(state: &BeaconState, deposit: &Deposit) -> Result<(), Error> {
     verify!(
         deposit.index == state.deposit_index,
@@ -88,7 +88,7 @@ pub fn get_existing_validator_index(
 
 /// Verify that a deposit is included in the state's eth1 deposit root.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn verify_deposit_merkle_proof(state: &BeaconState, deposit: &Deposit, spec: &ChainSpec) -> bool {
     let leaf = hash(&get_serialized_deposit_data(deposit));
     verify_merkle_proof(
@@ -102,7 +102,7 @@ fn verify_deposit_merkle_proof(state: &BeaconState, deposit: &Deposit, spec: &Ch
 
 /// Helper struct for easily getting the serialized data generated by the deposit contract.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(Encode)]
 struct SerializedDepositData {
     amount: u64,
@@ -113,7 +113,7 @@ struct SerializedDepositData {
 /// Return the serialized data generated by the deposit contract that is used to generate the
 /// merkle proof.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn get_serialized_deposit_data(deposit: &Deposit) -> Vec<u8> {
     let serialized_deposit_data = SerializedDepositData {
         amount: deposit.deposit_data.amount,
diff --git a/eth2/state_processing/src/per_block_processing/verify_exit.rs b/eth2/state_processing/src/per_block_processing/verify_exit.rs
index c5b8ebcb4..697188ee9 100644
--- a/eth2/state_processing/src/per_block_processing/verify_exit.rs
+++ b/eth2/state_processing/src/per_block_processing/verify_exit.rs
@@ -7,7 +7,7 @@ use types::*;
 ///
 /// Returns `Ok(())` if the `Exit` is valid, otherwise indicates the reason for invalidity.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn verify_exit(
     state: &BeaconState,
     exit: &VoluntaryExit,
diff --git a/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs b/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs
index b5113863e..bbc03dd62 100644
--- a/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs
+++ b/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs
@@ -7,7 +7,7 @@ use types::*;
 ///
 /// Returns `Ok(())` if the `ProposerSlashing` is valid, otherwise indicates the reason for invalidity.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn verify_proposer_slashing(
     proposer_slashing: &ProposerSlashing,
     state: &BeaconState,
@@ -67,7 +67,7 @@ pub fn verify_proposer_slashing(
 ///
 /// Returns `true` if the signature is valid.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn verify_header_signature(
     header: &BeaconBlockHeader,
     pubkey: &PublicKey,
diff --git a/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs b/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs
index d39ac6759..89cb93ce5 100644
--- a/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs
+++ b/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs
@@ -10,7 +10,7 @@ use types::*;
 ///
 /// Returns `Ok(())` if the `SlashableAttestation` is valid, otherwise indicates the reason for invalidity.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn verify_slashable_attestation(
     state: &BeaconState,
     slashable_attestation: &SlashableAttestation,
diff --git a/eth2/state_processing/src/per_block_processing/verify_transfer.rs b/eth2/state_processing/src/per_block_processing/verify_transfer.rs
index 978d0cfce..8b0415508 100644
--- a/eth2/state_processing/src/per_block_processing/verify_transfer.rs
+++ b/eth2/state_processing/src/per_block_processing/verify_transfer.rs
@@ -10,7 +10,7 @@ use types::*;
 ///
 /// Note: this function is incomplete.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn verify_transfer(
     state: &BeaconState,
     transfer: &Transfer,
@@ -122,7 +122,7 @@ fn verify_transfer_parametric(
 ///
 /// Does not check that the transfer is valid, however checks for overflow in all actions.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn execute_transfer(
     state: &mut BeaconState,
     transfer: &Transfer,
diff --git a/eth2/state_processing/src/per_epoch_processing.rs b/eth2/state_processing/src/per_epoch_processing.rs
index f4d6452a4..87c9b9398 100644
--- a/eth2/state_processing/src/per_epoch_processing.rs
+++ b/eth2/state_processing/src/per_epoch_processing.rs
@@ -32,7 +32,7 @@ pub type WinningRootHashSet = HashMap<u64, WinningRoot>;
 /// Mutates the given `BeaconState`, returning early if an error is encountered. If an error is
 /// returned, a state might be "half-processed" and therefore in an invalid state.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn per_epoch_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> {
     // Ensure the previous and next epoch caches are built.
     state.build_epoch_cache(RelativeEpoch::Previous, spec)?;
@@ -86,7 +86,7 @@ pub fn per_epoch_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result
 
 /// Maybe resets the eth1 period.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn maybe_reset_eth1_period(state: &mut BeaconState, spec: &ChainSpec) {
     let next_epoch = state.next_epoch(spec);
     let voting_period = spec.epochs_per_eth1_voting_period;
@@ -108,7 +108,7 @@ pub fn maybe_reset_eth1_period(state: &mut BeaconState, spec: &ChainSpec) {
 /// - `justified_epoch`
 /// - `previous_justified_epoch`
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn update_justification_and_finalization(
     state: &mut BeaconState,
     total_balances: &TotalBalances,
@@ -178,7 +178,7 @@ pub fn update_justification_and_finalization(
 ///
 /// Also returns a `WinningRootHashSet` for later use during epoch processing.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn process_crosslinks(
     state: &mut BeaconState,
     spec: &ChainSpec,
@@ -221,7 +221,7 @@ pub fn process_crosslinks(
 
 /// Finish up an epoch update.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn finish_epoch_update(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> {
     let current_epoch = state.current_epoch(spec);
     let next_epoch = state.next_epoch(spec);
diff --git a/eth2/state_processing/src/per_epoch_processing/apply_rewards.rs b/eth2/state_processing/src/per_epoch_processing/apply_rewards.rs
index ce5fccb21..9af1ee8c3 100644
--- a/eth2/state_processing/src/per_epoch_processing/apply_rewards.rs
+++ b/eth2/state_processing/src/per_epoch_processing/apply_rewards.rs
@@ -32,7 +32,7 @@ impl std::ops::AddAssign for Delta {
 
 /// Apply attester and proposer rewards.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn apply_rewards(
     state: &mut BeaconState,
     validator_statuses: &mut ValidatorStatuses,
@@ -79,7 +79,7 @@ pub fn apply_rewards(
 /// Applies the attestation inclusion reward to each proposer for every validator who included an
 /// attestation in the previous epoch.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn get_proposer_deltas(
     deltas: &mut Vec<Delta>,
     state: &mut BeaconState,
@@ -120,7 +120,7 @@ fn get_proposer_deltas(
 
 /// Apply rewards for participation in attestations during the previous epoch.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn get_justification_and_finalization_deltas(
     deltas: &mut Vec<Delta>,
     state: &BeaconState,
@@ -163,7 +163,7 @@ fn get_justification_and_finalization_deltas(
 
 /// Determine the delta for a single validator, if the chain is finalizing normally.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn compute_normal_justification_and_finalization_delta(
     validator: &ValidatorStatus,
     total_balances: &TotalBalances,
@@ -215,7 +215,7 @@ fn compute_normal_justification_and_finalization_delta(
 
 /// Determine the delta for a single delta, assuming the chain is _not_ finalizing normally.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn compute_inactivity_leak_delta(
     validator: &ValidatorStatus,
     base_reward: u64,
@@ -261,7 +261,7 @@ fn compute_inactivity_leak_delta(
 
 /// Calculate the deltas based upon the winning roots for attestations during the previous epoch.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn get_crosslink_deltas(
     deltas: &mut Vec<Delta>,
     state: &BeaconState,
@@ -295,7 +295,7 @@ fn get_crosslink_deltas(
 
 /// Returns the base reward for some validator.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn get_base_reward(
     state: &BeaconState,
     index: usize,
@@ -312,7 +312,7 @@ fn get_base_reward(
 
 /// Returns the inactivity penalty for some validator.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn get_inactivity_penalty(
     state: &BeaconState,
     index: usize,
@@ -328,7 +328,7 @@ fn get_inactivity_penalty(
 
 /// Returns the epochs since the last finalized epoch.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn epochs_since_finality(state: &BeaconState, spec: &ChainSpec) -> Epoch {
     state.current_epoch(spec) + 1 - state.finalized_epoch
 }
diff --git a/eth2/state_processing/src/per_epoch_processing/get_attestation_participants.rs b/eth2/state_processing/src/per_epoch_processing/get_attestation_participants.rs
index 52ba0274b..bea772204 100644
--- a/eth2/state_processing/src/per_epoch_processing/get_attestation_participants.rs
+++ b/eth2/state_processing/src/per_epoch_processing/get_attestation_participants.rs
@@ -3,7 +3,7 @@ use types::*;
 
 /// Returns validator indices which participated in the attestation.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn get_attestation_participants(
     state: &BeaconState,
     attestation_data: &AttestationData,
diff --git a/eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs b/eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs
index b52485947..6b221f513 100644
--- a/eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs
+++ b/eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs
@@ -5,7 +5,7 @@ use types::*;
 /// Returns the distance between the first included attestation for some validator and this
 /// slot.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn inclusion_distance(
     state: &BeaconState,
     attestations: &[&PendingAttestation],
@@ -18,7 +18,7 @@ pub fn inclusion_distance(
 
 /// Returns the slot of the earliest included attestation for some validator.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn inclusion_slot(
     state: &BeaconState,
     attestations: &[&PendingAttestation],
@@ -31,7 +31,7 @@ pub fn inclusion_slot(
 
 /// Finds the earliest included attestation for some validator.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn earliest_included_attestation(
     state: &BeaconState,
     attestations: &[&PendingAttestation],
diff --git a/eth2/state_processing/src/per_epoch_processing/process_ejections.rs b/eth2/state_processing/src/per_epoch_processing/process_ejections.rs
index a60d92187..6f64c46f7 100644
--- a/eth2/state_processing/src/per_epoch_processing/process_ejections.rs
+++ b/eth2/state_processing/src/per_epoch_processing/process_ejections.rs
@@ -4,7 +4,7 @@ use types::{BeaconStateError as Error, *};
 /// Iterate through the validator registry and eject active validators with balance below
 /// ``EJECTION_BALANCE``.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn process_ejections(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> {
     // There is an awkward double (triple?) loop here because we can't loop across the borrowed
     // active validator indices and mutate state in the one loop.
diff --git a/eth2/state_processing/src/per_epoch_processing/process_exit_queue.rs b/eth2/state_processing/src/per_epoch_processing/process_exit_queue.rs
index 074db1d08..a6362188d 100644
--- a/eth2/state_processing/src/per_epoch_processing/process_exit_queue.rs
+++ b/eth2/state_processing/src/per_epoch_processing/process_exit_queue.rs
@@ -2,7 +2,7 @@ use types::*;
 
 /// Process the exit queue.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn process_exit_queue(state: &mut BeaconState, spec: &ChainSpec) {
     let current_epoch = state.current_epoch(spec);
 
@@ -31,7 +31,7 @@ pub fn process_exit_queue(state: &mut BeaconState, spec: &ChainSpec) {
 
 /// Initiate an exit for the validator of the given `index`.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn prepare_validator_for_withdrawal(
     state: &mut BeaconState,
     validator_index: usize,
diff --git a/eth2/state_processing/src/per_epoch_processing/process_slashings.rs b/eth2/state_processing/src/per_epoch_processing/process_slashings.rs
index 88777472c..89a7dd484 100644
--- a/eth2/state_processing/src/per_epoch_processing/process_slashings.rs
+++ b/eth2/state_processing/src/per_epoch_processing/process_slashings.rs
@@ -2,7 +2,7 @@ use types::{BeaconStateError as Error, *};
 
 /// Process slashings.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn process_slashings(
     state: &mut BeaconState,
     current_total_balance: u64,
diff --git a/eth2/state_processing/src/per_epoch_processing/update_registry_and_shuffling_data.rs b/eth2/state_processing/src/per_epoch_processing/update_registry_and_shuffling_data.rs
index 0b18c2571..d290d2987 100644
--- a/eth2/state_processing/src/per_epoch_processing/update_registry_and_shuffling_data.rs
+++ b/eth2/state_processing/src/per_epoch_processing/update_registry_and_shuffling_data.rs
@@ -4,7 +4,7 @@ use types::*;
 
 /// Peforms a validator registry update, if required.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn update_registry_and_shuffling_data(
     state: &mut BeaconState,
     current_total_balance: u64,
@@ -49,7 +49,7 @@ pub fn update_registry_and_shuffling_data(
 
 /// Returns `true` if the validator registry should be updated during an epoch processing.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn should_update_validator_registry(
     state: &BeaconState,
     spec: &ChainSpec,
@@ -78,7 +78,7 @@ pub fn should_update_validator_registry(
 ///
 /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn update_validator_registry(
     state: &mut BeaconState,
     current_total_balance: u64,
@@ -133,7 +133,7 @@ pub fn update_validator_registry(
 
 /// Activate the validator of the given ``index``.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn activate_validator(
     state: &mut BeaconState,
     validator_index: usize,
diff --git a/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs b/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs
index 02149cc5a..afa78c9c0 100644
--- a/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs
+++ b/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs
@@ -160,7 +160,7 @@ impl ValidatorStatuses {
     /// - Active validators
     /// - Total balances for the current and previous epochs.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn new(state: &BeaconState, spec: &ChainSpec) -> Result<Self, BeaconStateError> {
         let mut statuses = Vec::with_capacity(state.validator_registry.len());
         let mut total_balances = TotalBalances::default();
@@ -195,7 +195,7 @@ impl ValidatorStatuses {
     /// Process some attestations from the given `state` updating the `statuses` and
     /// `total_balances` fields.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn process_attestations(
         &mut self,
         state: &BeaconState,
@@ -261,7 +261,7 @@ impl ValidatorStatuses {
     /// Update the `statuses` for each validator based upon whether or not they attested to the
     /// "winning" shard block root for the previous epoch.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn process_winning_roots(
         &mut self,
         state: &BeaconState,
@@ -297,14 +297,14 @@ impl ValidatorStatuses {
 /// Returns the distance between when the attestation was created and when it was included in a
 /// block.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn inclusion_distance(a: &PendingAttestation) -> Slot {
     a.inclusion_slot - a.data.slot
 }
 
 /// Returns `true` if some `PendingAttestation` is from the supplied `epoch`.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn is_from_epoch(a: &PendingAttestation, epoch: Epoch, spec: &ChainSpec) -> bool {
     a.data.slot.epoch(spec.slots_per_epoch) == epoch
 }
@@ -312,7 +312,7 @@ fn is_from_epoch(a: &PendingAttestation, epoch: Epoch, spec: &ChainSpec) -> bool
 /// Returns `true` if a `PendingAttestation` and `BeaconState` share the same beacon block hash for
 /// the first slot of the given epoch.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn has_common_epoch_boundary_root(
     a: &PendingAttestation,
     state: &BeaconState,
@@ -328,7 +328,7 @@ fn has_common_epoch_boundary_root(
 /// Returns `true` if a `PendingAttestation` and `BeaconState` share the same beacon block hash for
 /// the current slot of the `PendingAttestation`.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn has_common_beacon_block_root(
     a: &PendingAttestation,
     state: &BeaconState,
diff --git a/eth2/state_processing/src/per_epoch_processing/winning_root.rs b/eth2/state_processing/src/per_epoch_processing/winning_root.rs
index 97cff3e13..5d31dff31 100644
--- a/eth2/state_processing/src/per_epoch_processing/winning_root.rs
+++ b/eth2/state_processing/src/per_epoch_processing/winning_root.rs
@@ -16,7 +16,7 @@ impl WinningRoot {
     /// A winning root is "better" than another if it has a higher `total_attesting_balance`. Ties
     /// are broken by favouring the higher `crosslink_data_root` value.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn is_better_than(&self, other: &Self) -> bool {
         if self.total_attesting_balance > other.total_attesting_balance {
             true
@@ -34,7 +34,7 @@ impl WinningRoot {
 /// The `WinningRoot` object also contains additional fields that are useful in later stages of
 /// per-epoch processing.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn winning_root(
     state: &BeaconState,
     shard: u64,
@@ -89,7 +89,7 @@ pub fn winning_root(
 
 /// Returns `true` if pending attestation `a` is eligible to become a winning root.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn is_eligible_for_winning_root(state: &BeaconState, a: &PendingAttestation, shard: Shard) -> bool {
     if shard >= state.latest_crosslinks.len() as u64 {
         return false;
@@ -100,7 +100,7 @@ fn is_eligible_for_winning_root(state: &BeaconState, a: &PendingAttestation, sha
 
 /// Returns all indices which voted for a given crosslink. Does not contain duplicates.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn get_attesting_validator_indices(
     state: &BeaconState,
     shard: u64,
diff --git a/eth2/state_processing/src/per_slot_processing.rs b/eth2/state_processing/src/per_slot_processing.rs
index 7d2bb468f..378d5dd2e 100644
--- a/eth2/state_processing/src/per_slot_processing.rs
+++ b/eth2/state_processing/src/per_slot_processing.rs
@@ -10,7 +10,7 @@ pub enum Error {
 
 /// Advances a state forward by one slot, performing per-epoch processing if required.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn per_slot_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> {
     cache_state(state, spec)?;
 
diff --git a/eth2/types/src/attestation.rs b/eth2/types/src/attestation.rs
index c43692a7b..f7bfdaab9 100644
--- a/eth2/types/src/attestation.rs
+++ b/eth2/types/src/attestation.rs
@@ -9,7 +9,7 @@ use tree_hash_derive::{SignedRoot, TreeHash};
 
 /// Details an attestation that can be slashable.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(
     Debug,
     Clone,
diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data.rs
index 305ddafe0..f8a0ecd15 100644
--- a/eth2/types/src/attestation_data.rs
+++ b/eth2/types/src/attestation_data.rs
@@ -9,7 +9,7 @@ use tree_hash_derive::{SignedRoot, TreeHash};
 
 /// The data upon which an attestation is based.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(
     Debug,
     Clone,
diff --git a/eth2/types/src/attestation_data_and_custody_bit.rs b/eth2/types/src/attestation_data_and_custody_bit.rs
index 59a4eee77..e5dc920dc 100644
--- a/eth2/types/src/attestation_data_and_custody_bit.rs
+++ b/eth2/types/src/attestation_data_and_custody_bit.rs
@@ -7,7 +7,7 @@ use tree_hash_derive::TreeHash;
 
 /// Used for pairing an attestation with a proof-of-custody.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode, TreeHash)]
 pub struct AttestationDataAndCustodyBit {
     pub data: AttestationData,
diff --git a/eth2/types/src/attester_slashing.rs b/eth2/types/src/attester_slashing.rs
index 0600e0ecc..b5e851dbd 100644
--- a/eth2/types/src/attester_slashing.rs
+++ b/eth2/types/src/attester_slashing.rs
@@ -7,7 +7,7 @@ use tree_hash_derive::TreeHash;
 
 /// Two conflicting attestations.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
 pub struct AttesterSlashing {
     pub slashable_attestation_1: SlashableAttestation,
diff --git a/eth2/types/src/beacon_block.rs b/eth2/types/src/beacon_block.rs
index bc6ccb0d5..b4d2752d6 100644
--- a/eth2/types/src/beacon_block.rs
+++ b/eth2/types/src/beacon_block.rs
@@ -10,7 +10,7 @@ use tree_hash_derive::{SignedRoot, TreeHash};
 
 /// A block of the `BeaconChain`.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(
     Debug,
     PartialEq,
@@ -35,7 +35,7 @@ pub struct BeaconBlock {
 impl BeaconBlock {
     /// Returns an empty block to be used during genesis.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn empty(spec: &ChainSpec) -> BeaconBlock {
         BeaconBlock {
             slot: spec.genesis_slot,
@@ -60,7 +60,7 @@ impl BeaconBlock {
 
     /// Returns the `tree_hash_root | update` of the block.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn canonical_root(&self) -> Hash256 {
         Hash256::from_slice(&self.tree_hash_root()[..])
     }
@@ -72,7 +72,7 @@ impl BeaconBlock {
     ///
     /// Note: performs a full tree-hash of `self.body`.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn block_header(&self) -> BeaconBlockHeader {
         BeaconBlockHeader {
             slot: self.slot,
@@ -85,7 +85,7 @@ impl BeaconBlock {
 
     /// Returns a "temporary" header, where the `state_root` is `spec.zero_hash`.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn temporary_block_header(&self, spec: &ChainSpec) -> BeaconBlockHeader {
         BeaconBlockHeader {
             state_root: spec.zero_hash,
diff --git a/eth2/types/src/beacon_block_body.rs b/eth2/types/src/beacon_block_body.rs
index 0414d0d72..de4951f1f 100644
--- a/eth2/types/src/beacon_block_body.rs
+++ b/eth2/types/src/beacon_block_body.rs
@@ -8,7 +8,7 @@ use tree_hash_derive::TreeHash;
 
 /// The body of a `BeaconChain` block, containing operations.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
 pub struct BeaconBlockBody {
     pub randao_reveal: Signature,
diff --git a/eth2/types/src/beacon_block_header.rs b/eth2/types/src/beacon_block_header.rs
index 9076437c0..fa71bd26b 100644
--- a/eth2/types/src/beacon_block_header.rs
+++ b/eth2/types/src/beacon_block_header.rs
@@ -10,7 +10,7 @@ use tree_hash_derive::{SignedRoot, TreeHash};
 
 /// A header of a `BeaconBlock`.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(
     Debug,
     PartialEq,
@@ -35,14 +35,14 @@ pub struct BeaconBlockHeader {
 impl BeaconBlockHeader {
     /// Returns the `tree_hash_root` of the header.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn canonical_root(&self) -> Hash256 {
         Hash256::from_slice(&self.tree_hash_root()[..])
     }
 
     /// Given a `body`, consumes `self` and returns a complete `BeaconBlock`.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn into_block(self, body: BeaconBlockBody) -> BeaconBlock {
         BeaconBlock {
             slot: self.slot,
diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs
index 0461e947b..eef408308 100644
--- a/eth2/types/src/beacon_state.rs
+++ b/eth2/types/src/beacon_state.rs
@@ -46,7 +46,7 @@ pub enum Error {
 
 /// The state of the `BeaconChain` at some slot.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, TestRandom, Encode, Decode, TreeHash)]
 pub struct BeaconState {
     // Misc
@@ -120,7 +120,7 @@ impl BeaconState {
     /// This does not fully build a genesis beacon state, it omits processing of initial validator
     /// deposits. To obtain a full genesis beacon state, use the `BeaconStateBuilder`.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn genesis(genesis_time: u64, latest_eth1_data: Eth1Data, spec: &ChainSpec) -> BeaconState {
         let initial_crosslink = Crosslink {
             epoch: spec.genesis_epoch,
@@ -192,7 +192,7 @@ impl BeaconState {
 
     /// Returns the `tree_hash_root` of the state.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn canonical_root(&self) -> Hash256 {
         Hash256::from_slice(&self.tree_hash_root()[..])
     }
@@ -221,7 +221,7 @@ impl BeaconState {
 
     /// The epoch corresponding to `self.slot`.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn current_epoch(&self, spec: &ChainSpec) -> Epoch {
         self.slot.epoch(spec.slots_per_epoch)
     }
@@ -230,14 +230,14 @@ impl BeaconState {
     ///
     /// If the current epoch is the genesis epoch, the genesis_epoch is returned.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn previous_epoch(&self, spec: &ChainSpec) -> Epoch {
         self.current_epoch(&spec) - 1
     }
 
     /// The epoch following `self.current_epoch()`.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn next_epoch(&self, spec: &ChainSpec) -> Epoch {
         self.current_epoch(spec) + 1
     }
@@ -250,7 +250,7 @@ impl BeaconState {
     ///
     /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn get_cached_active_validator_indices(
         &self,
         relative_epoch: RelativeEpoch,
@@ -265,7 +265,7 @@ impl BeaconState {
     ///
     /// Does not utilize the cache, performs a full iteration over the validator registry.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn get_active_validator_indices(&self, epoch: Epoch) -> Vec<usize> {
         get_active_validator_indices(&self.validator_registry, epoch)
     }
@@ -274,7 +274,7 @@ impl BeaconState {
     ///
     /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn get_crosslink_committees_at_slot(
         &self,
         slot: Slot,
@@ -299,7 +299,7 @@ impl BeaconState {
     ///
     /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn get_crosslink_committee_for_shard(
         &self,
         epoch: Epoch,
@@ -325,7 +325,7 @@ impl BeaconState {
     ///
     /// If the state does not contain an index for a beacon proposer at the requested `slot`, then `None` is returned.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn get_beacon_proposer_index(
         &self,
         slot: Slot,
@@ -354,7 +354,7 @@ impl BeaconState {
 
     /// Safely obtains the index for latest block roots, given some `slot`.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     fn get_latest_block_roots_index(&self, slot: Slot, spec: &ChainSpec) -> Result<usize, Error> {
         if (slot < self.slot) && (self.slot <= slot + spec.slots_per_historical_root as u64) {
             let i = slot.as_usize() % spec.slots_per_historical_root;
@@ -370,7 +370,7 @@ impl BeaconState {
 
     /// Return the block root at a recent `slot`.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn get_block_root(
         &self,
         slot: Slot,
@@ -382,7 +382,7 @@ impl BeaconState {
 
     /// Sets the block root for some given slot.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn set_block_root(
         &mut self,
         slot: Slot,
@@ -396,7 +396,7 @@ impl BeaconState {
 
     /// Safely obtains the index for `latest_randao_mixes`
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     fn get_randao_mix_index(&self, epoch: Epoch, spec: &ChainSpec) -> Result<usize, Error> {
         let current_epoch = self.current_epoch(spec);
 
@@ -420,7 +420,7 @@ impl BeaconState {
     ///
     /// See `Self::get_randao_mix`.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn update_randao_mix(
         &mut self,
         epoch: Epoch,
@@ -438,7 +438,7 @@ impl BeaconState {
 
     /// Return the randao mix at a recent ``epoch``.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn get_randao_mix(&self, epoch: Epoch, spec: &ChainSpec) -> Result<&Hash256, Error> {
         let i = self.get_randao_mix_index(epoch, spec)?;
         Ok(&self.latest_randao_mixes[i])
@@ -446,7 +446,7 @@ impl BeaconState {
 
     /// Set the randao mix at a recent ``epoch``.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn set_randao_mix(
         &mut self,
         epoch: Epoch,
@@ -460,7 +460,7 @@ impl BeaconState {
 
     /// Safely obtains the index for `latest_active_index_roots`, given some `epoch`.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     fn get_active_index_root_index(&self, epoch: Epoch, spec: &ChainSpec) -> Result<usize, Error> {
         let current_epoch = self.current_epoch(spec);
 
@@ -482,7 +482,7 @@ impl BeaconState {
 
     /// Return the `active_index_root` at a recent `epoch`.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn get_active_index_root(&self, epoch: Epoch, spec: &ChainSpec) -> Result<Hash256, Error> {
         let i = self.get_active_index_root_index(epoch, spec)?;
         Ok(self.latest_active_index_roots[i])
@@ -490,7 +490,7 @@ impl BeaconState {
 
     /// Set the `active_index_root` at a recent `epoch`.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn set_active_index_root(
         &mut self,
         epoch: Epoch,
@@ -504,7 +504,7 @@ impl BeaconState {
 
     /// Replace `active_index_roots` with clones of `index_root`.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn fill_active_index_roots_with(&mut self, index_root: Hash256, spec: &ChainSpec) {
         self.latest_active_index_roots =
             vec![index_root; spec.latest_active_index_roots_length as usize].into()
@@ -512,7 +512,7 @@ impl BeaconState {
 
     /// Safely obtains the index for latest state roots, given some `slot`.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     fn get_latest_state_roots_index(&self, slot: Slot, spec: &ChainSpec) -> Result<usize, Error> {
         if (slot < self.slot) && (self.slot <= slot + spec.slots_per_historical_root as u64) {
             let i = slot.as_usize() % spec.slots_per_historical_root;
@@ -528,7 +528,7 @@ impl BeaconState {
 
     /// Gets the state root for some slot.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn get_state_root(&mut self, slot: Slot, spec: &ChainSpec) -> Result<&Hash256, Error> {
         let i = self.get_latest_state_roots_index(slot, spec)?;
         Ok(&self.latest_state_roots[i])
@@ -536,7 +536,7 @@ impl BeaconState {
 
     /// Sets the latest state root for slot.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn set_state_root(
         &mut self,
         slot: Slot,
@@ -550,7 +550,7 @@ impl BeaconState {
 
     /// Safely obtains the index for `latest_slashed_balances`, given some `epoch`.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     fn get_slashed_balance_index(&self, epoch: Epoch, spec: &ChainSpec) -> Result<usize, Error> {
         let i = epoch.as_usize() % spec.latest_slashed_exit_length;
 
@@ -565,7 +565,7 @@ impl BeaconState {
 
     /// Gets the total slashed balances for some epoch.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn get_slashed_balance(&self, epoch: Epoch, spec: &ChainSpec) -> Result<u64, Error> {
         let i = self.get_slashed_balance_index(epoch, spec)?;
         Ok(self.latest_slashed_balances[i])
@@ -573,7 +573,7 @@ impl BeaconState {
 
     /// Sets the total slashed balances for some epoch.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn set_slashed_balance(
         &mut self,
         epoch: Epoch,
@@ -587,7 +587,7 @@ impl BeaconState {
 
     /// Generate a seed for the given `epoch`.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn generate_seed(&self, epoch: Epoch, spec: &ChainSpec) -> Result<Hash256, Error> {
         let mut input = self
             .get_randao_mix(epoch - spec.min_seed_lookahead, spec)?
@@ -603,7 +603,7 @@ impl BeaconState {
 
     /// Return the effective balance (also known as "balance at stake") for a validator with the given ``index``.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn get_effective_balance(
         &self,
         validator_index: usize,
@@ -618,14 +618,14 @@ impl BeaconState {
 
     ///  Return the epoch at which an activation or exit triggered in ``epoch`` takes effect.
     ///
-    ///  Spec v0.5.0
+    ///  Spec v0.5.1
     pub fn get_delayed_activation_exit_epoch(&self, epoch: Epoch, spec: &ChainSpec) -> Epoch {
         epoch + 1 + spec.activation_exit_delay
     }
 
     /// Initiate an exit for the validator of the given `index`.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn initiate_validator_exit(&mut self, validator_index: usize) {
         self.validator_registry[validator_index].initiated_exit = true;
     }
@@ -637,7 +637,7 @@ impl BeaconState {
     ///
     /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn get_attestation_duties(
         &self,
         validator_index: usize,
@@ -653,7 +653,7 @@ impl BeaconState {
 
     /// Return the combined effective balance of an array of validators.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn get_total_balance(
         &self,
         validator_indices: &[usize],
diff --git a/eth2/types/src/beacon_state/epoch_cache.rs b/eth2/types/src/beacon_state/epoch_cache.rs
index dd9ae3403..1a63e9eb9 100644
--- a/eth2/types/src/beacon_state/epoch_cache.rs
+++ b/eth2/types/src/beacon_state/epoch_cache.rs
@@ -138,7 +138,7 @@ impl EpochCache {
 /// Returns a list of all `validator_registry` indices where the validator is active at the given
 /// `epoch`.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn get_active_validator_indices(validators: &[Validator], epoch: Epoch) -> Vec<usize> {
     let mut active = Vec::with_capacity(validators.len());
 
diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs
index 0042304f8..f3c92b42c 100644
--- a/eth2/types/src/chain_spec.rs
+++ b/eth2/types/src/chain_spec.rs
@@ -8,7 +8,7 @@ const GWEI: u64 = 1_000_000_000;
 
 /// Each of the BLS signature domains.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub enum Domain {
     BeaconBlock,
     Randao,
@@ -20,7 +20,7 @@ pub enum Domain {
 
 /// Holds all the "constants" for a BeaconChain.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(PartialEq, Debug, Clone, Deserialize)]
 #[serde(default)]
 pub struct ChainSpec {
@@ -126,7 +126,7 @@ pub struct ChainSpec {
 impl ChainSpec {
     /// Return the number of committees in one epoch.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn get_epoch_committee_count(&self, active_validator_count: usize) -> u64 {
         std::cmp::max(
             1,
@@ -139,7 +139,7 @@ impl ChainSpec {
 
     /// Get the domain number that represents the fork meta and signature domain.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn get_domain(&self, epoch: Epoch, domain: Domain, fork: &Fork) -> u64 {
         let domain_constant = match domain {
             Domain::BeaconBlock => self.domain_beacon_block,
@@ -161,7 +161,7 @@ impl ChainSpec {
 
     /// Returns a `ChainSpec` compatible with the Ethereum Foundation specification.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn foundation() -> Self {
         let genesis_slot = Slot::new(2_u64.pow(32));
         let slots_per_epoch = 64;
diff --git a/eth2/types/src/crosslink.rs b/eth2/types/src/crosslink.rs
index a0fd7e0b3..623226ad6 100644
--- a/eth2/types/src/crosslink.rs
+++ b/eth2/types/src/crosslink.rs
@@ -8,7 +8,7 @@ use tree_hash_derive::TreeHash;
 
 /// Specifies the block hash for a shard at an epoch.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(
     Debug,
     Clone,
diff --git a/eth2/types/src/deposit.rs b/eth2/types/src/deposit.rs
index bd3355a3f..291173d34 100644
--- a/eth2/types/src/deposit.rs
+++ b/eth2/types/src/deposit.rs
@@ -8,7 +8,7 @@ use tree_hash_derive::TreeHash;
 
 /// A deposit to potentially become a beacon chain validator.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
 pub struct Deposit {
     pub proof: TreeHashVector<Hash256>,
diff --git a/eth2/types/src/deposit_data.rs b/eth2/types/src/deposit_data.rs
index f8726e95d..bc96ac7c4 100644
--- a/eth2/types/src/deposit_data.rs
+++ b/eth2/types/src/deposit_data.rs
@@ -8,7 +8,7 @@ use tree_hash_derive::TreeHash;
 
 /// Data generated by the deposit contract.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
 pub struct DepositData {
     pub amount: u64,
diff --git a/eth2/types/src/deposit_input.rs b/eth2/types/src/deposit_input.rs
index 828496293..be2106cb4 100644
--- a/eth2/types/src/deposit_input.rs
+++ b/eth2/types/src/deposit_input.rs
@@ -10,7 +10,7 @@ use tree_hash_derive::{SignedRoot, TreeHash};
 
 /// The data supplied by the user to the deposit contract.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(
     Debug,
     PartialEq,
@@ -33,7 +33,7 @@ pub struct DepositInput {
 impl DepositInput {
     /// Generate the 'proof_of_posession' signature for a given DepositInput details.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn create_proof_of_possession(
         &self,
         secret_key: &SecretKey,
@@ -49,7 +49,7 @@ impl DepositInput {
 
     /// Verify that proof-of-possession is valid.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn validate_proof_of_possession(
         &self,
         epoch: Epoch,
diff --git a/eth2/types/src/eth1_data.rs b/eth2/types/src/eth1_data.rs
index c1348cfba..2ad460d13 100644
--- a/eth2/types/src/eth1_data.rs
+++ b/eth2/types/src/eth1_data.rs
@@ -8,7 +8,7 @@ use tree_hash_derive::TreeHash;
 
 /// Contains data obtained from the Eth1 chain.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(
     Debug, PartialEq, Clone, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom,
 )]
diff --git a/eth2/types/src/eth1_data_vote.rs b/eth2/types/src/eth1_data_vote.rs
index a9741f065..7a77c8ff0 100644
--- a/eth2/types/src/eth1_data_vote.rs
+++ b/eth2/types/src/eth1_data_vote.rs
@@ -8,7 +8,7 @@ use tree_hash_derive::TreeHash;
 
 /// A summation of votes for some `Eth1Data`.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(
     Debug, PartialEq, Clone, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom,
 )]
diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs
index 99908e9ed..d99842855 100644
--- a/eth2/types/src/fork.rs
+++ b/eth2/types/src/fork.rs
@@ -11,7 +11,7 @@ use tree_hash_derive::TreeHash;
 
 /// Specifies a fork of the `BeaconChain`, to prevent replay attacks.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(
     Debug, Clone, PartialEq, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom,
 )]
@@ -26,7 +26,7 @@ pub struct Fork {
 impl Fork {
     /// Initialize the `Fork` from the genesis parameters in the `spec`.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn genesis(spec: &ChainSpec) -> Self {
         let mut current_version: [u8; 4] = [0; 4];
         current_version.copy_from_slice(&int_to_bytes4(spec.genesis_fork_version));
@@ -40,7 +40,7 @@ impl Fork {
 
     /// Return the fork version of the given ``epoch``.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn get_fork_version(&self, epoch: Epoch) -> [u8; 4] {
         if epoch < self.epoch {
             return self.previous_version;
diff --git a/eth2/types/src/historical_batch.rs b/eth2/types/src/historical_batch.rs
index 23c26901e..c4f62fcfc 100644
--- a/eth2/types/src/historical_batch.rs
+++ b/eth2/types/src/historical_batch.rs
@@ -8,7 +8,7 @@ use tree_hash_derive::TreeHash;
 
 /// Historical block and state roots.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
 pub struct HistoricalBatch {
     pub block_roots: TreeHashVector<Hash256>,
diff --git a/eth2/types/src/pending_attestation.rs b/eth2/types/src/pending_attestation.rs
index 5cbe1edeb..ce9ce3d77 100644
--- a/eth2/types/src/pending_attestation.rs
+++ b/eth2/types/src/pending_attestation.rs
@@ -8,7 +8,7 @@ use tree_hash_derive::TreeHash;
 
 /// An attestation that has been included in the state but not yet fully processed.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
 pub struct PendingAttestation {
     pub aggregation_bitfield: Bitfield,
diff --git a/eth2/types/src/proposer_slashing.rs b/eth2/types/src/proposer_slashing.rs
index 901f02388..a3501a5bd 100644
--- a/eth2/types/src/proposer_slashing.rs
+++ b/eth2/types/src/proposer_slashing.rs
@@ -8,7 +8,7 @@ use tree_hash_derive::TreeHash;
 
 /// Two conflicting proposals from the same proposer (validator).
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
 pub struct ProposerSlashing {
     pub proposer_index: u64,
diff --git a/eth2/types/src/relative_epoch.rs b/eth2/types/src/relative_epoch.rs
index 8f895e97a..6538ca4aa 100644
--- a/eth2/types/src/relative_epoch.rs
+++ b/eth2/types/src/relative_epoch.rs
@@ -10,7 +10,7 @@ pub enum Error {
 /// Defines the epochs relative to some epoch. Most useful when referring to the committees prior
 /// to and following some epoch.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(Debug, PartialEq, Clone, Copy)]
 pub enum RelativeEpoch {
     /// The prior epoch.
@@ -32,7 +32,7 @@ pub enum RelativeEpoch {
 impl RelativeEpoch {
     /// Returns the `epoch` that `self` refers to, with respect to the `base` epoch.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn into_epoch(self, base: Epoch) -> Epoch {
         match self {
             RelativeEpoch::Previous => base - 1,
@@ -51,7 +51,7 @@ impl RelativeEpoch {
     /// - `AmbiguiousNextEpoch` whenever `other` is one after `base`, because it's unknowable if
     ///   there will be a registry change.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn from_epoch(base: Epoch, other: Epoch) -> Result<Self, Error> {
         if other == base - 1 {
             Ok(RelativeEpoch::Previous)
diff --git a/eth2/types/src/slashable_attestation.rs b/eth2/types/src/slashable_attestation.rs
index 37462f006..9c460e482 100644
--- a/eth2/types/src/slashable_attestation.rs
+++ b/eth2/types/src/slashable_attestation.rs
@@ -10,7 +10,7 @@ use tree_hash_derive::{SignedRoot, TreeHash};
 ///
 /// To be included in an `AttesterSlashing`.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(
     Debug,
     PartialEq,
@@ -35,14 +35,14 @@ pub struct SlashableAttestation {
 impl SlashableAttestation {
     /// Check if ``attestation_data_1`` and ``attestation_data_2`` have the same target.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn is_double_vote(&self, other: &SlashableAttestation, spec: &ChainSpec) -> bool {
         self.data.slot.epoch(spec.slots_per_epoch) == other.data.slot.epoch(spec.slots_per_epoch)
     }
 
     /// Check if ``attestation_data_1`` surrounds ``attestation_data_2``.
     ///
-    /// Spec v0.5.0
+    /// Spec v0.5.1
     pub fn is_surround_vote(&self, other: &SlashableAttestation, spec: &ChainSpec) -> bool {
         let source_epoch_1 = self.data.source_epoch;
         let source_epoch_2 = other.data.source_epoch;
diff --git a/eth2/types/src/transfer.rs b/eth2/types/src/transfer.rs
index f40050bc4..82ead03d5 100644
--- a/eth2/types/src/transfer.rs
+++ b/eth2/types/src/transfer.rs
@@ -11,7 +11,7 @@ use tree_hash_derive::{SignedRoot, TreeHash};
 
 /// The data submitted to the deposit contract.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(
     Debug,
     Clone,
diff --git a/eth2/types/src/validator.rs b/eth2/types/src/validator.rs
index 67b4e85df..bbd68ed2b 100644
--- a/eth2/types/src/validator.rs
+++ b/eth2/types/src/validator.rs
@@ -7,7 +7,7 @@ use tree_hash_derive::TreeHash;
 
 /// Information about a `BeaconChain` validator.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TestRandom, TreeHash)]
 pub struct Validator {
     pub pubkey: PublicKey,
diff --git a/eth2/types/src/voluntary_exit.rs b/eth2/types/src/voluntary_exit.rs
index 16d22c544..cb872cb98 100644
--- a/eth2/types/src/voluntary_exit.rs
+++ b/eth2/types/src/voluntary_exit.rs
@@ -9,7 +9,7 @@ use tree_hash_derive::{SignedRoot, TreeHash};
 
 /// An exit voluntarily submitted a validator who wishes to withdraw.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(
     Debug,
     PartialEq,

From 701cc00d08029343dd6b298e1f42f84d56934316 Mon Sep 17 00:00:00 2001
From: Michael Sproul <michael@sigmaprime.io>
Date: Wed, 17 Apr 2019 11:29:06 +1000
Subject: [PATCH 076/137] questionable patch for TreeHashVector

---
 eth2/state_processing/tests/tests.rs | 2 +-
 eth2/types/src/tree_hash_vector.rs   | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/eth2/state_processing/tests/tests.rs b/eth2/state_processing/tests/tests.rs
index 6491e255a..ccad198bb 100644
--- a/eth2/state_processing/tests/tests.rs
+++ b/eth2/state_processing/tests/tests.rs
@@ -27,7 +27,7 @@ impl ExpectedState {
             ($field_name:ident) => {
                 if self.$field_name.as_ref().map_or(true, |$field_name| {
                     println!("  > Checking {}", stringify!($field_name));
-                    $field_name == &state.$field_name
+                    &state.$field_name == $field_name
                 }) {
                     vec![]
                 } else {
diff --git a/eth2/types/src/tree_hash_vector.rs b/eth2/types/src/tree_hash_vector.rs
index 1cc8e40a5..9b77e13dc 100644
--- a/eth2/types/src/tree_hash_vector.rs
+++ b/eth2/types/src/tree_hash_vector.rs
@@ -33,6 +33,12 @@ impl<T> DerefMut for TreeHashVector<T> {
     }
 }
 
+impl<T: PartialEq> PartialEq<Vec<T>> for TreeHashVector<T> {
+    fn eq(&self, other: &Vec<T>) -> bool {
+        &self.0 == other
+    }
+}
+
 impl<T> tree_hash::TreeHash for TreeHashVector<T>
 where
     T: TreeHash,

From f592183aa9153ebe4aba05f6a9905469e71dbe65 Mon Sep 17 00:00:00 2001
From: Michael Sproul <michael@sigmaprime.io>
Date: Wed, 17 Apr 2019 11:59:40 +1000
Subject: [PATCH 077/137] Fix signed_root vs tree_hash_root in per_slot

---
 eth2/state_processing/src/per_slot_processing.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eth2/state_processing/src/per_slot_processing.rs b/eth2/state_processing/src/per_slot_processing.rs
index 378d5dd2e..194e0d6c9 100644
--- a/eth2/state_processing/src/per_slot_processing.rs
+++ b/eth2/state_processing/src/per_slot_processing.rs
@@ -1,5 +1,5 @@
 use crate::*;
-use tree_hash::TreeHash;
+use tree_hash::{SignedRoot, TreeHash};
 use types::*;
 
 #[derive(Debug, PartialEq)]
@@ -38,7 +38,7 @@ fn cache_state(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> {
         state.latest_block_header.state_root = previous_slot_state_root
     }
 
-    let latest_block_root = Hash256::from_slice(&state.latest_block_header.tree_hash_root()[..]);
+    let latest_block_root = Hash256::from_slice(&state.latest_block_header.signed_root()[..]);
     state.set_block_root(previous_slot, latest_block_root, spec)?;
 
     // Set the state slot back to what it should be.

From c3779caedefd86b9ecb24eba8cd553be065d9bb5 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 17 Apr 2019 13:59:40 +1000
Subject: [PATCH 078/137] Add extra info to block proc. error message

---
 eth2/state_processing/src/per_block_processing.rs        | 9 +++++++--
 eth2/state_processing/src/per_block_processing/errors.rs | 5 ++++-
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/eth2/state_processing/src/per_block_processing.rs b/eth2/state_processing/src/per_block_processing.rs
index 257d92acf..58b948f62 100644
--- a/eth2/state_processing/src/per_block_processing.rs
+++ b/eth2/state_processing/src/per_block_processing.rs
@@ -107,9 +107,14 @@ pub fn process_block_header(
 ) -> Result<(), Error> {
     verify!(block.slot == state.slot, Invalid::StateSlotMismatch);
 
+    let expected_previous_block_root =
+        Hash256::from_slice(&state.latest_block_header.signed_root());
     verify!(
-        block.previous_block_root == Hash256::from_slice(&state.latest_block_header.signed_root()),
-        Invalid::ParentBlockRootMismatch
+        block.previous_block_root == expected_previous_block_root,
+        Invalid::ParentBlockRootMismatch {
+            state: expected_previous_block_root,
+            block: block.previous_block_root,
+        }
     );
 
     state.latest_block_header = block.temporary_block_header(spec);
diff --git a/eth2/state_processing/src/per_block_processing/errors.rs b/eth2/state_processing/src/per_block_processing/errors.rs
index 9c36e0238..d8627d359 100644
--- a/eth2/state_processing/src/per_block_processing/errors.rs
+++ b/eth2/state_processing/src/per_block_processing/errors.rs
@@ -67,7 +67,10 @@ impl_from_beacon_state_error!(BlockProcessingError);
 #[derive(Debug, PartialEq)]
 pub enum BlockInvalid {
     StateSlotMismatch,
-    ParentBlockRootMismatch,
+    ParentBlockRootMismatch {
+        state: Hash256,
+        block: Hash256,
+    },
     BadSignature,
     BadRandaoSignature,
     MaxAttestationsExceeded,

From bf1a93f44422d6f048f185d36d30f5acb76d5d94 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 17 Apr 2019 14:00:00 +1000
Subject: [PATCH 079/137] Allocate correctly for tree hash

---
 eth2/utils/tree_hash_derive/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs
index 4b7761f91..343287313 100644
--- a/eth2/utils/tree_hash_derive/src/lib.rs
+++ b/eth2/utils/tree_hash_derive/src/lib.rs
@@ -166,11 +166,12 @@ pub fn tree_hash_signed_root_derive(input: TokenStream) -> TokenStream {
     };
 
     let idents = get_signed_root_named_field_idents(&struct_data);
+    let num_elems = idents.len();
 
     let output = quote! {
         impl tree_hash::SignedRoot for #name {
             fn signed_root(&self) -> Vec<u8> {
-                let mut leaves = Vec::with_capacity(4 * tree_hash::HASHSIZE);
+                let mut leaves = Vec::with_capacity(#num_elems * tree_hash::HASHSIZE);
 
                 #(
                     leaves.append(&mut self.#idents.tree_hash_root());

From 343909ef31010a1e5b49e6c255d3474fb6ef9a32 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 17 Apr 2019 17:17:07 +1000
Subject: [PATCH 080/137] Fix boolean-bitfield serialization

---
 .../src/common/verify_bitfield.rs             | 54 ++++++++++-
 eth2/utils/boolean-bitfield/src/lib.rs        | 93 ++++++++++---------
 2 files changed, 98 insertions(+), 49 deletions(-)

diff --git a/eth2/state_processing/src/common/verify_bitfield.rs b/eth2/state_processing/src/common/verify_bitfield.rs
index 7b3c07086..570a240f1 100644
--- a/eth2/state_processing/src/common/verify_bitfield.rs
+++ b/eth2/state_processing/src/common/verify_bitfield.rs
@@ -25,9 +25,55 @@ mod test {
 
     #[test]
     fn bitfield_length() {
-        assert!(verify_bitfield_length(
-            &Bitfield::from_bytes(&[0b10000000]),
-            4
-        ));
+        assert_eq!(
+            verify_bitfield_length(&Bitfield::from_bytes(&[0b0000_0001]), 4),
+            true
+        );
+
+        assert_eq!(
+            verify_bitfield_length(&Bitfield::from_bytes(&[0b0001_0001]), 4),
+            false
+        );
+
+        assert_eq!(
+            verify_bitfield_length(&Bitfield::from_bytes(&[0b0000_0000]), 4),
+            true
+        );
+
+        assert_eq!(
+            verify_bitfield_length(&Bitfield::from_bytes(&[0b1000_0000]), 8),
+            true
+        );
+
+        assert_eq!(
+            verify_bitfield_length(&Bitfield::from_bytes(&[0b1000_0000, 0b0000_0000]), 16),
+            true
+        );
+
+        assert_eq!(
+            verify_bitfield_length(&Bitfield::from_bytes(&[0b1000_0000, 0b0000_0000]), 15),
+            false
+        );
+
+        assert_eq!(
+            verify_bitfield_length(&Bitfield::from_bytes(&[0b0000_0000, 0b0000_0000]), 8),
+            false
+        );
+
+        assert_eq!(
+            verify_bitfield_length(
+                &Bitfield::from_bytes(&[0b0000_0000, 0b0000_0000, 0b0000_0000]),
+                8
+            ),
+            false
+        );
+
+        assert_eq!(
+            verify_bitfield_length(
+                &Bitfield::from_bytes(&[0b0000_0000, 0b0000_0000, 0b0000_0000]),
+                24
+            ),
+            true
+        );
     }
 }
diff --git a/eth2/utils/boolean-bitfield/src/lib.rs b/eth2/utils/boolean-bitfield/src/lib.rs
index c19702ec9..a744c9498 100644
--- a/eth2/utils/boolean-bitfield/src/lib.rs
+++ b/eth2/utils/boolean-bitfield/src/lib.rs
@@ -54,10 +54,15 @@ impl BooleanBitfield {
     /// Create a new bitfield using the supplied `bytes` as input
     pub fn from_bytes(bytes: &[u8]) -> Self {
         Self {
-            0: BitVec::from_bytes(bytes),
+            0: BitVec::from_bytes(&reverse_bit_order(bytes.to_vec())),
         }
     }
 
+    /// Returns a vector of bytes representing the bitfield
+    pub fn to_bytes(&self) -> Vec<u8> {
+        reverse_bit_order(self.0.to_bytes().to_vec())
+    }
+
     /// Read the value of a bit.
     ///
     /// If the index is in bounds, then result is Ok(value) where value is `true` if the bit is 1 and `false` if the bit is 0.
@@ -86,11 +91,6 @@ impl BooleanBitfield {
         previous
     }
 
-    /// Returns the index of the highest set bit. Some(n) if some bit is set, None otherwise.
-    pub fn highest_set_bit(&self) -> Option<usize> {
-        self.0.iter().rposition(|bit| bit)
-    }
-
     /// Returns the number of bits in this bitfield.
     pub fn len(&self) -> usize {
         self.0.len()
@@ -116,12 +116,6 @@ impl BooleanBitfield {
         self.0.iter().filter(|&bit| bit).count()
     }
 
-    /// Returns a vector of bytes representing the bitfield
-    /// Note that this returns the bit layout of the underlying implementation in the `bit-vec` crate.
-    pub fn to_bytes(&self) -> Vec<u8> {
-        self.0.to_bytes()
-    }
-
     /// Compute the intersection (binary-and) of this bitfield with another. Lengths must match.
     pub fn intersection(&self, other: &Self) -> Self {
         let mut res = self.clone();
@@ -218,17 +212,7 @@ impl Decodable for BooleanBitfield {
             Ok((BooleanBitfield::new(), index + ssz::LENGTH_BYTES))
         } else {
             let bytes = &bytes[(index + 4)..(index + len + 4)];
-
-            let count = len * 8;
-            let mut field = BooleanBitfield::with_capacity(count);
-            for (byte_index, byte) in bytes.iter().enumerate() {
-                for i in 0..8 {
-                    let bit = byte & (128 >> i);
-                    if bit != 0 {
-                        field.set(8 * byte_index + i, true);
-                    }
-                }
-            }
+            let field = BooleanBitfield::from_bytes(bytes);
 
             let index = index + ssz::LENGTH_BYTES + len;
             Ok((field, index))
@@ -251,7 +235,7 @@ impl Serialize for BooleanBitfield {
     where
         S: Serializer,
     {
-        serializer.serialize_str(&encode(&reverse_bit_order(self.to_bytes())))
+        serializer.serialize_str(&encode(self.to_bytes()))
     }
 }
 
@@ -265,11 +249,27 @@ impl<'de> Deserialize<'de> for BooleanBitfield {
         // bit from the end of the hex string, e.g.
         // "0xef01" => [0xef, 0x01] => [0b1000_0000, 0b1111_1110]
         let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?;
-        Ok(BooleanBitfield::from_bytes(&reverse_bit_order(bytes)))
+        Ok(BooleanBitfield::from_bytes(&bytes))
     }
 }
 
-tree_hash_ssz_encoding_as_list!(BooleanBitfield);
+impl tree_hash::TreeHash for BooleanBitfield {
+    fn tree_hash_type() -> tree_hash::TreeHashType {
+        tree_hash::TreeHashType::List
+    }
+
+    fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+        unreachable!("List should never be packed.")
+    }
+
+    fn tree_hash_packing_factor() -> usize {
+        unreachable!("List should never be packed.")
+    }
+
+    fn tree_hash_root(&self) -> Vec<u8> {
+        self.to_bytes().tree_hash_root()
+    }
+}
 
 #[cfg(test)]
 mod tests {
@@ -322,7 +322,7 @@ mod tests {
         assert_eq!(field.num_set_bits(), 100);
     }
 
-    const INPUT: &[u8] = &[0b0000_0010, 0b0000_0010];
+    const INPUT: &[u8] = &[0b0100_0000, 0b0100_0000];
 
     #[test]
     fn test_get_from_bitfield() {
@@ -348,18 +348,6 @@ mod tests {
         assert!(!previous);
     }
 
-    #[test]
-    fn test_highest_set_bit() {
-        let field = BooleanBitfield::from_bytes(INPUT);
-        assert_eq!(field.highest_set_bit().unwrap(), 14);
-
-        let field = BooleanBitfield::from_bytes(&[0b0000_0011]);
-        assert_eq!(field.highest_set_bit().unwrap(), 7);
-
-        let field = BooleanBitfield::new();
-        assert_eq!(field.highest_set_bit(), None);
-    }
-
     #[test]
     fn test_len() {
         let field = BooleanBitfield::from_bytes(INPUT);
@@ -440,15 +428,30 @@ mod tests {
     #[test]
     fn test_ssz_encode() {
         let field = create_test_bitfield();
-
         let mut stream = SszStream::new();
         stream.append(&field);
-        assert_eq!(stream.drain(), vec![2, 0, 0, 0, 225, 192]);
+        assert_eq!(stream.drain(), vec![2, 0, 0, 0, 0b0000_0011, 0b1000_0111]);
 
         let field = BooleanBitfield::from_elem(18, true);
         let mut stream = SszStream::new();
         stream.append(&field);
-        assert_eq!(stream.drain(), vec![3, 0, 0, 0, 255, 255, 192]);
+        assert_eq!(
+            stream.drain(),
+            vec![3, 0, 0, 0, 0b0000_0011, 0b1111_1111, 0b1111_1111]
+        );
+
+        let mut b = BooleanBitfield::new();
+        b.set(1, true);
+        assert_eq!(
+            ssz_encode(&b),
+            vec![
+                0b0000_0001,
+                0b0000_0000,
+                0b0000_0000,
+                0b0000_0000,
+                0b0000_0010
+            ]
+        );
     }
 
     fn create_test_bitfield() -> BooleanBitfield {
@@ -464,7 +467,7 @@ mod tests {
 
     #[test]
     fn test_ssz_decode() {
-        let encoded = vec![2, 0, 0, 0, 225, 192];
+        let encoded = vec![2, 0, 0, 0, 0b0000_0011, 0b1000_0111];
         let field = decode::<BooleanBitfield>(&encoded).unwrap();
         let expected = create_test_bitfield();
         assert_eq!(field, expected);
@@ -480,8 +483,8 @@ mod tests {
         use serde_yaml::Value;
 
         let data: &[(_, &[_])] = &[
-            ("0x01", &[0b10000000]),
-            ("0xf301", &[0b10000000, 0b11001111]),
+            ("0x01", &[0b00000001]),
+            ("0xf301", &[0b11110011, 0b00000001]),
         ];
         for (hex_data, bytes) in data {
             let bitfield = BooleanBitfield::from_bytes(bytes);

From 745d3605669705b3e2b74742e5a961ed364682fc Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 17 Apr 2019 17:17:43 +1000
Subject: [PATCH 081/137] Store state roots during slot processing

---
 eth2/state_processing/src/per_slot_processing.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/eth2/state_processing/src/per_slot_processing.rs b/eth2/state_processing/src/per_slot_processing.rs
index 194e0d6c9..a68f98c6d 100644
--- a/eth2/state_processing/src/per_slot_processing.rs
+++ b/eth2/state_processing/src/per_slot_processing.rs
@@ -38,6 +38,9 @@ fn cache_state(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> {
         state.latest_block_header.state_root = previous_slot_state_root
     }
 
+    // Store the previous slot's post state transition root.
+    state.set_state_root(previous_slot, previous_slot_state_root, spec)?;
+
     let latest_block_root = Hash256::from_slice(&state.latest_block_header.signed_root()[..]);
     state.set_block_root(previous_slot, latest_block_root, spec)?;
 

From 332795e8b7e14444871ff4bfc7f492cd0902f546 Mon Sep 17 00:00:00 2001
From: Michael Sproul <michael@sigmaprime.io>
Date: Wed, 17 Apr 2019 18:00:14 +1000
Subject: [PATCH 082/137] Revert "questionable patch for TreeHashVector"

This reverts commit 701cc00d08029343dd6b298e1f42f84d56934316.
---
 eth2/state_processing/tests/tests.rs | 2 +-
 eth2/types/src/tree_hash_vector.rs   | 6 ------
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/eth2/state_processing/tests/tests.rs b/eth2/state_processing/tests/tests.rs
index ccad198bb..6491e255a 100644
--- a/eth2/state_processing/tests/tests.rs
+++ b/eth2/state_processing/tests/tests.rs
@@ -27,7 +27,7 @@ impl ExpectedState {
             ($field_name:ident) => {
                 if self.$field_name.as_ref().map_or(true, |$field_name| {
                     println!("  > Checking {}", stringify!($field_name));
-                    &state.$field_name == $field_name
+                    $field_name == &state.$field_name
                 }) {
                     vec![]
                 } else {
diff --git a/eth2/types/src/tree_hash_vector.rs b/eth2/types/src/tree_hash_vector.rs
index 9b77e13dc..1cc8e40a5 100644
--- a/eth2/types/src/tree_hash_vector.rs
+++ b/eth2/types/src/tree_hash_vector.rs
@@ -33,12 +33,6 @@ impl<T> DerefMut for TreeHashVector<T> {
     }
 }
 
-impl<T: PartialEq> PartialEq<Vec<T>> for TreeHashVector<T> {
-    fn eq(&self, other: &Vec<T>) -> bool {
-        &self.0 == other
-    }
-}
-
 impl<T> tree_hash::TreeHash for TreeHashVector<T>
 where
     T: TreeHash,

From b201c52140134ddefb7b19ad8597f4b6054ebc09 Mon Sep 17 00:00:00 2001
From: Michael Sproul <michael@sigmaprime.io>
Date: Wed, 17 Apr 2019 18:07:28 +1000
Subject: [PATCH 083/137] state transition tests: use TreeHashVector

---
 eth2/state_processing/tests/tests.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eth2/state_processing/tests/tests.rs b/eth2/state_processing/tests/tests.rs
index 6491e255a..fcd034158 100644
--- a/eth2/state_processing/tests/tests.rs
+++ b/eth2/state_processing/tests/tests.rs
@@ -16,7 +16,7 @@ pub struct ExpectedState {
     pub current_epoch_attestations: Option<Vec<PendingAttestation>>,
     pub historical_roots: Option<Vec<Hash256>>,
     pub finalized_epoch: Option<Epoch>,
-    pub latest_block_roots: Option<Vec<Hash256>>,
+    pub latest_block_roots: Option<TreeHashVector<Hash256>>,
 }
 
 impl ExpectedState {

From 2155e3e293390539d2391eeda906f573a65fa59f Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 17 Apr 2019 18:54:21 +1000
Subject: [PATCH 084/137] Fix non-compiling tests

---
 eth2/state_processing/tests/tests.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/eth2/state_processing/tests/tests.rs b/eth2/state_processing/tests/tests.rs
index ccad198bb..dd611b459 100644
--- a/eth2/state_processing/tests/tests.rs
+++ b/eth2/state_processing/tests/tests.rs
@@ -1,6 +1,5 @@
 use serde_derive::Deserialize;
 use serde_yaml;
-#[cfg(not(debug_assertions))]
 use state_processing::{per_block_processing, per_slot_processing};
 use std::{fs::File, io::prelude::*, path::PathBuf};
 use types::*;

From 5e81a995ea4b9d7e623c3d7720aa935be58a2aca Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 17 Apr 2019 18:54:39 +1000
Subject: [PATCH 085/137] Use signed_root for canonical header ID

---
 eth2/types/src/beacon_block_header.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eth2/types/src/beacon_block_header.rs b/eth2/types/src/beacon_block_header.rs
index fa71bd26b..e4db3a721 100644
--- a/eth2/types/src/beacon_block_header.rs
+++ b/eth2/types/src/beacon_block_header.rs
@@ -5,7 +5,7 @@ use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
-use tree_hash::TreeHash;
+use tree_hash::{SignedRoot, TreeHash};
 use tree_hash_derive::{SignedRoot, TreeHash};
 
 /// A header of a `BeaconBlock`.
@@ -37,7 +37,7 @@ impl BeaconBlockHeader {
     ///
     /// Spec v0.5.1
     pub fn canonical_root(&self) -> Hash256 {
-        Hash256::from_slice(&self.tree_hash_root()[..])
+        Hash256::from_slice(&self.signed_root()[..])
     }
 
     /// Given a `body`, consumes `self` and returns a complete `BeaconBlock`.

From 7b853b33d54052c448020f4ffef3a630241f666b Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 17 Apr 2019 21:57:48 +1000
Subject: [PATCH 086/137] Add env vars to travis

---
 .travis.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 6233ea68b..f75f9e6ea 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,11 +6,12 @@ before_install:
   - sudo mv protoc3/include/* /usr/local/include/
   - sudo chown $USER /usr/local/bin/protoc
   - sudo chown -R $USER /usr/local/include/google
+env:
+  - BUILD=
+  - BUILD=--release
 script:
-  - cargo build --verbose --all
-  - cargo build --verbose --release --all
-  - cargo test --verbose --all
-  - cargo test --verbose --release --all
+  - cargo build --verbose $BUILD --all
+  - cargo test --verbose $BUILD --all
   - cargo test --manifest-path eth2/state_processing/Cargo.toml --verbose --release --features fake_crypto
   - cargo fmt --all -- --check
   # No clippy until later...

From 381388d9c2ee5f12035958f36e147ef8952da503 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Thu, 18 Apr 2019 06:45:25 +1000
Subject: [PATCH 087/137] Move state processing test into own build

---
 .travis.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index f75f9e6ea..7a0849894 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,12 +7,12 @@ before_install:
   - sudo chown $USER /usr/local/bin/protoc
   - sudo chown -R $USER /usr/local/include/google
 env:
-  - BUILD=
-  - BUILD=--release
+  - BUILD=--all
+  - BUILD=--release --all
+  - BUILD= --manifest-path eth2/state_processing/Cargo.toml --release --features fake_crypto
 script:
-  - cargo build --verbose $BUILD --all
-  - cargo test --verbose $BUILD --all
-  - cargo test --manifest-path eth2/state_processing/Cargo.toml --verbose --release --features fake_crypto
+  - cargo build --verbose $BUILD
+  - cargo test --verbose $BUILD
   - cargo fmt --all -- --check
   # No clippy until later...
   #- cargo clippy

From 2ee3b05bd382afcbeaf62934f390d8a17befd5e0 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Thu, 18 Apr 2019 19:10:13 +1000
Subject: [PATCH 088/137] Only build in debug for beta and nightly

---
 .travis.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 7a0849894..70b9d2133 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -24,6 +24,15 @@ matrix:
   allow_failures:
     - rust: nightly
   fast_finish: true
+  exclude:
+      - rust: beta
+        env: BUILD=--release --all
+      - rust: beta
+        env: BUILD= --manifest-path eth2/state_processing/Cargo.toml --release --features fake_crypto
+      - rust: nightly
+        env: BUILD=--release --all
+      - rust: nightly
+        env: BUILD= --manifest-path eth2/state_processing/Cargo.toml --release --features fake_crypto
 install:
   - rustup component add rustfmt
   - rustup component add clippy

From 4aeadfa60f6ec1b5361caa06cc51e619d4cb5e1c Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Sun, 21 Apr 2019 12:12:47 +1000
Subject: [PATCH 089/137] Remove "old" item requirement from treehash

---
 eth2/utils/tree_hash/Cargo.toml               |   3 +
 eth2/utils/tree_hash/src/cached_tree_hash.rs  | 143 +++++++--
 .../src/cached_tree_hash/btree_overlay.rs     | 120 ++++----
 .../tree_hash/src/cached_tree_hash/impls.rs   |  19 +-
 .../src/cached_tree_hash/impls/vec.rs         | 171 +++++------
 eth2/utils/tree_hash/tests/tests.rs           | 289 +++++++++++++++++-
 eth2/utils/tree_hash_derive/src/lib.rs        |  37 +--
 7 files changed, 561 insertions(+), 221 deletions(-)

diff --git a/eth2/utils/tree_hash/Cargo.toml b/eth2/utils/tree_hash/Cargo.toml
index 328d91577..7e23d2165 100644
--- a/eth2/utils/tree_hash/Cargo.toml
+++ b/eth2/utils/tree_hash/Cargo.toml
@@ -4,6 +4,9 @@ version = "0.1.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 
+[dev-dependencies]
+tree_hash_derive = { path = "../tree_hash_derive" }
+
 [dependencies]
 ethereum-types = "0.5"
 hashing = { path = "../hashing" }
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash.rs b/eth2/utils/tree_hash/src/cached_tree_hash.rs
index e093b2dd7..c2011fdf8 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash.rs
@@ -17,10 +17,13 @@ pub enum Error {
     UnableToObtainSlices,
     UnableToGrowMerkleTree,
     UnableToShrinkMerkleTree,
+    TreeCannotHaveZeroNodes,
     ShouldNeverBePacked(TreeHashType),
     BytesAreNotEvenChunks(usize),
     NoModifiedFieldForChunk(usize),
     NoBytesForChunk(usize),
+    NoOverlayForIndex(usize),
+    NotLeafNode(usize),
 }
 
 pub trait CachedTreeHash<T>: CachedTreeHashSubTree<T> + Sized {
@@ -36,12 +39,7 @@ pub trait CachedTreeHashSubTree<Item>: TreeHash {
 
     fn new_tree_hash_cache(&self) -> Result<TreeHashCache, Error>;
 
-    fn update_tree_hash_cache(
-        &self,
-        other: &Item,
-        cache: &mut TreeHashCache,
-        chunk: usize,
-    ) -> Result<usize, Error>;
+    fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error>;
 }
 
 fn children(parent: usize) -> (usize, usize) {
@@ -123,6 +121,10 @@ fn num_bytes(num_leaves: usize) -> usize {
 pub struct TreeHashCache {
     cache: Vec<u8>,
     chunk_modified: Vec<bool>,
+    overlays: Vec<BTreeOverlay>,
+
+    pub chunk_index: usize,
+    pub overlay_index: usize,
 }
 
 impl Into<Vec<u8>> for TreeHashCache {
@@ -139,10 +141,17 @@ impl TreeHashCache {
         item.new_tree_hash_cache()
     }
 
-    pub fn from_elems(cache: Vec<u8>, chunk_modified: Vec<bool>) -> Self {
+    pub fn from_elems(
+        cache: Vec<u8>,
+        chunk_modified: Vec<bool>,
+        overlays: Vec<BTreeOverlay>,
+    ) -> Self {
         Self {
             cache,
             chunk_modified,
+            overlays,
+            chunk_index: 0,
+            overlay_index: 0,
         }
     }
 
@@ -153,7 +162,7 @@ impl TreeHashCache {
     where
         T: CachedTreeHashSubTree<T>,
     {
-        let offset_handler = BTreeOverlay::new(item, 0)?;
+        let overlay = BTreeOverlay::new(item, 0)?;
 
         // Note how many leaves were provided. If is not a power-of-two, we'll need to pad it out
         // later.
@@ -161,7 +170,7 @@ impl TreeHashCache {
 
         // Allocate enough bytes to store the internal nodes and the leaves and subtrees, then fill
         // all the to-be-built internal nodes with zeros and append the leaves and subtrees.
-        let internal_node_bytes = offset_handler.num_internal_nodes * BYTES_PER_CHUNK;
+        let internal_node_bytes = overlay.num_internal_nodes() * BYTES_PER_CHUNK;
         let leaves_and_subtrees_bytes = leaves_and_subtrees
             .iter()
             .fold(0, |acc, t| acc + t.bytes_len());
@@ -169,13 +178,19 @@ impl TreeHashCache {
         cache.resize(internal_node_bytes, 0);
 
         // Allocate enough bytes to store all the leaves.
-        let mut leaves = Vec::with_capacity(offset_handler.num_leaf_nodes * HASHSIZE);
+        let mut leaves = Vec::with_capacity(overlay.num_leaf_nodes() * HASHSIZE);
+        let mut overlays = Vec::with_capacity(leaves_and_subtrees.len());
+        overlays.push(overlay);
 
         // Iterate through all of the leaves/subtrees, adding their root as a leaf node and then
         // concatenating their merkle trees.
         for t in leaves_and_subtrees {
             leaves.append(&mut t.root().ok_or_else(|| Error::NoBytesForRoot)?.to_vec());
-            cache.append(&mut t.into_merkle_tree());
+
+            let (mut bytes, _bools, mut t_overlays) = t.into_components();
+
+            cache.append(&mut bytes);
+            overlays.append(&mut t_overlays);
         }
 
         // Pad the leaves to an even power-of-two, using zeros.
@@ -190,10 +205,17 @@ impl TreeHashCache {
         Ok(Self {
             chunk_modified: vec![false; cache.len() / BYTES_PER_CHUNK],
             cache,
+            overlays,
+            chunk_index: 0,
+            overlay_index: 0,
         })
     }
 
-    pub fn from_bytes(bytes: Vec<u8>, initial_modified_state: bool) -> Result<Self, Error> {
+    pub fn from_bytes(
+        bytes: Vec<u8>,
+        initial_modified_state: bool,
+        overlay: BTreeOverlay,
+    ) -> Result<Self, Error> {
         if bytes.len() % BYTES_PER_CHUNK > 0 {
             return Err(Error::BytesAreNotEvenChunks(bytes.len()));
         }
@@ -201,9 +223,84 @@ impl TreeHashCache {
         Ok(Self {
             chunk_modified: vec![initial_modified_state; bytes.len() / BYTES_PER_CHUNK],
             cache: bytes,
+            overlays: vec![overlay],
+            chunk_index: 0,
+            overlay_index: 0,
         })
     }
 
+    pub fn get_overlay(
+        &self,
+        overlay_index: usize,
+        chunk_index: usize,
+    ) -> Result<BTreeOverlay, Error> {
+        let mut overlay = self
+            .overlays
+            .get(overlay_index)
+            .ok_or_else(|| Error::NoOverlayForIndex(overlay_index))?
+            .clone();
+
+        overlay.offset = chunk_index;
+
+        Ok(overlay)
+    }
+
+    pub fn replace_overlay(
+        &mut self,
+        overlay_index: usize,
+        new_overlay: BTreeOverlay,
+    ) -> Result<BTreeOverlay, Error> {
+        let old_overlay = self
+            .overlays
+            .get(overlay_index)
+            .ok_or_else(|| Error::NoOverlayForIndex(overlay_index))?;
+
+        // Get slices of the exsiting tree from the cache.
+        let (old_bytes, old_flags) = self
+            .slices(old_overlay.chunk_range())
+            .ok_or_else(|| Error::UnableToObtainSlices)?;
+
+        let (new_bytes, new_bools) = if new_overlay.num_leaf_nodes() > old_overlay.num_leaf_nodes()
+        {
+            resize::grow_merkle_cache(
+                old_bytes,
+                old_flags,
+                old_overlay.height(),
+                new_overlay.height(),
+            )
+            .ok_or_else(|| Error::UnableToGrowMerkleTree)?
+        } else {
+            resize::shrink_merkle_cache(
+                old_bytes,
+                old_flags,
+                old_overlay.height(),
+                new_overlay.height(),
+                new_overlay.total_chunks(),
+            )
+            .ok_or_else(|| Error::UnableToShrinkMerkleTree)?
+        };
+
+        // Splice the newly created `TreeHashCache` over the existing elements.
+        self.splice(old_overlay.chunk_range(), new_bytes, new_bools);
+
+        Ok(std::mem::replace(
+            &mut self.overlays[overlay_index],
+            new_overlay,
+        ))
+    }
+
+    pub fn update_internal_nodes(&mut self, overlay: &BTreeOverlay) -> Result<(), Error> {
+        for (parent, children) in overlay.internal_parents_and_children().into_iter().rev() {
+            dbg!(parent);
+            dbg!(&children);
+            if self.either_modified(children)? {
+                self.modify_chunk(parent, &self.hash_children(children)?)?;
+            }
+        }
+
+        Ok(())
+    }
+
     pub fn bytes_len(&self) -> usize {
         self.cache.len()
     }
@@ -212,9 +309,7 @@ impl TreeHashCache {
         self.cache.get(0..HASHSIZE)
     }
 
-    pub fn splice(&mut self, chunk_range: Range<usize>, replace_with: Self) {
-        let (bytes, bools) = replace_with.into_components();
-
+    pub fn splice(&mut self, chunk_range: Range<usize>, bytes: Vec<u8>, bools: Vec<bool>) {
         // Update the `chunk_modified` vec, marking all spliced-in nodes as changed.
         self.chunk_modified.splice(chunk_range.clone(), bools);
         self.cache
@@ -278,14 +373,14 @@ impl TreeHashCache {
             .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))
     }
 
-    pub fn either_modified(&self, children: (&usize, &usize)) -> Result<bool, Error> {
-        Ok(self.changed(*children.0)? | self.changed(*children.1)?)
+    pub fn either_modified(&self, children: (usize, usize)) -> Result<bool, Error> {
+        Ok(self.changed(children.0)? | self.changed(children.1)?)
     }
 
-    pub fn hash_children(&self, children: (&usize, &usize)) -> Result<Vec<u8>, Error> {
+    pub fn hash_children(&self, children: (usize, usize)) -> Result<Vec<u8>, Error> {
         let mut child_bytes = Vec::with_capacity(BYTES_PER_CHUNK * 2);
-        child_bytes.append(&mut self.get_chunk(*children.0)?.to_vec());
-        child_bytes.append(&mut self.get_chunk(*children.1)?.to_vec());
+        child_bytes.append(&mut self.get_chunk(children.0)?.to_vec());
+        child_bytes.append(&mut self.get_chunk(children.1)?.to_vec());
 
         Ok(hash(&child_bytes))
     }
@@ -299,11 +394,7 @@ impl TreeHashCache {
         Ok(hash(&bytes))
     }
 
-    pub fn into_merkle_tree(self) -> Vec<u8> {
-        self.cache
-    }
-
-    pub fn into_components(self) -> (Vec<u8>, Vec<bool>) {
-        (self.cache, self.chunk_modified)
+    pub fn into_components(self) -> (Vec<u8>, Vec<bool>, Vec<BTreeOverlay>) {
+        (self.cache, self.chunk_modified, self.overlays)
     }
 }
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs b/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
index e8c04a91e..58c844867 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
@@ -1,12 +1,9 @@
 use super::*;
 
-#[derive(Debug)]
+#[derive(Debug, PartialEq, Clone)]
 pub struct BTreeOverlay {
-    pub num_internal_nodes: usize,
-    pub num_leaf_nodes: usize,
-    pub first_node: usize,
-    pub next_node: usize,
-    offsets: Vec<usize>,
+    pub offset: usize,
+    lengths: Vec<usize>,
 }
 
 impl BTreeOverlay {
@@ -17,84 +14,87 @@ impl BTreeOverlay {
         item.tree_hash_cache_overlay(initial_offset)
     }
 
-    pub fn from_lengths(offset: usize, mut lengths: Vec<usize>) -> Result<Self, Error> {
-        // Extend it to the next power-of-two, if it is not already.
-        let num_leaf_nodes = if lengths.len().is_power_of_two() {
-            lengths.len()
+    pub fn from_lengths(offset: usize, lengths: Vec<usize>) -> Result<Self, Error> {
+        if lengths.is_empty() {
+            Err(Error::TreeCannotHaveZeroNodes)
         } else {
-            let num_leaf_nodes = lengths.len().next_power_of_two();
-            lengths.resize(num_leaf_nodes, 1);
-            num_leaf_nodes
-        };
-
-        let num_nodes = num_nodes(num_leaf_nodes);
-        let num_internal_nodes = num_nodes - num_leaf_nodes;
-
-        let mut offsets = Vec::with_capacity(num_nodes);
-        offsets.append(&mut (offset..offset + num_internal_nodes).collect());
-
-        let mut next_node = num_internal_nodes + offset;
-        for i in 0..num_leaf_nodes {
-            offsets.push(next_node);
-            next_node += lengths[i];
+            Ok(Self { offset, lengths })
         }
+    }
 
-        Ok(Self {
-            num_internal_nodes,
-            num_leaf_nodes,
-            offsets,
-            first_node: offset,
-            next_node,
-        })
+    pub fn num_leaf_nodes(&self) -> usize {
+        self.lengths.len().next_power_of_two()
+    }
+
+    fn num_padding_leaves(&self) -> usize {
+        self.num_leaf_nodes() - self.lengths.len()
+    }
+
+    pub fn num_nodes(&self) -> usize {
+        2 * self.num_leaf_nodes() - 1
+    }
+
+    pub fn num_internal_nodes(&self) -> usize {
+        self.num_leaf_nodes() - 1
+    }
+
+    fn first_node(&self) -> usize {
+        self.offset
     }
 
     pub fn root(&self) -> usize {
-        self.first_node
+        self.first_node()
+    }
+
+    pub fn next_node(&self) -> usize {
+        self.first_node() + self.lengths.iter().sum::<usize>()
     }
 
     pub fn height(&self) -> usize {
-        self.num_leaf_nodes.trailing_zeros() as usize
+        self.num_leaf_nodes().trailing_zeros() as usize
     }
 
     pub fn chunk_range(&self) -> Range<usize> {
-        self.first_node..self.next_node
+        self.first_node()..self.next_node()
     }
 
     pub fn total_chunks(&self) -> usize {
-        self.next_node - self.first_node
+        self.next_node() - self.first_node()
     }
 
-    pub fn total_nodes(&self) -> usize {
-        self.num_internal_nodes + self.num_leaf_nodes
+    pub fn first_leaf_node(&self) -> usize {
+        self.offset + self.num_internal_nodes()
     }
 
-    pub fn first_leaf_node(&self) -> Result<usize, Error> {
-        self.offsets
-            .get(self.num_internal_nodes)
-            .cloned()
-            .ok_or_else(|| Error::NoFirstNode)
+    pub fn get_leaf_node(&self, i: usize) -> Result<Option<Range<usize>>, Error> {
+        if i >= self.num_leaf_nodes() {
+            return Err(Error::NotLeafNode(i));
+        } else if i >= self.num_leaf_nodes() - self.num_padding_leaves() {
+            Ok(None)
+        } else {
+            let first_node = self.offset + self.lengths.iter().take(i).sum::<usize>();
+            let last_node = first_node + self.lengths[i];
+            Ok(Some(first_node..last_node))
+        }
     }
 
     /// Returns an iterator visiting each internal node, providing the left and right child chunks
     /// for the node.
-    pub fn iter_internal_nodes<'a>(
-        &'a self,
-    ) -> impl DoubleEndedIterator<Item = (&'a usize, (&'a usize, &'a usize))> {
-        let internal_nodes = &self.offsets[0..self.num_internal_nodes];
-
-        internal_nodes.iter().enumerate().map(move |(i, parent)| {
-            let children = children(i);
-            (
-                parent,
-                (&self.offsets[children.0], &self.offsets[children.1]),
-            )
-        })
+    pub fn internal_parents_and_children(&self) -> Vec<(usize, (usize, usize))> {
+        (0..self.num_internal_nodes())
+            .into_iter()
+            .map(|parent| {
+                let children = children(parent);
+                (
+                    parent + self.offset,
+                    (children.0 + self.offset, children.1 + self.offset),
+                )
+            })
+            .collect()
     }
 
-    /// Returns an iterator visiting each leaf node, providing the chunk for that node.
-    pub fn iter_leaf_nodes<'a>(&'a self) -> impl DoubleEndedIterator<Item = &'a usize> {
-        let leaf_nodes = &self.offsets[self.num_internal_nodes..];
-
-        leaf_nodes.iter()
+    // Returns a `Vec` of chunk indices for each internal node of the tree.
+    pub fn internal_node_chunks(&self) -> Vec<usize> {
+        (self.offset..self.offset + self.num_internal_nodes()).collect()
     }
 }
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
index 6500e4eff..4a05a4046 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
@@ -8,6 +8,7 @@ impl CachedTreeHashSubTree<u64> for u64 {
         Ok(TreeHashCache::from_bytes(
             merkleize(self.to_le_bytes().to_vec()),
             false,
+            self.tree_hash_cache_overlay(0)?,
         )?)
     }
 
@@ -15,17 +16,13 @@ impl CachedTreeHashSubTree<u64> for u64 {
         BTreeOverlay::from_lengths(chunk_offset, vec![1])
     }
 
-    fn update_tree_hash_cache(
-        &self,
-        other: &Self,
-        cache: &mut TreeHashCache,
-        chunk: usize,
-    ) -> Result<usize, Error> {
-        if self != other {
-            let leaf = merkleize(self.to_le_bytes().to_vec());
-            cache.modify_chunk(chunk, &leaf)?;
-        }
+    fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
+        let leaf = merkleize(self.to_le_bytes().to_vec());
+        cache.maybe_update_chunk(cache.chunk_index, &leaf)?;
 
-        Ok(chunk + 1)
+        cache.chunk_index += 1;
+        cache.overlay_index += 1;
+
+        Ok(())
     }
 }
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
index 1cd7eb902..df49d7a97 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
@@ -5,10 +5,14 @@ where
     T: CachedTreeHashSubTree<T> + TreeHash,
 {
     fn new_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
-        match T::tree_hash_type() {
-            TreeHashType::Basic => {
-                TreeHashCache::from_bytes(merkleize(get_packed_leaves(self)?), false)
-            }
+        let overlay = self.tree_hash_cache_overlay(0)?;
+
+        let mut cache = match T::tree_hash_type() {
+            TreeHashType::Basic => TreeHashCache::from_bytes(
+                merkleize(get_packed_leaves(self)?),
+                false,
+                overlay.clone(),
+            ),
             TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
                 let subtrees = self
                     .iter()
@@ -17,17 +21,29 @@ where
 
                 TreeHashCache::from_leaves_and_subtrees(self, subtrees)
             }
-        }
+        }?;
+
+        // Mix in the length of the list.
+        let root_node = overlay.root();
+        cache.modify_chunk(root_node, &cache.mix_in_length(root_node, self.len())?)?;
+
+        Ok(cache)
     }
 
     fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
         let lengths = match T::tree_hash_type() {
-            TreeHashType::Basic => vec![1; self.len() / T::tree_hash_packing_factor()],
+            TreeHashType::Basic => {
+                // Ceil division.
+                let num_leaves = (self.len() + T::tree_hash_packing_factor() - 1)
+                    / T::tree_hash_packing_factor();
+
+                vec![1; num_leaves]
+            }
             TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
                 let mut lengths = vec![];
 
                 for item in self {
-                    lengths.push(BTreeOverlay::new(item, 0)?.total_nodes())
+                    lengths.push(BTreeOverlay::new(item, 0)?.num_nodes())
                 }
 
                 lengths
@@ -37,120 +53,93 @@ where
         BTreeOverlay::from_lengths(chunk_offset, lengths)
     }
 
-    fn update_tree_hash_cache(
-        &self,
-        other: &Vec<T>,
-        cache: &mut TreeHashCache,
-        chunk: usize,
-    ) -> Result<usize, Error> {
-        let offset_handler = BTreeOverlay::new(self, chunk)?;
-        let old_offset_handler = BTreeOverlay::new(other, chunk)?;
+    fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
+        let new_overlay = BTreeOverlay::new(self, cache.chunk_index)?;
+        let old_overlay = cache
+            .get_overlay(cache.overlay_index, cache.chunk_index)?
+            .clone();
 
-        if offset_handler.num_leaf_nodes != old_offset_handler.num_leaf_nodes {
-            let old_offset_handler = BTreeOverlay::new(other, chunk)?;
-
-            // Get slices of the exsiting tree from the cache.
-            let (old_bytes, old_flags) = cache
-                .slices(old_offset_handler.chunk_range())
-                .ok_or_else(|| Error::UnableToObtainSlices)?;
-
-            let (new_bytes, new_flags) =
-                if offset_handler.num_leaf_nodes > old_offset_handler.num_leaf_nodes {
-                    grow_merkle_cache(
-                        old_bytes,
-                        old_flags,
-                        old_offset_handler.height(),
-                        offset_handler.height(),
-                    )
-                    .ok_or_else(|| Error::UnableToGrowMerkleTree)?
-                } else {
-                    shrink_merkle_cache(
-                        old_bytes,
-                        old_flags,
-                        old_offset_handler.height(),
-                        offset_handler.height(),
-                        offset_handler.total_chunks(),
-                    )
-                    .ok_or_else(|| Error::UnableToShrinkMerkleTree)?
-                };
-
-            // Create a `TreeHashCache` from the raw elements.
-            let modified_cache = TreeHashCache::from_elems(new_bytes, new_flags);
-
-            // Splice the newly created `TreeHashCache` over the existing elements.
-            cache.splice(old_offset_handler.chunk_range(), modified_cache);
+        // If the merkle tree required to represent the new list is of a different size to the one
+        // required for the previous list, then update our cache.
+        //
+        // This grows/shrinks the bytes to accomodate the new tree, preserving as much of the tree
+        // as possible.
+        if new_overlay.num_leaf_nodes() != old_overlay.num_leaf_nodes() {
+            cache.replace_overlay(cache.overlay_index, new_overlay.clone())?;
         }
 
         match T::tree_hash_type() {
             TreeHashType::Basic => {
-                let leaves = get_packed_leaves(self)?;
+                let mut buf = vec![0; HASHSIZE];
+                let item_bytes = HASHSIZE / T::tree_hash_packing_factor();
 
-                for (i, chunk) in offset_handler.iter_leaf_nodes().enumerate() {
-                    if let Some(latest) = leaves.get(i * HASHSIZE..(i + 1) * HASHSIZE) {
-                        cache.maybe_update_chunk(*chunk, latest)?;
+                // Iterate through each of the leaf nodes.
+                for i in 0..new_overlay.num_leaf_nodes() {
+                    // Iterate through the number of items that may be packing into the leaf node.
+                    for j in 0..T::tree_hash_packing_factor() {
+                        // Create a mut slice that can either be filled with a serialized item or
+                        // padding.
+                        let buf_slice = &mut buf[j * item_bytes..(j + 1) * item_bytes];
+
+                        // Attempt to get the item for this portion of the chunk. If it exists,
+                        // update `buf` with it's serialized bytes. If it doesn't exist, update
+                        // `buf` with padding.
+                        match self.get(i * T::tree_hash_packing_factor() + j) {
+                            Some(item) => {
+                                buf_slice.copy_from_slice(&item.tree_hash_packed_encoding());
+                            }
+                            None => buf_slice.copy_from_slice(&vec![0; item_bytes]),
+                        }
                     }
-                }
-                let first_leaf_chunk = offset_handler.first_leaf_node()?;
 
-                cache.splice(
-                    first_leaf_chunk..offset_handler.next_node,
-                    TreeHashCache::from_bytes(leaves, true)?,
-                );
+                    // Update the chunk if the generated `buf` is not the same as the cache.
+                    let chunk = new_overlay.first_leaf_node() + i;
+                    cache.maybe_update_chunk(chunk, &buf)?;
+                }
             }
             TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
-                let mut i = offset_handler.num_leaf_nodes;
-                for &start_chunk in offset_handler.iter_leaf_nodes().rev() {
-                    i -= 1;
-                    match (other.get(i), self.get(i)) {
-                        // The item existed in the previous list and exsits in the current list.
-                        (Some(old), Some(new)) => {
-                            new.update_tree_hash_cache(old, cache, start_chunk)?;
+                for i in (0..new_overlay.num_leaf_nodes()).rev() {
+                    match (old_overlay.get_leaf_node(i)?, new_overlay.get_leaf_node(i)?) {
+                        // The item existed in the previous list and exists in the current list.
+                        (Some(_old), Some(new)) => {
+                            cache.chunk_index = new.start;
+                            self[i].update_tree_hash_cache(cache)?;
                         }
                         // The item existed in the previous list but does not exist in this list.
                         //
-                        // I.e., the list has been shortened.
+                        // Viz., the list has been shortened.
                         (Some(old), None) => {
                             // Splice out the entire tree of the removed node, replacing it with a
                             // single padding node.
-                            let end_chunk = BTreeOverlay::new(old, start_chunk)?.next_node;
-
-                            cache.splice(
-                                start_chunk..end_chunk,
-                                TreeHashCache::from_bytes(vec![0; HASHSIZE], true)?,
-                            );
+                            cache.splice(old, vec![0; HASHSIZE], vec![true]);
                         }
-                        // The item existed in the previous list but does exist in this list.
+                        // The item did not exist in the previous list but does exist in this list.
                         //
-                        // I.e., the list has been lengthened.
+                        // Viz., the list has been lengthened.
                         (None, Some(new)) => {
-                            let bytes: Vec<u8> = TreeHashCache::new(new)?.into();
+                            let bytes: Vec<u8> = TreeHashCache::new(&self[i])?.into();
+                            let bools = vec![true; bytes.len() / HASHSIZE];
 
-                            cache.splice(
-                                start_chunk..start_chunk + 1,
-                                TreeHashCache::from_bytes(bytes, true)?,
-                            );
+                            cache.splice(new.start..new.start + 1, bytes, bools);
                         }
                         // The item didn't exist in the old list and doesn't exist in the new list,
                         // nothing to do.
                         (None, None) => {}
-                    };
+                    }
                 }
             }
         }
 
-        for (&parent, children) in offset_handler.iter_internal_nodes().rev() {
-            if cache.either_modified(children)? {
-                cache.modify_chunk(parent, &cache.hash_children(children)?)?;
-            }
-        }
+        cache.update_internal_nodes(&new_overlay)?;
 
-        // If the root node or the length has changed, mix in the length of the list.
-        let root_node = offset_handler.root();
-        if cache.changed(root_node)? | (self.len() != other.len()) {
-            cache.modify_chunk(root_node, &cache.mix_in_length(root_node, self.len())?)?;
-        }
+        // Always update the root node as we don't have a reliable check to know if the list len
+        // has changed.
+        let root_node = new_overlay.root();
+        cache.modify_chunk(root_node, &cache.mix_in_length(root_node, self.len())?)?;
 
-        Ok(offset_handler.next_node)
+        cache.chunk_index = new_overlay.next_node();
+
+        Ok(())
     }
 }
 
diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
index 4d2c6f282..c3bd9a7b9 100644
--- a/eth2/utils/tree_hash/tests/tests.rs
+++ b/eth2/utils/tree_hash/tests/tests.rs
@@ -1,9 +1,275 @@
-use hashing::hash;
-use int_to_bytes::{int_to_bytes32, int_to_bytes8};
+use int_to_bytes::int_to_bytes32;
 use tree_hash::cached_tree_hash::*;
 use tree_hash::standard_tree_hash::*;
 use tree_hash::*;
+use tree_hash_derive::{CachedTreeHashSubTree, TreeHash};
 
+#[derive(Clone, Debug, TreeHash, CachedTreeHashSubTree)]
+pub struct Nested {
+    pub a: u64,
+    pub b: Inner,
+}
+
+#[derive(Clone, Debug, TreeHash, CachedTreeHashSubTree)]
+pub struct Thing {
+    pub a: u64,
+    pub b: Inner,
+    pub c: Vec<u64>,
+}
+
+fn test_routine<T>(original: T, modified: Vec<T>)
+where
+    T: CachedTreeHashSubTree<T>,
+{
+    let mut cache = original.new_tree_hash_cache().unwrap();
+
+    let standard_root = original.tree_hash_root();
+    let cached_root = cache.root().unwrap().to_vec();
+    assert_eq!(standard_root, cached_root, "Initial cache build failed.");
+
+    for (i, modified) in modified.iter().enumerate() {
+        // Test after a modification
+        modified.update_tree_hash_cache(&mut cache).unwrap();
+        let standard_root = modified.tree_hash_root();
+        let cached_root = cache.root().unwrap().to_vec();
+        assert_eq!(standard_root, cached_root, "Modification {} failed.", i);
+    }
+}
+
+#[test]
+fn test_nested() {
+    let original = Nested {
+        a: 42,
+        b: Inner {
+            a: 12,
+            b: 13,
+            c: 14,
+            d: 15,
+        },
+    };
+    let modified = vec![Nested {
+        a: 99,
+        ..original.clone()
+    }];
+
+    test_routine(original, modified);
+}
+
+#[test]
+fn test_inner() {
+    let original = Inner {
+        a: 12,
+        b: 13,
+        c: 14,
+        d: 15,
+    };
+
+    let modified = vec![Inner {
+        a: 99,
+        ..original.clone()
+    }];
+
+    test_routine(original, modified);
+}
+
+#[test]
+fn test_thing() {
+    let original = Thing {
+        a: 42,
+        b: Inner {
+            a: 12,
+            b: 13,
+            c: 14,
+            d: 15,
+        },
+        c: vec![1, 2, 3, 4, 5],
+    };
+
+    let modified = vec![Thing {
+        a: 99,
+        ..original.clone()
+    }];
+
+    test_routine(original, modified);
+}
+
+#[test]
+fn test_vec() {
+    let original = vec![1, 2, 3, 4, 5];
+
+    let modified = vec![vec![1, 2, 3, 4, 42]];
+
+    test_routine(original, modified);
+}
+
+#[derive(Clone, Debug)]
+pub struct Inner {
+    pub a: u64,
+    pub b: u64,
+    pub c: u64,
+    pub d: u64,
+}
+
+impl TreeHash for Inner {
+    fn tree_hash_type() -> TreeHashType {
+        TreeHashType::Container
+    }
+
+    fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+        unreachable!("Struct should never be packed.")
+    }
+
+    fn tree_hash_packing_factor() -> usize {
+        unreachable!("Struct should never be packed.")
+    }
+
+    fn tree_hash_root(&self) -> Vec<u8> {
+        let mut leaves = Vec::with_capacity(4 * HASHSIZE);
+
+        leaves.append(&mut self.a.tree_hash_root());
+        leaves.append(&mut self.b.tree_hash_root());
+        leaves.append(&mut self.c.tree_hash_root());
+        leaves.append(&mut self.d.tree_hash_root());
+
+        efficient_merkleize(&leaves)[0..32].to_vec()
+    }
+}
+
+impl CachedTreeHashSubTree<Inner> for Inner {
+    fn new_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
+        let tree = TreeHashCache::from_leaves_and_subtrees(
+            self,
+            vec![
+                self.a.new_tree_hash_cache()?,
+                self.b.new_tree_hash_cache()?,
+                self.c.new_tree_hash_cache()?,
+                self.d.new_tree_hash_cache()?,
+            ],
+        )?;
+
+        Ok(tree)
+    }
+
+    fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
+        let mut lengths = vec![];
+
+        lengths.push(BTreeOverlay::new(&self.a, 0)?.num_nodes());
+        lengths.push(BTreeOverlay::new(&self.b, 0)?.num_nodes());
+        lengths.push(BTreeOverlay::new(&self.c, 0)?.num_nodes());
+        lengths.push(BTreeOverlay::new(&self.d, 0)?.num_nodes());
+
+        BTreeOverlay::from_lengths(chunk_offset, lengths)
+    }
+
+    fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
+        let overlay = cache.get_overlay(cache.overlay_index, cache.chunk_index)?;
+        dbg!(&overlay);
+
+        // Skip the chunk index to the first leaf node of this struct.
+        cache.chunk_index = overlay.first_leaf_node();
+        // Skip the overlay index to the first leaf node of this struct.
+        cache.overlay_index += 1;
+
+        // Recurse into the struct items, updating their caches.
+        self.a.update_tree_hash_cache(cache)?;
+        self.b.update_tree_hash_cache(cache)?;
+        self.c.update_tree_hash_cache(cache)?;
+        self.d.update_tree_hash_cache(cache)?;
+
+        // Iterate through the internal nodes, updating them if their children have changed.
+        cache.update_internal_nodes(&overlay)?;
+
+        Ok(())
+    }
+}
+
+fn generic_test(index: usize) {
+    let inner = Inner {
+        a: 1,
+        b: 2,
+        c: 3,
+        d: 4,
+    };
+
+    let mut cache = TreeHashCache::new(&inner).unwrap();
+
+    let changed_inner = match index {
+        0 => Inner {
+            a: 42,
+            ..inner.clone()
+        },
+        1 => Inner {
+            b: 42,
+            ..inner.clone()
+        },
+        2 => Inner {
+            c: 42,
+            ..inner.clone()
+        },
+        3 => Inner {
+            d: 42,
+            ..inner.clone()
+        },
+        _ => panic!("bad index"),
+    };
+
+    changed_inner.update_tree_hash_cache(&mut cache).unwrap();
+
+    let data1 = int_to_bytes32(1);
+    let data2 = int_to_bytes32(2);
+    let data3 = int_to_bytes32(3);
+    let data4 = int_to_bytes32(4);
+
+    let mut data = vec![data1, data2, data3, data4];
+
+    data[index] = int_to_bytes32(42);
+
+    let expected = merkleize(join(data));
+
+    let cache_bytes: Vec<u8> = cache.into();
+
+    assert_eq!(expected, cache_bytes);
+}
+
+#[test]
+fn cached_hash_on_inner() {
+    generic_test(0);
+    generic_test(1);
+    generic_test(2);
+    generic_test(3);
+}
+
+#[test]
+fn inner_builds() {
+    let data1 = int_to_bytes32(1);
+    let data2 = int_to_bytes32(2);
+    let data3 = int_to_bytes32(3);
+    let data4 = int_to_bytes32(4);
+
+    let data = join(vec![data1, data2, data3, data4]);
+    let expected = merkleize(data);
+
+    let inner = Inner {
+        a: 1,
+        b: 2,
+        c: 3,
+        d: 4,
+    };
+
+    let cache: Vec<u8> = TreeHashCache::new(&inner).unwrap().into();
+
+    assert_eq!(expected, cache);
+}
+
+fn join(many: Vec<Vec<u8>>) -> Vec<u8> {
+    let mut all = vec![];
+    for one in many {
+        all.extend_from_slice(&mut one.clone())
+    }
+    all
+}
+
+/*
 #[derive(Clone, Debug)]
 pub struct InternalCache {
     pub a: u64,
@@ -101,8 +367,8 @@ impl CachedTreeHashSubTree<InternalCache> for InternalCache {
     fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
         let mut lengths = vec![];
 
-        lengths.push(BTreeOverlay::new(&self.a, 0)?.total_nodes());
-        lengths.push(BTreeOverlay::new(&self.b, 0)?.total_nodes());
+        lengths.push(BTreeOverlay::new(&self.a, 0)?.num_nodes());
+        lengths.push(BTreeOverlay::new(&self.b, 0)?.num_nodes());
 
         BTreeOverlay::from_lengths(chunk_offset, lengths)
     }
@@ -187,10 +453,10 @@ impl CachedTreeHashSubTree<Inner> for Inner {
     fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
         let mut lengths = vec![];
 
-        lengths.push(BTreeOverlay::new(&self.a, 0)?.total_nodes());
-        lengths.push(BTreeOverlay::new(&self.b, 0)?.total_nodes());
-        lengths.push(BTreeOverlay::new(&self.c, 0)?.total_nodes());
-        lengths.push(BTreeOverlay::new(&self.d, 0)?.total_nodes());
+        lengths.push(BTreeOverlay::new(&self.a, 0)?.num_nodes());
+        lengths.push(BTreeOverlay::new(&self.b, 0)?.num_nodes());
+        lengths.push(BTreeOverlay::new(&self.c, 0)?.num_nodes());
+        lengths.push(BTreeOverlay::new(&self.d, 0)?.num_nodes());
 
         BTreeOverlay::from_lengths(chunk_offset, lengths)
     }
@@ -270,9 +536,9 @@ impl CachedTreeHashSubTree<Outer> for Outer {
     fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
         let mut lengths = vec![];
 
-        lengths.push(BTreeOverlay::new(&self.a, 0)?.total_nodes());
-        lengths.push(BTreeOverlay::new(&self.b, 0)?.total_nodes());
-        lengths.push(BTreeOverlay::new(&self.c, 0)?.total_nodes());
+        lengths.push(BTreeOverlay::new(&self.a, 0)?.num_nodes());
+        lengths.push(BTreeOverlay::new(&self.b, 0)?.num_nodes());
+        lengths.push(BTreeOverlay::new(&self.c, 0)?.num_nodes());
 
         BTreeOverlay::from_lengths(chunk_offset, lengths)
     }
@@ -1078,3 +1344,4 @@ fn merkleize_4_leaves() {
         assert_eq!(chunk, &expected[..], "failed at {}", i);
     }
 }
+*/
diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs
index 343287313..b308953a3 100644
--- a/eth2/utils/tree_hash_derive/src/lib.rs
+++ b/eth2/utils/tree_hash_derive/src/lib.rs
@@ -54,7 +54,6 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
     let idents_a = get_hashable_named_field_idents(&struct_data);
     let idents_b = idents_a.clone();
     let idents_c = idents_a.clone();
-    let idents_d = idents_a.clone();
 
     let output = quote! {
         impl tree_hash::CachedTreeHashSubTree<#name> for #name {
@@ -75,35 +74,29 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
                 let mut lengths = vec![];
 
                 #(
-                    lengths.push(tree_hash::BTreeOverlay::new(&self.#idents_b, 0)?.total_nodes());
+                    lengths.push(tree_hash::BTreeOverlay::new(&self.#idents_b, 0)?.num_nodes());
                 )*
 
                 tree_hash::BTreeOverlay::from_lengths(chunk_offset, lengths)
             }
 
-            fn update_tree_hash_cache(
-                &self,
-                other: &Self,
-                cache: &mut tree_hash::TreeHashCache,
-                chunk: usize,
-            ) -> Result<usize, tree_hash::Error> {
-                let offset_handler = tree_hash::BTreeOverlay::new(self, chunk)?;
+            fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
+                let overlay = cache.get_overlay(cache.overlay_index, cache.chunk_index)?;
 
-                // Skip past the internal nodes and update any changed leaf nodes.
-                {
-                    let chunk = offset_handler.first_leaf_node()?;
-                    #(
-                        let chunk = self.#idents_c.update_tree_hash_cache(&other.#idents_d, cache, chunk)?;
-                    )*
-                }
+                // Skip the chunk index to the first leaf node of this struct.
+                cache.chunk_index = overlay.first_leaf_node();
+                // Skip the overlay index to the first leaf node of this struct.
+                cache.overlay_index += 1;
 
-                for (&parent, children) in offset_handler.iter_internal_nodes().rev() {
-                    if cache.either_modified(children)? {
-                        cache.modify_chunk(parent, &cache.hash_children(children)?)?;
-                    }
-                }
+                // Recurse into the struct items, updating their caches.
+                #(
+                    self.#idents_c.update_tree_hash_cache(cache)?;
+                )*
 
-                Ok(offset_handler.next_node)
+                // Iterate through the internal nodes, updating them if their children have changed.
+                cache.update_internal_nodes(&overlay)?;
+
+                Ok(())
             }
         }
     };

From 2c12aabf04f9411d0828296b4cae42ed476fc668 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Mon, 22 Apr 2019 09:20:13 +1000
Subject: [PATCH 090/137] Implement further cache tests and bug fixes

---
 eth2/utils/tree_hash/src/cached_tree_hash.rs  |  90 +++++++-----
 .../src/cached_tree_hash/btree_overlay.rs     | 129 ++++++++++++++++--
 .../tree_hash/src/cached_tree_hash/impls.rs   |   3 +-
 .../src/cached_tree_hash/impls/vec.rs         |  23 +++-
 eth2/utils/tree_hash/tests/tests.rs           |  98 ++++++++++---
 eth2/utils/tree_hash_derive/src/lib.rs        |   6 +-
 6 files changed, 278 insertions(+), 71 deletions(-)

diff --git a/eth2/utils/tree_hash/src/cached_tree_hash.rs b/eth2/utils/tree_hash/src/cached_tree_hash.rs
index c2011fdf8..35ddf10cd 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash.rs
@@ -9,6 +9,44 @@ pub mod resize;
 
 pub use btree_overlay::BTreeOverlay;
 
+#[derive(Debug, PartialEq)]
+pub struct CachedTreeHasher {
+    cache: TreeHashCache,
+}
+
+impl CachedTreeHasher {
+    pub fn new<T>(item: &T) -> Result<Self, Error>
+    where
+        T: CachedTreeHashSubTree<T>,
+    {
+        Ok(Self {
+            cache: TreeHashCache::new(item)?,
+        })
+    }
+
+    pub fn update<T>(&mut self, item: &T) -> Result<(), Error>
+    where
+        T: CachedTreeHashSubTree<T>,
+    {
+        // Reset the per-hash counters.
+        self.cache.chunk_index = 0;
+        self.cache.overlay_index = 0;
+
+        // Reset the "modified" flags for the cache.
+        self.cache.reset_modifications();
+
+        // Update the cache with the (maybe) changed object.
+        item.update_tree_hash_cache(&mut self.cache)?;
+
+        Ok(())
+    }
+
+    pub fn tree_hash_root(&self) -> Result<Vec<u8>, Error> {
+        // Return the root of the cache -- the merkle root.
+        Ok(self.cache.root()?.to_vec())
+    }
+}
+
 #[derive(Debug, PartialEq, Clone)]
 pub enum Error {
     ShouldNotProduceBTreeOverlay,
@@ -141,20 +179,6 @@ impl TreeHashCache {
         item.new_tree_hash_cache()
     }
 
-    pub fn from_elems(
-        cache: Vec<u8>,
-        chunk_modified: Vec<bool>,
-        overlays: Vec<BTreeOverlay>,
-    ) -> Self {
-        Self {
-            cache,
-            chunk_modified,
-            overlays,
-            chunk_index: 0,
-            overlay_index: 0,
-        }
-    }
-
     pub fn from_leaves_and_subtrees<T>(
         item: &T,
         leaves_and_subtrees: Vec<Self>,
@@ -185,7 +209,7 @@ impl TreeHashCache {
         // Iterate through all of the leaves/subtrees, adding their root as a leaf node and then
         // concatenating their merkle trees.
         for t in leaves_and_subtrees {
-            leaves.append(&mut t.root().ok_or_else(|| Error::NoBytesForRoot)?.to_vec());
+            leaves.append(&mut t.root()?.to_vec());
 
             let (mut bytes, _bools, mut t_overlays) = t.into_components();
 
@@ -245,15 +269,19 @@ impl TreeHashCache {
         Ok(overlay)
     }
 
+    pub fn reset_modifications(&mut self) {
+        for chunk_modified in &mut self.chunk_modified {
+            *chunk_modified = false;
+        }
+    }
+
     pub fn replace_overlay(
         &mut self,
         overlay_index: usize,
+        chunk_index: usize,
         new_overlay: BTreeOverlay,
     ) -> Result<BTreeOverlay, Error> {
-        let old_overlay = self
-            .overlays
-            .get(overlay_index)
-            .ok_or_else(|| Error::NoOverlayForIndex(overlay_index))?;
+        let old_overlay = self.get_overlay(overlay_index, chunk_index)?;
 
         // Get slices of the exsiting tree from the cache.
         let (old_bytes, old_flags) = self
@@ -291,8 +319,6 @@ impl TreeHashCache {
 
     pub fn update_internal_nodes(&mut self, overlay: &BTreeOverlay) -> Result<(), Error> {
         for (parent, children) in overlay.internal_parents_and_children().into_iter().rev() {
-            dbg!(parent);
-            dbg!(&children);
             if self.either_modified(children)? {
                 self.modify_chunk(parent, &self.hash_children(children)?)?;
             }
@@ -301,15 +327,17 @@ impl TreeHashCache {
         Ok(())
     }
 
-    pub fn bytes_len(&self) -> usize {
+    fn bytes_len(&self) -> usize {
         self.cache.len()
     }
 
-    pub fn root(&self) -> Option<&[u8]> {
-        self.cache.get(0..HASHSIZE)
+    pub fn root(&self) -> Result<&[u8], Error> {
+        self.cache
+            .get(0..HASHSIZE)
+            .ok_or_else(|| Error::NoBytesForRoot)
     }
 
-    pub fn splice(&mut self, chunk_range: Range<usize>, bytes: Vec<u8>, bools: Vec<bool>) {
+    fn splice(&mut self, chunk_range: Range<usize>, bytes: Vec<u8>, bools: Vec<bool>) {
         // Update the `chunk_modified` vec, marking all spliced-in nodes as changed.
         self.chunk_modified.splice(chunk_range.clone(), bools);
         self.cache
@@ -331,14 +359,14 @@ impl TreeHashCache {
         Ok(())
     }
 
-    pub fn slices(&self, chunk_range: Range<usize>) -> Option<(&[u8], &[bool])> {
+    fn slices(&self, chunk_range: Range<usize>) -> Option<(&[u8], &[bool])> {
         Some((
             self.cache.get(node_range_to_byte_range(&chunk_range))?,
             self.chunk_modified.get(chunk_range)?,
         ))
     }
 
-    pub fn modify_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> {
+    fn modify_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> {
         let start = chunk * BYTES_PER_CHUNK;
         let end = start + BYTES_PER_CHUNK;
 
@@ -352,7 +380,7 @@ impl TreeHashCache {
         Ok(())
     }
 
-    pub fn get_chunk(&self, chunk: usize) -> Result<&[u8], Error> {
+    fn get_chunk(&self, chunk: usize) -> Result<&[u8], Error> {
         let start = chunk * BYTES_PER_CHUNK;
         let end = start + BYTES_PER_CHUNK;
 
@@ -362,7 +390,7 @@ impl TreeHashCache {
             .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))?)
     }
 
-    pub fn chunk_equals(&mut self, chunk: usize, other: &[u8]) -> Result<bool, Error> {
+    fn chunk_equals(&mut self, chunk: usize, other: &[u8]) -> Result<bool, Error> {
         Ok(self.get_chunk(chunk)? == other)
     }
 
@@ -373,11 +401,11 @@ impl TreeHashCache {
             .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))
     }
 
-    pub fn either_modified(&self, children: (usize, usize)) -> Result<bool, Error> {
+    fn either_modified(&self, children: (usize, usize)) -> Result<bool, Error> {
         Ok(self.changed(children.0)? | self.changed(children.1)?)
     }
 
-    pub fn hash_children(&self, children: (usize, usize)) -> Result<Vec<u8>, Error> {
+    fn hash_children(&self, children: (usize, usize)) -> Result<Vec<u8>, Error> {
         let mut child_bytes = Vec::with_capacity(BYTES_PER_CHUNK * 2);
         child_bytes.append(&mut self.get_chunk(children.0)?.to_vec());
         child_bytes.append(&mut self.get_chunk(children.1)?.to_vec());
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs b/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
index 58c844867..cfd58df8a 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
@@ -3,7 +3,8 @@ use super::*;
 #[derive(Debug, PartialEq, Clone)]
 pub struct BTreeOverlay {
     pub offset: usize,
-    lengths: Vec<usize>,
+    pub num_items: usize,
+    pub lengths: Vec<usize>,
 }
 
 impl BTreeOverlay {
@@ -14,11 +15,19 @@ impl BTreeOverlay {
         item.tree_hash_cache_overlay(initial_offset)
     }
 
-    pub fn from_lengths(offset: usize, lengths: Vec<usize>) -> Result<Self, Error> {
+    pub fn from_lengths(
+        offset: usize,
+        num_items: usize,
+        lengths: Vec<usize>,
+    ) -> Result<Self, Error> {
         if lengths.is_empty() {
             Err(Error::TreeCannotHaveZeroNodes)
         } else {
-            Ok(Self { offset, lengths })
+            Ok(Self {
+                offset,
+                num_items,
+                lengths,
+            })
         }
     }
 
@@ -47,7 +56,8 @@ impl BTreeOverlay {
     }
 
     pub fn next_node(&self) -> usize {
-        self.first_node() + self.lengths.iter().sum::<usize>()
+        self.first_node() + self.num_internal_nodes() + self.num_leaf_nodes() - self.lengths.len()
+            + self.lengths.iter().sum::<usize>()
     }
 
     pub fn height(&self) -> usize {
@@ -78,17 +88,28 @@ impl BTreeOverlay {
         }
     }
 
-    /// Returns an iterator visiting each internal node, providing the left and right child chunks
-    /// for the node.
+    pub fn child_chunks(&self, parent: usize) -> (usize, usize) {
+        let children = children(parent);
+
+        if children.1 < self.num_internal_nodes() {
+            (children.0 + self.offset, children.1 + self.offset)
+        } else {
+            let chunks = self.n_leaf_node_chunks(children.1);
+            (chunks[chunks.len() - 2], chunks[chunks.len() - 1])
+        }
+    }
+
+    /// (parent, (left_child, right_child))
     pub fn internal_parents_and_children(&self) -> Vec<(usize, (usize, usize))> {
+        let mut chunks = Vec::with_capacity(self.num_nodes());
+        chunks.append(&mut self.internal_node_chunks());
+        chunks.append(&mut self.leaf_node_chunks());
+
         (0..self.num_internal_nodes())
             .into_iter()
             .map(|parent| {
                 let children = children(parent);
-                (
-                    parent + self.offset,
-                    (children.0 + self.offset, children.1 + self.offset),
-                )
+                (chunks[parent], (chunks[children.0], chunks[children.1]))
             })
             .collect()
     }
@@ -97,4 +118,92 @@ impl BTreeOverlay {
     pub fn internal_node_chunks(&self) -> Vec<usize> {
         (self.offset..self.offset + self.num_internal_nodes()).collect()
     }
+
+    // Returns a `Vec` of the first chunk index for each leaf node of the tree.
+    pub fn leaf_node_chunks(&self) -> Vec<usize> {
+        self.n_leaf_node_chunks(self.num_leaf_nodes())
+    }
+
+    // Returns a `Vec` of the first chunk index for the first `n` leaf nodes of the tree.
+    fn n_leaf_node_chunks(&self, n: usize) -> Vec<usize> {
+        let mut chunks = Vec::with_capacity(n);
+
+        let mut chunk = self.offset + self.num_internal_nodes();
+        for i in 0..n {
+            chunks.push(chunk);
+
+            match self.lengths.get(i) {
+                Some(len) => {
+                    chunk += len;
+                }
+                None => chunk += 1,
+            }
+        }
+
+        chunks
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    fn get_tree_a(n: usize) -> BTreeOverlay {
+        BTreeOverlay::from_lengths(0, n, vec![1; n]).unwrap()
+    }
+
+    #[test]
+    fn leaf_node_chunks() {
+        let tree = get_tree_a(4);
+
+        assert_eq!(tree.leaf_node_chunks(), vec![3, 4, 5, 6])
+    }
+
+    #[test]
+    fn internal_node_chunks() {
+        let tree = get_tree_a(4);
+
+        assert_eq!(tree.internal_node_chunks(), vec![0, 1, 2])
+    }
+
+    #[test]
+    fn internal_parents_and_children() {
+        let tree = get_tree_a(4);
+
+        assert_eq!(
+            tree.internal_parents_and_children(),
+            vec![(0, (1, 2)), (1, (3, 4)), (2, (5, 6))]
+        )
+    }
+
+    #[test]
+    fn chunk_range() {
+        let tree = get_tree_a(4);
+        assert_eq!(tree.chunk_range(), 0..7);
+
+        let tree = get_tree_a(1);
+        assert_eq!(tree.chunk_range(), 0..1);
+
+        let tree = get_tree_a(2);
+        assert_eq!(tree.chunk_range(), 0..3);
+
+        let tree = BTreeOverlay::from_lengths(11, 4, vec![1, 1]).unwrap();
+        assert_eq!(tree.chunk_range(), 11..14);
+    }
+
+    #[test]
+    fn root_of_one_node() {
+        let tree = get_tree_a(1);
+
+        assert_eq!(tree.root(), 0);
+        assert_eq!(tree.num_internal_nodes(), 0);
+        assert_eq!(tree.num_leaf_nodes(), 1);
+    }
+
+    #[test]
+    fn child_chunks() {
+        let tree = get_tree_a(4);
+
+        assert_eq!(tree.child_chunks(0), (1, 2))
+    }
 }
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
index 4a05a4046..ece025ccd 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
@@ -1,4 +1,3 @@
-use super::resize::{grow_merkle_cache, shrink_merkle_cache};
 use super::*;
 
 mod vec;
@@ -13,7 +12,7 @@ impl CachedTreeHashSubTree<u64> for u64 {
     }
 
     fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
-        BTreeOverlay::from_lengths(chunk_offset, vec![1])
+        BTreeOverlay::from_lengths(chunk_offset, 1, vec![1])
     }
 
     fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
index df49d7a97..d71b58816 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
@@ -37,7 +37,8 @@ where
                 let num_leaves = (self.len() + T::tree_hash_packing_factor() - 1)
                     / T::tree_hash_packing_factor();
 
-                vec![1; num_leaves]
+                // Disallow zero-length as an empty list still has one all-padding node.
+                vec![1; std::cmp::max(1, num_leaves)]
             }
             TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
                 let mut lengths = vec![];
@@ -50,7 +51,7 @@ where
             }
         };
 
-        BTreeOverlay::from_lengths(chunk_offset, lengths)
+        BTreeOverlay::from_lengths(chunk_offset, self.len(), lengths)
     }
 
     fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
@@ -65,7 +66,7 @@ where
         // This grows/shrinks the bytes to accomodate the new tree, preserving as much of the tree
         // as possible.
         if new_overlay.num_leaf_nodes() != old_overlay.num_leaf_nodes() {
-            cache.replace_overlay(cache.overlay_index, new_overlay.clone())?;
+            cache.replace_overlay(cache.overlay_index, cache.chunk_index, new_overlay.clone())?;
         }
 
         match T::tree_hash_type() {
@@ -132,10 +133,20 @@ where
 
         cache.update_internal_nodes(&new_overlay)?;
 
-        // Always update the root node as we don't have a reliable check to know if the list len
-        // has changed.
+        // Mix in length.
         let root_node = new_overlay.root();
-        cache.modify_chunk(root_node, &cache.mix_in_length(root_node, self.len())?)?;
+        if cache.changed(root_node)? {
+            dbg!(cache.get_chunk(12));
+            cache.modify_chunk(root_node, &cache.mix_in_length(root_node, self.len())?)?;
+        } else if old_overlay.num_items != new_overlay.num_items {
+            if new_overlay.num_internal_nodes() == 0 {
+                cache.modify_chunk(root_node, &cache.mix_in_length(root_node, self.len())?)?;
+            } else {
+                let children = new_overlay.child_chunks(0);
+                cache.modify_chunk(root_node, &cache.hash_children(children)?)?;
+                cache.modify_chunk(root_node, &cache.mix_in_length(root_node, self.len())?)?;
+            }
+        }
 
         cache.chunk_index = new_overlay.next_node();
 
diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
index c3bd9a7b9..52f7a7cb1 100644
--- a/eth2/utils/tree_hash/tests/tests.rs
+++ b/eth2/utils/tree_hash/tests/tests.rs
@@ -5,13 +5,13 @@ use tree_hash::*;
 use tree_hash_derive::{CachedTreeHashSubTree, TreeHash};
 
 #[derive(Clone, Debug, TreeHash, CachedTreeHashSubTree)]
-pub struct Nested {
+pub struct NestedStruct {
     pub a: u64,
     pub b: Inner,
 }
 
 #[derive(Clone, Debug, TreeHash, CachedTreeHashSubTree)]
-pub struct Thing {
+pub struct StructWithVec {
     pub a: u64,
     pub b: Inner,
     pub c: Vec<u64>,
@@ -21,24 +21,32 @@ fn test_routine<T>(original: T, modified: Vec<T>)
 where
     T: CachedTreeHashSubTree<T>,
 {
-    let mut cache = original.new_tree_hash_cache().unwrap();
+    let mut hasher = CachedTreeHasher::new(&original).unwrap();
 
     let standard_root = original.tree_hash_root();
-    let cached_root = cache.root().unwrap().to_vec();
+    let cached_root = hasher.tree_hash_root().unwrap();
     assert_eq!(standard_root, cached_root, "Initial cache build failed.");
 
     for (i, modified) in modified.iter().enumerate() {
         // Test after a modification
-        modified.update_tree_hash_cache(&mut cache).unwrap();
+        hasher
+            .update(modified)
+            .expect(&format!("Modification {}", i));
         let standard_root = modified.tree_hash_root();
-        let cached_root = cache.root().unwrap().to_vec();
-        assert_eq!(standard_root, cached_root, "Modification {} failed.", i);
+        let cached_root = hasher
+            .tree_hash_root()
+            .expect(&format!("Modification {}", i));
+        assert_eq!(
+            standard_root, cached_root,
+            "Modification {} failed. \n Cache: {:?}",
+            i, hasher
+        );
     }
 }
 
 #[test]
-fn test_nested() {
-    let original = Nested {
+fn test_nested_struct() {
+    let original = NestedStruct {
         a: 42,
         b: Inner {
             a: 12,
@@ -47,7 +55,7 @@ fn test_nested() {
             d: 15,
         },
     };
-    let modified = vec![Nested {
+    let modified = vec![NestedStruct {
         a: 99,
         ..original.clone()
     }];
@@ -73,8 +81,8 @@ fn test_inner() {
 }
 
 #[test]
-fn test_thing() {
-    let original = Thing {
+fn test_struct_with_vec() {
+    let original = StructWithVec {
         a: 42,
         b: Inner {
             a: 12,
@@ -85,10 +93,51 @@ fn test_thing() {
         c: vec![1, 2, 3, 4, 5],
     };
 
-    let modified = vec![Thing {
-        a: 99,
-        ..original.clone()
-    }];
+    let modified = vec![
+        StructWithVec {
+            a: 99,
+            ..original.clone()
+        },
+        StructWithVec {
+            a: 100,
+            ..original.clone()
+        },
+        StructWithVec {
+            c: vec![1, 2, 3, 4, 5],
+            ..original.clone()
+        },
+        StructWithVec {
+            c: vec![1, 3, 4, 5, 6],
+            ..original.clone()
+        },
+        StructWithVec {
+            c: vec![1, 3, 4, 5, 6, 7, 8, 9],
+            ..original.clone()
+        },
+        StructWithVec {
+            c: vec![1, 3, 4, 5],
+            ..original.clone()
+        },
+        StructWithVec {
+            b: Inner {
+                a: u64::max_value(),
+                b: u64::max_value(),
+                c: u64::max_value(),
+                d: u64::max_value(),
+            },
+            c: vec![],
+            ..original.clone()
+        },
+        StructWithVec {
+            b: Inner {
+                a: 0,
+                b: 1,
+                c: 2,
+                d: 3,
+            },
+            ..original.clone()
+        },
+    ];
 
     test_routine(original, modified);
 }
@@ -97,7 +146,17 @@ fn test_thing() {
 fn test_vec() {
     let original = vec![1, 2, 3, 4, 5];
 
-    let modified = vec![vec![1, 2, 3, 4, 42]];
+    let modified = vec![
+        vec![1, 2, 3, 4, 42],
+        vec![1, 2, 3, 4],
+        vec![],
+        vec![42; 2_usize.pow(4)],
+        vec![],
+        vec![],
+        vec![1, 2, 3, 4, 42],
+        vec![1, 2, 3],
+        vec![1],
+    ];
 
     test_routine(original, modified);
 }
@@ -158,12 +217,11 @@ impl CachedTreeHashSubTree<Inner> for Inner {
         lengths.push(BTreeOverlay::new(&self.c, 0)?.num_nodes());
         lengths.push(BTreeOverlay::new(&self.d, 0)?.num_nodes());
 
-        BTreeOverlay::from_lengths(chunk_offset, lengths)
+        BTreeOverlay::from_lengths(chunk_offset, 4, lengths)
     }
 
     fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
-        let overlay = cache.get_overlay(cache.overlay_index, cache.chunk_index)?;
-        dbg!(&overlay);
+        let overlay = BTreeOverlay::new(self, cache.chunk_index)?;
 
         // Skip the chunk index to the first leaf node of this struct.
         cache.chunk_index = overlay.first_leaf_node();
diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs
index b308953a3..d0a9e7a4e 100644
--- a/eth2/utils/tree_hash_derive/src/lib.rs
+++ b/eth2/utils/tree_hash_derive/src/lib.rs
@@ -55,6 +55,8 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
     let idents_b = idents_a.clone();
     let idents_c = idents_a.clone();
 
+    let num_items = idents_a.len();
+
     let output = quote! {
         impl tree_hash::CachedTreeHashSubTree<#name> for #name {
             fn new_tree_hash_cache(&self) -> Result<tree_hash::TreeHashCache, tree_hash::Error> {
@@ -77,11 +79,11 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
                     lengths.push(tree_hash::BTreeOverlay::new(&self.#idents_b, 0)?.num_nodes());
                 )*
 
-                tree_hash::BTreeOverlay::from_lengths(chunk_offset, lengths)
+                tree_hash::BTreeOverlay::from_lengths(chunk_offset, #num_items, lengths)
             }
 
             fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
-                let overlay = cache.get_overlay(cache.overlay_index, cache.chunk_index)?;
+                let overlay = BTreeOverlay::new(self, cache.chunk_index)?;
 
                 // Skip the chunk index to the first leaf node of this struct.
                 cache.chunk_index = overlay.first_leaf_node();

From 7c64a5a21bdbf11346ec52dd4f432ac9400125a5 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Mon, 22 Apr 2019 16:09:29 +1000
Subject: [PATCH 091/137] Add tests, fix bugs

---
 .../src/cached_tree_hash/btree_overlay.rs     | 37 ++++++++++--
 .../src/cached_tree_hash/impls/vec.rs         | 58 +++++++++++++++----
 eth2/utils/tree_hash/tests/tests.rs           | 19 ++++++
 3 files changed, 99 insertions(+), 15 deletions(-)

diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs b/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
index cfd58df8a..28cde2d9a 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
@@ -35,7 +35,7 @@ impl BTreeOverlay {
         self.lengths.len().next_power_of_two()
     }
 
-    fn num_padding_leaves(&self) -> usize {
+    pub fn num_padding_leaves(&self) -> usize {
         self.num_leaf_nodes() - self.lengths.len()
     }
 
@@ -76,14 +76,31 @@ impl BTreeOverlay {
         self.offset + self.num_internal_nodes()
     }
 
+    /// Returns the chunk-range for a given leaf node.
+    ///
+    /// Returns `None` if:
+    ///     - The specified node is internal.
+    ///     - The specified node is padding.
+    ///     - The specified node is OOB of the tree.
     pub fn get_leaf_node(&self, i: usize) -> Result<Option<Range<usize>>, Error> {
-        if i >= self.num_leaf_nodes() {
-            return Err(Error::NotLeafNode(i));
-        } else if i >= self.num_leaf_nodes() - self.num_padding_leaves() {
+        if i >= self.num_nodes() - self.num_padding_leaves() {
+            Ok(None)
+        /*
+        } else if i < self.num_internal_nodes() {
+            Ok(None)
+        */
+        } else if (i == self.num_internal_nodes()) && (self.num_items == 0) {
+            // If this is the first leaf node and the overlay contains zero items, return `None` as
+            // this node must be padding.
             Ok(None)
         } else {
-            let first_node = self.offset + self.lengths.iter().take(i).sum::<usize>();
+            let i = i - self.num_internal_nodes();
+
+            let first_node = self.offset
+                + self.num_internal_nodes()
+                + self.lengths.iter().take(i).sum::<usize>();
             let last_node = first_node + self.lengths[i];
+
             Ok(Some(first_node..last_node))
         }
     }
@@ -191,6 +208,16 @@ mod test {
         assert_eq!(tree.chunk_range(), 11..14);
     }
 
+    #[test]
+    fn get_leaf_node() {
+        let tree = get_tree_a(4);
+
+        assert_eq!(tree.get_leaf_node(3), Ok(Some(3..4)));
+        assert_eq!(tree.get_leaf_node(4), Ok(Some(4..5)));
+        assert_eq!(tree.get_leaf_node(5), Ok(Some(5..6)));
+        assert_eq!(tree.get_leaf_node(6), Ok(Some(6..7)));
+    }
+
     #[test]
     fn root_of_one_node() {
         let tree = get_tree_a(1);
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
index d71b58816..d8d503af9 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
@@ -47,6 +47,11 @@ where
                     lengths.push(BTreeOverlay::new(item, 0)?.num_nodes())
                 }
 
+                // Disallow zero-length as an empty list still has one all-padding node.
+                if lengths.is_empty() {
+                    lengths.push(1);
+                }
+
                 lengths
             }
         };
@@ -56,9 +61,7 @@ where
 
     fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
         let new_overlay = BTreeOverlay::new(self, cache.chunk_index)?;
-        let old_overlay = cache
-            .get_overlay(cache.overlay_index, cache.chunk_index)?
-            .clone();
+        let old_overlay = cache.get_overlay(cache.overlay_index, cache.chunk_index)?;
 
         // If the merkle tree required to represent the new list is of a different size to the one
         // required for the previous list, then update our cache.
@@ -69,6 +72,8 @@ where
             cache.replace_overlay(cache.overlay_index, cache.chunk_index, new_overlay.clone())?;
         }
 
+        cache.overlay_index += 1;
+
         match T::tree_hash_type() {
             TreeHashType::Basic => {
                 let mut buf = vec![0; HASHSIZE];
@@ -78,7 +83,7 @@ where
                 for i in 0..new_overlay.num_leaf_nodes() {
                     // Iterate through the number of items that may be packing into the leaf node.
                     for j in 0..T::tree_hash_packing_factor() {
-                        // Create a mut slice that can either be filled with a serialized item or
+                        // Create a mut slice that can be filled with either a serialized item or
                         // padding.
                         let buf_slice = &mut buf[j * item_bytes..(j + 1) * item_bytes];
 
@@ -99,20 +104,47 @@ where
                 }
             }
             TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
-                for i in (0..new_overlay.num_leaf_nodes()).rev() {
-                    match (old_overlay.get_leaf_node(i)?, new_overlay.get_leaf_node(i)?) {
+                let mut local_overlay_index = cache.overlay_index;
+
+
+                for i in 0..new_overlay.num_leaf_nodes() {
+                    cache.overlay_index = local_overlay_index;
+
+                    // Adjust `i` so it is a leaf node for each of the overlays.
+                    let old_i = i + old_overlay.num_internal_nodes();
+                    let new_i = i + new_overlay.num_internal_nodes();
+
+                    match (
+                        old_overlay.get_leaf_node(old_i)?,
+                        new_overlay.get_leaf_node(new_i)?,
+                    ) {
                         // The item existed in the previous list and exists in the current list.
                         (Some(_old), Some(new)) => {
                             cache.chunk_index = new.start;
+
+
                             self[i].update_tree_hash_cache(cache)?;
+
+                            local_overlay_index += 1;
                         }
                         // The item existed in the previous list but does not exist in this list.
                         //
                         // Viz., the list has been shortened.
                         (Some(old), None) => {
-                            // Splice out the entire tree of the removed node, replacing it with a
-                            // single padding node.
-                            cache.splice(old, vec![0; HASHSIZE], vec![true]);
+                            if new_overlay.num_items == 0  {
+                                // In this case, the list has been made empty and we should make
+                                // this node padding.
+                                cache.maybe_update_chunk(new_overlay.root(), &[0; HASHSIZE])?;
+                            } else {
+                                // In this case, there are some items in the new list and we should
+                                // splice out the entire tree of the removed node, replacing it
+                                // with a single padding node.
+                                cache.splice(old, vec![0; HASHSIZE], vec![true]);
+
+                                cache.overlays.remove(cache.overlay_index);
+                            }
+
+                            local_overlay_index += 1;
                         }
                         // The item did not exist in the previous list but does exist in this list.
                         //
@@ -122,6 +154,13 @@ where
                             let bools = vec![true; bytes.len() / HASHSIZE];
 
                             cache.splice(new.start..new.start + 1, bytes, bools);
+
+                            cache.overlays.insert(
+                                std::cmp::min(cache.overlay_index, cache.overlays.len()),
+                                BTreeOverlay::new(&self[i], 0)?,
+                            );
+
+                            local_overlay_index += 1;
                         }
                         // The item didn't exist in the old list and doesn't exist in the new list,
                         // nothing to do.
@@ -136,7 +175,6 @@ where
         // Mix in length.
         let root_node = new_overlay.root();
         if cache.changed(root_node)? {
-            dbg!(cache.get_chunk(12));
             cache.modify_chunk(root_node, &cache.mix_in_length(root_node, self.len())?)?;
         } else if old_overlay.num_items != new_overlay.num_items {
             if new_overlay.num_internal_nodes() == 0 {
diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
index 52f7a7cb1..726ce5626 100644
--- a/eth2/utils/tree_hash/tests/tests.rs
+++ b/eth2/utils/tree_hash/tests/tests.rs
@@ -28,6 +28,7 @@ where
     assert_eq!(standard_root, cached_root, "Initial cache build failed.");
 
     for (i, modified) in modified.iter().enumerate() {
+        println!("-- Start of modification {} --", i);
         // Test after a modification
         hasher
             .update(modified)
@@ -161,6 +162,24 @@ fn test_vec() {
     test_routine(original, modified);
 }
 
+#[test]
+fn test_nested_list() {
+    let original: Vec<Vec<u64>> = vec![vec![1]];
+
+    let modified = vec![
+        vec![vec![1]],
+        vec![vec![1], vec![2]],
+        vec![vec![1], vec![3], vec![4]],
+        vec![],
+        vec![vec![1], vec![3], vec![4]],
+        vec![],
+        vec![vec![1, 2], vec![3], vec![4, 5, 6, 7, 8]],
+        vec![],
+    ];
+
+    test_routine(original, modified);
+}
+
 #[derive(Clone, Debug)]
 pub struct Inner {
     pub a: u64,

From 2f69185ccb46652607a68d9d936277a678b499e9 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Mon, 22 Apr 2019 16:58:40 +1000
Subject: [PATCH 092/137] Fix tree hash bug

---
 eth2/utils/tree_hash/src/cached_tree_hash/resize.rs | 12 ++++++++++--
 eth2/utils/tree_hash/tests/tests.rs                 |  4 +++-
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/resize.rs b/eth2/utils/tree_hash/src/cached_tree_hash/resize.rs
index 44b3f0ea5..c87746654 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/resize.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/resize.rs
@@ -1,4 +1,5 @@
 use super::*;
+use std::cmp::min;
 
 /// New vec is bigger than old vec.
 pub fn grow_merkle_cache(
@@ -100,8 +101,15 @@ pub fn shrink_merkle_cache(
             )
         };
 
-        to_byte_slice.copy_from_slice(from_byte_slice.get(0..to_byte_slice.len())?);
-        to_flag_slice.copy_from_slice(from_flag_slice.get(0..to_flag_slice.len())?);
+        let num_bytes = min(from_byte_slice.len(), to_byte_slice.len());
+        let num_flags = min(from_flag_slice.len(), to_flag_slice.len());
+
+        to_byte_slice
+            .get_mut(0..num_bytes)?
+            .copy_from_slice(from_byte_slice.get(0..num_bytes)?);
+        to_flag_slice
+            .get_mut(0..num_flags)?
+            .copy_from_slice(from_flag_slice.get(0..num_flags)?);
     }
 
     Some((bytes, flags))
diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
index 726ce5626..27a20cd77 100644
--- a/eth2/utils/tree_hash/tests/tests.rs
+++ b/eth2/utils/tree_hash/tests/tests.rs
@@ -163,7 +163,7 @@ fn test_vec() {
 }
 
 #[test]
-fn test_nested_list() {
+fn test_nested_list_of_u64() {
     let original: Vec<Vec<u64>> = vec![vec![1]];
 
     let modified = vec![
@@ -175,6 +175,8 @@ fn test_nested_list() {
         vec![],
         vec![vec![1, 2], vec![3], vec![4, 5, 6, 7, 8]],
         vec![],
+        vec![vec![1], vec![2], vec![3]],
+        vec![vec![1, 2, 3, 4, 5, 6], vec![1, 2, 3, 4, 5, 6, 7]],
     ];
 
     test_routine(original, modified);

From ec43a4085c8f8917f9e5c3b010797460947c11a4 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Mon, 22 Apr 2019 21:31:39 +1000
Subject: [PATCH 093/137] Introduce failing test

---
 .../tree_hash/src/cached_tree_hash/impls.rs   |  4 ++
 .../src/cached_tree_hash/impls/vec.rs         | 11 +++--
 eth2/utils/tree_hash/tests/tests.rs           | 43 +++++++++++++++++++
 eth2/utils/tree_hash_derive/src/lib.rs        |  4 ++
 4 files changed, 59 insertions(+), 3 deletions(-)

diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
index ece025ccd..64fab5cf8 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
@@ -19,9 +19,13 @@ impl CachedTreeHashSubTree<u64> for u64 {
         let leaf = merkleize(self.to_le_bytes().to_vec());
         cache.maybe_update_chunk(cache.chunk_index, &leaf)?;
 
+        dbg!(cache.overlay_index);
+
         cache.chunk_index += 1;
         cache.overlay_index += 1;
 
+        dbg!(cache.overlay_index);
+
         Ok(())
     }
 }
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
index d8d503af9..43c9ce7e3 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
@@ -63,6 +63,11 @@ where
         let new_overlay = BTreeOverlay::new(self, cache.chunk_index)?;
         let old_overlay = cache.get_overlay(cache.overlay_index, cache.chunk_index)?;
 
+        dbg!(cache.overlay_index);
+
+        // dbg!(&new_overlay);
+        // dbg!(&old_overlay);
+
         // If the merkle tree required to represent the new list is of a different size to the one
         // required for the previous list, then update our cache.
         //
@@ -106,7 +111,6 @@ where
             TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
                 let mut local_overlay_index = cache.overlay_index;
 
-
                 for i in 0..new_overlay.num_leaf_nodes() {
                     cache.overlay_index = local_overlay_index;
 
@@ -122,7 +126,6 @@ where
                         (Some(_old), Some(new)) => {
                             cache.chunk_index = new.start;
 
-
                             self[i].update_tree_hash_cache(cache)?;
 
                             local_overlay_index += 1;
@@ -131,7 +134,7 @@ where
                         //
                         // Viz., the list has been shortened.
                         (Some(old), None) => {
-                            if new_overlay.num_items == 0  {
+                            if new_overlay.num_items == 0 {
                                 // In this case, the list has been made empty and we should make
                                 // this node padding.
                                 cache.maybe_update_chunk(new_overlay.root(), &[0; HASHSIZE])?;
@@ -188,6 +191,8 @@ where
 
         cache.chunk_index = new_overlay.next_node();
 
+        dbg!(&cache.overlay_index);
+
         Ok(())
     }
 }
diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
index 27a20cd77..d4897d2c0 100644
--- a/eth2/utils/tree_hash/tests/tests.rs
+++ b/eth2/utils/tree_hash/tests/tests.rs
@@ -177,6 +177,47 @@ fn test_nested_list_of_u64() {
         vec![],
         vec![vec![1], vec![2], vec![3]],
         vec![vec![1, 2, 3, 4, 5, 6], vec![1, 2, 3, 4, 5, 6, 7]],
+        vec![vec![], vec![], vec![]],
+        vec![vec![0, 0, 0], vec![0], vec![0]],
+    ];
+
+    test_routine(original, modified);
+}
+
+#[test]
+fn test_list_of_struct_with_vec() {
+    let a = StructWithVec {
+        a: 42,
+        b: Inner {
+            a: 12,
+            b: 13,
+            c: 14,
+            d: 15,
+        },
+        c: vec![1, 2, 3, 4, 5],
+    };
+    let b = StructWithVec {
+        c: vec![],
+        ..a.clone()
+    };
+    let c = StructWithVec {
+        b: Inner {
+            a: 99,
+            b: 100,
+            c: 101,
+            d: 102,
+        },
+        ..a.clone()
+    };
+    let d = StructWithVec { a: 0, ..a.clone() };
+
+    let original: Vec<StructWithVec> = vec![a.clone(), c.clone()];
+
+    let modified = vec![
+        vec![a.clone(), c.clone()],
+        // vec![a.clone(), b.clone(), c.clone(), d.clone()],
+        // vec![b.clone(), a.clone(), c.clone(), d.clone()],
+        vec![],
     ];
 
     test_routine(original, modified);
@@ -255,6 +296,8 @@ impl CachedTreeHashSubTree<Inner> for Inner {
         self.c.update_tree_hash_cache(cache)?;
         self.d.update_tree_hash_cache(cache)?;
 
+        dbg!(cache.overlay_index);
+
         // Iterate through the internal nodes, updating them if their children have changed.
         cache.update_internal_nodes(&overlay)?;
 
diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs
index d0a9e7a4e..6160913ee 100644
--- a/eth2/utils/tree_hash_derive/src/lib.rs
+++ b/eth2/utils/tree_hash_derive/src/lib.rs
@@ -85,6 +85,8 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
             fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
                 let overlay = BTreeOverlay::new(self, cache.chunk_index)?;
 
+                println!("start derive - cache.overlay_index: {}", cache.overlay_index);
+
                 // Skip the chunk index to the first leaf node of this struct.
                 cache.chunk_index = overlay.first_leaf_node();
                 // Skip the overlay index to the first leaf node of this struct.
@@ -95,6 +97,8 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
                     self.#idents_c.update_tree_hash_cache(cache)?;
                 )*
 
+                println!("end derive - cache.overlay_index: {}", cache.overlay_index);
+
                 // Iterate through the internal nodes, updating them if their children have changed.
                 cache.update_internal_nodes(&overlay)?;
 

From a84a063c25d332bbcf27a38ad2deac4e4a311e5a Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 24 Apr 2019 09:29:32 +1000
Subject: [PATCH 094/137] Update depth variable

---
 eth2/utils/tree_hash/src/cached_tree_hash.rs  | 42 +++++++---
 .../src/cached_tree_hash/btree_overlay.rs     | 11 ++-
 .../tree_hash/src/cached_tree_hash/impls.rs   | 19 ++---
 .../src/cached_tree_hash/impls/vec.rs         | 80 ++++++++++---------
 eth2/utils/tree_hash/tests/tests.rs           | 12 +--
 eth2/utils/tree_hash_derive/src/lib.rs        | 19 ++---
 6 files changed, 105 insertions(+), 78 deletions(-)

diff --git a/eth2/utils/tree_hash/src/cached_tree_hash.rs b/eth2/utils/tree_hash/src/cached_tree_hash.rs
index 35ddf10cd..878e53766 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash.rs
@@ -20,7 +20,7 @@ impl CachedTreeHasher {
         T: CachedTreeHashSubTree<T>,
     {
         Ok(Self {
-            cache: TreeHashCache::new(item)?,
+            cache: TreeHashCache::new(item, 0)?,
         })
     }
 
@@ -73,9 +73,13 @@ pub trait CachedTreeHash<T>: CachedTreeHashSubTree<T> + Sized {
 }
 
 pub trait CachedTreeHashSubTree<Item>: TreeHash {
-    fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error>;
+    fn tree_hash_cache_overlay(
+        &self,
+        chunk_offset: usize,
+        depth: usize,
+    ) -> Result<BTreeOverlay, Error>;
 
-    fn new_tree_hash_cache(&self) -> Result<TreeHashCache, Error>;
+    fn new_tree_hash_cache(&self, depth: usize) -> Result<TreeHashCache, Error>;
 
     fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error>;
 }
@@ -172,21 +176,22 @@ impl Into<Vec<u8>> for TreeHashCache {
 }
 
 impl TreeHashCache {
-    pub fn new<T>(item: &T) -> Result<Self, Error>
+    pub fn new<T>(item: &T, depth: usize) -> Result<Self, Error>
     where
         T: CachedTreeHashSubTree<T>,
     {
-        item.new_tree_hash_cache()
+        item.new_tree_hash_cache(depth)
     }
 
     pub fn from_leaves_and_subtrees<T>(
         item: &T,
         leaves_and_subtrees: Vec<Self>,
+        depth: usize,
     ) -> Result<Self, Error>
     where
         T: CachedTreeHashSubTree<T>,
     {
-        let overlay = BTreeOverlay::new(item, 0)?;
+        let overlay = BTreeOverlay::new(item, 0, depth)?;
 
         // Note how many leaves were provided. If is not a power-of-two, we'll need to pad it out
         // later.
@@ -204,7 +209,10 @@ impl TreeHashCache {
         // Allocate enough bytes to store all the leaves.
         let mut leaves = Vec::with_capacity(overlay.num_leaf_nodes() * HASHSIZE);
         let mut overlays = Vec::with_capacity(leaves_and_subtrees.len());
-        overlays.push(overlay);
+
+        if T::tree_hash_type() == TreeHashType::List {
+            overlays.push(overlay);
+        }
 
         // Iterate through all of the leaves/subtrees, adding their root as a leaf node and then
         // concatenating their merkle trees.
@@ -238,16 +246,21 @@ impl TreeHashCache {
     pub fn from_bytes(
         bytes: Vec<u8>,
         initial_modified_state: bool,
-        overlay: BTreeOverlay,
+        overlay: Option<BTreeOverlay>,
     ) -> Result<Self, Error> {
         if bytes.len() % BYTES_PER_CHUNK > 0 {
             return Err(Error::BytesAreNotEvenChunks(bytes.len()));
         }
 
+        let overlays = match overlay {
+            Some(overlay) => vec![overlay],
+            None => vec![],
+        };
+
         Ok(Self {
             chunk_modified: vec![initial_modified_state; bytes.len() / BYTES_PER_CHUNK],
             cache: bytes,
-            overlays: vec![overlay],
+            overlays,
             chunk_index: 0,
             overlay_index: 0,
         })
@@ -317,6 +330,17 @@ impl TreeHashCache {
         ))
     }
 
+    pub fn remove_proceeding_child_overlays(&mut self, overlay_index: usize, depth: usize) {
+        let end = self
+            .overlays
+            .iter()
+            .skip(overlay_index)
+            .position(|o| o.depth <= depth)
+            .unwrap_or_else(|| self.overlays.len());
+
+        self.overlays.splice(overlay_index..end, vec![]);
+    }
+
     pub fn update_internal_nodes(&mut self, overlay: &BTreeOverlay) -> Result<(), Error> {
         for (parent, children) in overlay.internal_parents_and_children().into_iter().rev() {
             if self.either_modified(children)? {
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs b/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
index 28cde2d9a..989b8bd98 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
@@ -3,21 +3,23 @@ use super::*;
 #[derive(Debug, PartialEq, Clone)]
 pub struct BTreeOverlay {
     pub offset: usize,
+    pub depth: usize,
     pub num_items: usize,
     pub lengths: Vec<usize>,
 }
 
 impl BTreeOverlay {
-    pub fn new<T>(item: &T, initial_offset: usize) -> Result<Self, Error>
+    pub fn new<T>(item: &T, initial_offset: usize, depth: usize) -> Result<Self, Error>
     where
         T: CachedTreeHashSubTree<T>,
     {
-        item.tree_hash_cache_overlay(initial_offset)
+        item.tree_hash_cache_overlay(initial_offset, depth)
     }
 
     pub fn from_lengths(
         offset: usize,
         num_items: usize,
+        depth: usize,
         lengths: Vec<usize>,
     ) -> Result<Self, Error> {
         if lengths.is_empty() {
@@ -26,6 +28,7 @@ impl BTreeOverlay {
             Ok(Self {
                 offset,
                 num_items,
+                depth,
                 lengths,
             })
         }
@@ -166,7 +169,7 @@ mod test {
     use super::*;
 
     fn get_tree_a(n: usize) -> BTreeOverlay {
-        BTreeOverlay::from_lengths(0, n, vec![1; n]).unwrap()
+        BTreeOverlay::from_lengths(0, n, 0, vec![1; n]).unwrap()
     }
 
     #[test]
@@ -204,7 +207,7 @@ mod test {
         let tree = get_tree_a(2);
         assert_eq!(tree.chunk_range(), 0..3);
 
-        let tree = BTreeOverlay::from_lengths(11, 4, vec![1, 1]).unwrap();
+        let tree = BTreeOverlay::from_lengths(11, 4, 0, vec![1, 1]).unwrap();
         assert_eq!(tree.chunk_range(), 11..14);
     }
 
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
index 64fab5cf8..42d77d11d 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
@@ -3,28 +3,29 @@ use super::*;
 mod vec;
 
 impl CachedTreeHashSubTree<u64> for u64 {
-    fn new_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
+    fn new_tree_hash_cache(&self, depth: usize) -> Result<TreeHashCache, Error> {
         Ok(TreeHashCache::from_bytes(
             merkleize(self.to_le_bytes().to_vec()),
             false,
-            self.tree_hash_cache_overlay(0)?,
+            // self.tree_hash_cache_overlay(0, depth)?,
+            None,
         )?)
     }
 
-    fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
-        BTreeOverlay::from_lengths(chunk_offset, 1, vec![1])
+    fn tree_hash_cache_overlay(
+        &self,
+        chunk_offset: usize,
+        depth: usize,
+    ) -> Result<BTreeOverlay, Error> {
+        BTreeOverlay::from_lengths(chunk_offset, 1, depth, vec![1])
     }
 
     fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
         let leaf = merkleize(self.to_le_bytes().to_vec());
         cache.maybe_update_chunk(cache.chunk_index, &leaf)?;
 
-        dbg!(cache.overlay_index);
-
         cache.chunk_index += 1;
-        cache.overlay_index += 1;
-
-        dbg!(cache.overlay_index);
+        // cache.overlay_index += 1;
 
         Ok(())
     }
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
index 43c9ce7e3..4574624b3 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
@@ -4,22 +4,22 @@ impl<T> CachedTreeHashSubTree<Vec<T>> for Vec<T>
 where
     T: CachedTreeHashSubTree<T> + TreeHash,
 {
-    fn new_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
-        let overlay = self.tree_hash_cache_overlay(0)?;
+    fn new_tree_hash_cache(&self, depth: usize) -> Result<TreeHashCache, Error> {
+        let overlay = self.tree_hash_cache_overlay(0, depth)?;
 
         let mut cache = match T::tree_hash_type() {
             TreeHashType::Basic => TreeHashCache::from_bytes(
                 merkleize(get_packed_leaves(self)?),
                 false,
-                overlay.clone(),
+                Some(overlay.clone()),
             ),
             TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
                 let subtrees = self
                     .iter()
-                    .map(|item| TreeHashCache::new(item))
+                    .map(|item| TreeHashCache::new(item, depth + 1))
                     .collect::<Result<Vec<TreeHashCache>, _>>()?;
 
-                TreeHashCache::from_leaves_and_subtrees(self, subtrees)
+                TreeHashCache::from_leaves_and_subtrees(self, subtrees, depth)
             }
         }?;
 
@@ -30,7 +30,11 @@ where
         Ok(cache)
     }
 
-    fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
+    fn tree_hash_cache_overlay(
+        &self,
+        chunk_offset: usize,
+        depth: usize,
+    ) -> Result<BTreeOverlay, Error> {
         let lengths = match T::tree_hash_type() {
             TreeHashType::Basic => {
                 // Ceil division.
@@ -44,7 +48,7 @@ where
                 let mut lengths = vec![];
 
                 for item in self {
-                    lengths.push(BTreeOverlay::new(item, 0)?.num_nodes())
+                    lengths.push(BTreeOverlay::new(item, 0, depth)?.num_nodes())
                 }
 
                 // Disallow zero-length as an empty list still has one all-padding node.
@@ -56,17 +60,12 @@ where
             }
         };
 
-        BTreeOverlay::from_lengths(chunk_offset, self.len(), lengths)
+        BTreeOverlay::from_lengths(chunk_offset, self.len(), depth, lengths)
     }
 
     fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
-        let new_overlay = BTreeOverlay::new(self, cache.chunk_index)?;
         let old_overlay = cache.get_overlay(cache.overlay_index, cache.chunk_index)?;
-
-        dbg!(cache.overlay_index);
-
-        // dbg!(&new_overlay);
-        // dbg!(&old_overlay);
+        let new_overlay = BTreeOverlay::new(self, cache.chunk_index, old_overlay.depth)?;
 
         // If the merkle tree required to represent the new list is of a different size to the one
         // required for the previous list, then update our cache.
@@ -109,11 +108,7 @@ where
                 }
             }
             TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
-                let mut local_overlay_index = cache.overlay_index;
-
                 for i in 0..new_overlay.num_leaf_nodes() {
-                    cache.overlay_index = local_overlay_index;
-
                     // Adjust `i` so it is a leaf node for each of the overlays.
                     let old_i = i + old_overlay.num_internal_nodes();
                     let new_i = i + new_overlay.num_internal_nodes();
@@ -127,8 +122,27 @@ where
                             cache.chunk_index = new.start;
 
                             self[i].update_tree_hash_cache(cache)?;
+                        }
+                        // The item did not exist in the previous list but does exist in this list.
+                        //
+                        // Viz., the list has been lengthened.
+                        (None, Some(new)) => {
+                            let (bytes, mut bools, overlays) =
+                                TreeHashCache::new(&self[i], new_overlay.depth + 1)?
+                                    .into_components();
 
-                            local_overlay_index += 1;
+                            // Record the number of overlays, this will be used later in the fn.
+                            let num_overlays = overlays.len();
+
+                            // Flag the root node of the new tree as dirty.
+                            bools[0] = true;
+
+                            cache.splice(new.start..new.start + 1, bytes, bools);
+                            cache
+                                .overlays
+                                .splice(cache.overlay_index..cache.overlay_index, overlays);
+
+                            cache.overlay_index += num_overlays;
                         }
                         // The item existed in the previous list but does not exist in this list.
                         //
@@ -144,37 +158,27 @@ where
                                 // with a single padding node.
                                 cache.splice(old, vec![0; HASHSIZE], vec![true]);
 
-                                cache.overlays.remove(cache.overlay_index);
+                                // cache.overlays.remove(cache.overlay_index);
                             }
 
-                            local_overlay_index += 1;
-                        }
-                        // The item did not exist in the previous list but does exist in this list.
-                        //
-                        // Viz., the list has been lengthened.
-                        (None, Some(new)) => {
-                            let bytes: Vec<u8> = TreeHashCache::new(&self[i])?.into();
-                            let bools = vec![true; bytes.len() / HASHSIZE];
-
-                            cache.splice(new.start..new.start + 1, bytes, bools);
-
-                            cache.overlays.insert(
-                                std::cmp::min(cache.overlay_index, cache.overlays.len()),
-                                BTreeOverlay::new(&self[i], 0)?,
-                            );
-
-                            local_overlay_index += 1;
+                            // local_overlay_index += 1;
                         }
                         // The item didn't exist in the old list and doesn't exist in the new list,
                         // nothing to do.
                         (None, None) => {}
                     }
                 }
+
+                // Clean out any excess overlays that may or may not be remaining if the list was
+                // shortened.
+                cache.remove_proceeding_child_overlays(cache.overlay_index, new_overlay.depth);
             }
         }
 
         cache.update_internal_nodes(&new_overlay)?;
 
+        dbg!(&new_overlay);
+
         // Mix in length.
         let root_node = new_overlay.root();
         if cache.changed(root_node)? {
@@ -191,8 +195,6 @@ where
 
         cache.chunk_index = new_overlay.next_node();
 
-        dbg!(&cache.overlay_index);
-
         Ok(())
     }
 }
diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
index d4897d2c0..26dd1ae38 100644
--- a/eth2/utils/tree_hash/tests/tests.rs
+++ b/eth2/utils/tree_hash/tests/tests.rs
@@ -217,13 +217,13 @@ fn test_list_of_struct_with_vec() {
         vec![a.clone(), c.clone()],
         // vec![a.clone(), b.clone(), c.clone(), d.clone()],
         // vec![b.clone(), a.clone(), c.clone(), d.clone()],
-        vec![],
+        // vec![],
     ];
 
     test_routine(original, modified);
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, TreeHash, CachedTreeHashSubTree)]
 pub struct Inner {
     pub a: u64,
     pub b: u64,
@@ -231,6 +231,7 @@ pub struct Inner {
     pub d: u64,
 }
 
+/*
 impl TreeHash for Inner {
     fn tree_hash_type() -> TreeHashType {
         TreeHashType::Container
@@ -296,14 +297,13 @@ impl CachedTreeHashSubTree<Inner> for Inner {
         self.c.update_tree_hash_cache(cache)?;
         self.d.update_tree_hash_cache(cache)?;
 
-        dbg!(cache.overlay_index);
-
         // Iterate through the internal nodes, updating them if their children have changed.
         cache.update_internal_nodes(&overlay)?;
 
         Ok(())
     }
 }
+*/
 
 fn generic_test(index: usize) {
     let inner = Inner {
@@ -313,7 +313,7 @@ fn generic_test(index: usize) {
         d: 4,
     };
 
-    let mut cache = TreeHashCache::new(&inner).unwrap();
+    let mut cache = TreeHashCache::new(&inner, 0).unwrap();
 
     let changed_inner = match index {
         0 => Inner {
@@ -378,7 +378,7 @@ fn inner_builds() {
         d: 4,
     };
 
-    let cache: Vec<u8> = TreeHashCache::new(&inner).unwrap().into();
+    let cache: Vec<u8> = TreeHashCache::new(&inner, 0).unwrap().into();
 
     assert_eq!(expected, cache);
 }
diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs
index 6160913ee..c2b56c19b 100644
--- a/eth2/utils/tree_hash_derive/src/lib.rs
+++ b/eth2/utils/tree_hash_derive/src/lib.rs
@@ -59,46 +59,43 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
 
     let output = quote! {
         impl tree_hash::CachedTreeHashSubTree<#name> for #name {
-            fn new_tree_hash_cache(&self) -> Result<tree_hash::TreeHashCache, tree_hash::Error> {
+            fn new_tree_hash_cache(&self, depth: usize) -> Result<tree_hash::TreeHashCache, tree_hash::Error> {
                 let tree = tree_hash::TreeHashCache::from_leaves_and_subtrees(
                     self,
                     vec![
                         #(
-                            self.#idents_a.new_tree_hash_cache()?,
+                            self.#idents_a.new_tree_hash_cache(depth)?,
                         )*
                     ],
+                    depth
                 )?;
 
                 Ok(tree)
             }
 
-            fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result<tree_hash::BTreeOverlay, tree_hash::Error> {
+            fn tree_hash_cache_overlay(&self, chunk_offset: usize, depth: usize) -> Result<tree_hash::BTreeOverlay, tree_hash::Error> {
                 let mut lengths = vec![];
 
                 #(
-                    lengths.push(tree_hash::BTreeOverlay::new(&self.#idents_b, 0)?.num_nodes());
+                    lengths.push(tree_hash::BTreeOverlay::new(&self.#idents_b, 0, depth)?.num_nodes());
                 )*
 
-                tree_hash::BTreeOverlay::from_lengths(chunk_offset, #num_items, lengths)
+                tree_hash::BTreeOverlay::from_lengths(chunk_offset, #num_items, depth, lengths)
             }
 
             fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
-                let overlay = BTreeOverlay::new(self, cache.chunk_index)?;
-
-                println!("start derive - cache.overlay_index: {}", cache.overlay_index);
+                let overlay = BTreeOverlay::new(self, cache.chunk_index, 0)?;
 
                 // Skip the chunk index to the first leaf node of this struct.
                 cache.chunk_index = overlay.first_leaf_node();
                 // Skip the overlay index to the first leaf node of this struct.
-                cache.overlay_index += 1;
+                // cache.overlay_index += 1;
 
                 // Recurse into the struct items, updating their caches.
                 #(
                     self.#idents_c.update_tree_hash_cache(cache)?;
                 )*
 
-                println!("end derive - cache.overlay_index: {}", cache.overlay_index);
-
                 // Iterate through the internal nodes, updating them if their children have changed.
                 cache.update_internal_nodes(&overlay)?;
 

From e19abee7f99c192ede46aa527f00204bc98f23d2 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 24 Apr 2019 10:17:05 +1000
Subject: [PATCH 095/137] Fix bug with num_nodes/num_chunks

---
 eth2/utils/tree_hash/src/cached_tree_hash.rs           |  2 +-
 .../tree_hash/src/cached_tree_hash/btree_overlay.rs    | 10 +++++++++-
 eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs |  4 +---
 eth2/utils/tree_hash/tests/tests.rs                    |  6 +++---
 4 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/eth2/utils/tree_hash/src/cached_tree_hash.rs b/eth2/utils/tree_hash/src/cached_tree_hash.rs
index 878e53766..6f96fcbf2 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash.rs
@@ -316,7 +316,7 @@ impl TreeHashCache {
                 old_flags,
                 old_overlay.height(),
                 new_overlay.height(),
-                new_overlay.total_chunks(),
+                new_overlay.num_chunks(),
             )
             .ok_or_else(|| Error::UnableToShrinkMerkleTree)?
         };
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs b/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
index 989b8bd98..450b4b2c6 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
@@ -42,6 +42,10 @@ impl BTreeOverlay {
         self.num_leaf_nodes() - self.lengths.len()
     }
 
+    /// Returns the number of nodes in the tree.
+    ///
+    /// Note: this is distinct from `num_chunks`, which returns the total number of chunks in
+    /// this tree.
     pub fn num_nodes(&self) -> usize {
         2 * self.num_leaf_nodes() - 1
     }
@@ -71,7 +75,11 @@ impl BTreeOverlay {
         self.first_node()..self.next_node()
     }
 
-    pub fn total_chunks(&self) -> usize {
+    /// Returns the number of chunks inside this tree (including subtrees).
+    ///
+    /// Note: this is distinct from `num_nodes` which returns the number of nodes in the binary
+    /// tree.
+    pub fn num_chunks(&self) -> usize {
         self.next_node() - self.first_node()
     }
 
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
index 4574624b3..964d2a229 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
@@ -48,7 +48,7 @@ where
                 let mut lengths = vec![];
 
                 for item in self {
-                    lengths.push(BTreeOverlay::new(item, 0, depth)?.num_nodes())
+                    lengths.push(BTreeOverlay::new(item, 0, depth)?.num_chunks())
                 }
 
                 // Disallow zero-length as an empty list still has one all-padding node.
@@ -177,8 +177,6 @@ where
 
         cache.update_internal_nodes(&new_overlay)?;
 
-        dbg!(&new_overlay);
-
         // Mix in length.
         let root_node = new_overlay.root();
         if cache.changed(root_node)? {
diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
index 26dd1ae38..c09ade8f2 100644
--- a/eth2/utils/tree_hash/tests/tests.rs
+++ b/eth2/utils/tree_hash/tests/tests.rs
@@ -215,9 +215,9 @@ fn test_list_of_struct_with_vec() {
 
     let modified = vec![
         vec![a.clone(), c.clone()],
-        // vec![a.clone(), b.clone(), c.clone(), d.clone()],
-        // vec![b.clone(), a.clone(), c.clone(), d.clone()],
-        // vec![],
+        vec![a.clone(), b.clone(), c.clone(), d.clone()],
+        vec![b.clone(), a.clone(), c.clone(), d.clone()],
+        vec![],
     ];
 
     test_routine(original, modified);

From ab75f7cbc713f85fa56d2f41e7c928818c8d005c Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 24 Apr 2019 11:37:19 +1000
Subject: [PATCH 096/137] Fix cargo cmd in Jenkinsfile

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 48a07e1e7..845cd357f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -24,7 +24,7 @@ pipeline {
 				sh 'cargo test --verbose --all'
 				sh 'cargo test --verbose --all --release'
                 sh 'cargo test --manifest-path eth2/state_processing/Cargo.toml --verbose \
-                               --release --features fake_crypto --ignored'
+                               --release --features fake_crypto -- --include-ignored'
 
 			}
 		}

From 6ae00838437a932add676cbb821763d99c99d5af Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 24 Apr 2019 11:41:33 +1000
Subject: [PATCH 097/137] Add travis caching.

Reference for commands: https://levans.fr/rust_travis_cache.html
---
 .travis.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 70b9d2133..f89db54c9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,4 +1,9 @@
 language: rust
+cache:
+  directories:
+    - /home/travis/.cargo
+before_cache:
+  - rm -rf /home/travis/.cargo/registry
 before_install:
   - curl -OL https://github.com/google/protobuf/releases/download/v3.4.0/protoc-3.4.0-linux-x86_64.zip
   - unzip protoc-3.4.0-linux-x86_64.zip -d protoc3

From e12fa58e6e06a6b8e2c4768badd669a9c753dfaa Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 24 Apr 2019 14:56:39 +1000
Subject: [PATCH 098/137] Fix failing test, add hacky fix

---
 eth2/utils/tree_hash/src/cached_tree_hash.rs  |  58 ++--
 .../src/cached_tree_hash/btree_overlay.rs     |   4 -
 .../src/cached_tree_hash/impls/vec.rs         |  16 +-
 eth2/utils/tree_hash/tests/tests.rs           | 257 +++++++++---------
 eth2/utils/tree_hash_derive/src/lib.rs        |   5 +-
 5 files changed, 171 insertions(+), 169 deletions(-)

diff --git a/eth2/utils/tree_hash/src/cached_tree_hash.rs b/eth2/utils/tree_hash/src/cached_tree_hash.rs
index 6f96fcbf2..46190ff3c 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash.rs
@@ -220,7 +220,6 @@ impl TreeHashCache {
             leaves.append(&mut t.root()?.to_vec());
 
             let (mut bytes, _bools, mut t_overlays) = t.into_components();
-
             cache.append(&mut bytes);
             overlays.append(&mut t_overlays);
         }
@@ -296,33 +295,40 @@ impl TreeHashCache {
     ) -> Result<BTreeOverlay, Error> {
         let old_overlay = self.get_overlay(overlay_index, chunk_index)?;
 
-        // Get slices of the exsiting tree from the cache.
-        let (old_bytes, old_flags) = self
-            .slices(old_overlay.chunk_range())
-            .ok_or_else(|| Error::UnableToObtainSlices)?;
+        // If the merkle tree required to represent the new list is of a different size to the one
+        // required for the previous list, then update our cache.
+        //
+        // This grows/shrinks the bytes to accomodate the new tree, preserving as much of the tree
+        // as possible.
+        if new_overlay.num_leaf_nodes() != old_overlay.num_leaf_nodes() {
+            // Get slices of the exsiting tree from the cache.
+            let (old_bytes, old_flags) = self
+                .slices(old_overlay.chunk_range())
+                .ok_or_else(|| Error::UnableToObtainSlices)?;
 
-        let (new_bytes, new_bools) = if new_overlay.num_leaf_nodes() > old_overlay.num_leaf_nodes()
-        {
-            resize::grow_merkle_cache(
-                old_bytes,
-                old_flags,
-                old_overlay.height(),
-                new_overlay.height(),
-            )
-            .ok_or_else(|| Error::UnableToGrowMerkleTree)?
-        } else {
-            resize::shrink_merkle_cache(
-                old_bytes,
-                old_flags,
-                old_overlay.height(),
-                new_overlay.height(),
-                new_overlay.num_chunks(),
-            )
-            .ok_or_else(|| Error::UnableToShrinkMerkleTree)?
-        };
+            let (new_bytes, new_bools) =
+                if new_overlay.num_leaf_nodes() > old_overlay.num_leaf_nodes() {
+                    resize::grow_merkle_cache(
+                        old_bytes,
+                        old_flags,
+                        old_overlay.height(),
+                        new_overlay.height(),
+                    )
+                    .ok_or_else(|| Error::UnableToGrowMerkleTree)?
+                } else {
+                    resize::shrink_merkle_cache(
+                        old_bytes,
+                        old_flags,
+                        old_overlay.height(),
+                        new_overlay.height(),
+                        new_overlay.num_chunks(),
+                    )
+                    .ok_or_else(|| Error::UnableToShrinkMerkleTree)?
+                };
 
-        // Splice the newly created `TreeHashCache` over the existing elements.
-        self.splice(old_overlay.chunk_range(), new_bytes, new_bools);
+            // Splice the newly created `TreeHashCache` over the existing elements.
+            self.splice(old_overlay.chunk_range(), new_bytes, new_bools);
+        }
 
         Ok(std::mem::replace(
             &mut self.overlays[overlay_index],
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs b/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
index 450b4b2c6..463586d40 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
@@ -96,10 +96,6 @@ impl BTreeOverlay {
     pub fn get_leaf_node(&self, i: usize) -> Result<Option<Range<usize>>, Error> {
         if i >= self.num_nodes() - self.num_padding_leaves() {
             Ok(None)
-        /*
-        } else if i < self.num_internal_nodes() {
-            Ok(None)
-        */
         } else if (i == self.num_internal_nodes()) && (self.num_items == 0) {
             // If this is the first leaf node and the overlay contains zero items, return `None` as
             // this node must be padding.
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
index 964d2a229..5ab7d06e4 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
@@ -67,14 +67,7 @@ where
         let old_overlay = cache.get_overlay(cache.overlay_index, cache.chunk_index)?;
         let new_overlay = BTreeOverlay::new(self, cache.chunk_index, old_overlay.depth)?;
 
-        // If the merkle tree required to represent the new list is of a different size to the one
-        // required for the previous list, then update our cache.
-        //
-        // This grows/shrinks the bytes to accomodate the new tree, preserving as much of the tree
-        // as possible.
-        if new_overlay.num_leaf_nodes() != old_overlay.num_leaf_nodes() {
-            cache.replace_overlay(cache.overlay_index, cache.chunk_index, new_overlay.clone())?;
-        }
+        cache.replace_overlay(cache.overlay_index, cache.chunk_index, new_overlay.clone())?;
 
         cache.overlay_index += 1;
 
@@ -120,6 +113,9 @@ where
                         // The item existed in the previous list and exists in the current list.
                         (Some(_old), Some(new)) => {
                             cache.chunk_index = new.start;
+                            if cache.chunk_index + 1 < cache.chunk_modified.len() {
+                                cache.chunk_modified[cache.chunk_index + 1] = true;
+                            }
 
                             self[i].update_tree_hash_cache(cache)?;
                         }
@@ -157,11 +153,7 @@ where
                                 // splice out the entire tree of the removed node, replacing it
                                 // with a single padding node.
                                 cache.splice(old, vec![0; HASHSIZE], vec![true]);
-
-                                // cache.overlays.remove(cache.overlay_index);
                             }
-
-                            // local_overlay_index += 1;
                         }
                         // The item didn't exist in the old list and doesn't exist in the new list,
                         // nothing to do.
diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
index c09ade8f2..bc3c5538f 100644
--- a/eth2/utils/tree_hash/tests/tests.rs
+++ b/eth2/utils/tree_hash/tests/tests.rs
@@ -1,7 +1,5 @@
 use int_to_bytes::int_to_bytes32;
 use tree_hash::cached_tree_hash::*;
-use tree_hash::standard_tree_hash::*;
-use tree_hash::*;
 use tree_hash_derive::{CachedTreeHashSubTree, TreeHash};
 
 #[derive(Clone, Debug, TreeHash, CachedTreeHashSubTree)]
@@ -10,13 +8,6 @@ pub struct NestedStruct {
     pub b: Inner,
 }
 
-#[derive(Clone, Debug, TreeHash, CachedTreeHashSubTree)]
-pub struct StructWithVec {
-    pub a: u64,
-    pub b: Inner,
-    pub c: Vec<u64>,
-}
-
 fn test_routine<T>(original: T, modified: Vec<T>)
 where
     T: CachedTreeHashSubTree<T>,
@@ -81,6 +72,54 @@ fn test_inner() {
     test_routine(original, modified);
 }
 
+#[test]
+fn test_vec() {
+    let original = vec![1, 2, 3, 4, 5];
+
+    let modified = vec![
+        vec![1, 2, 3, 4, 42],
+        vec![1, 2, 3, 4],
+        vec![],
+        vec![42; 2_usize.pow(4)],
+        vec![],
+        vec![],
+        vec![1, 2, 3, 4, 42],
+        vec![1, 2, 3],
+        vec![1],
+    ];
+
+    test_routine(original, modified);
+}
+
+#[test]
+fn test_nested_list_of_u64() {
+    let original: Vec<Vec<u64>> = vec![vec![1]];
+
+    let modified = vec![
+        vec![vec![1]],
+        vec![vec![1], vec![2]],
+        vec![vec![1], vec![3], vec![4]],
+        vec![],
+        vec![vec![1], vec![3], vec![4]],
+        vec![],
+        vec![vec![1, 2], vec![3], vec![4, 5, 6, 7, 8]],
+        vec![],
+        vec![vec![1], vec![2], vec![3]],
+        vec![vec![1, 2, 3, 4, 5, 6], vec![1, 2, 3, 4, 5, 6, 7]],
+        vec![vec![], vec![], vec![]],
+        vec![vec![0, 0, 0], vec![0], vec![0]],
+    ];
+
+    test_routine(original, modified);
+}
+
+#[derive(Clone, Debug, TreeHash, CachedTreeHashSubTree)]
+pub struct StructWithVec {
+    pub a: u64,
+    pub b: Inner,
+    pub c: Vec<u64>,
+}
+
 #[test]
 fn test_struct_with_vec() {
     let original = StructWithVec {
@@ -144,48 +183,7 @@ fn test_struct_with_vec() {
 }
 
 #[test]
-fn test_vec() {
-    let original = vec![1, 2, 3, 4, 5];
-
-    let modified = vec![
-        vec![1, 2, 3, 4, 42],
-        vec![1, 2, 3, 4],
-        vec![],
-        vec![42; 2_usize.pow(4)],
-        vec![],
-        vec![],
-        vec![1, 2, 3, 4, 42],
-        vec![1, 2, 3],
-        vec![1],
-    ];
-
-    test_routine(original, modified);
-}
-
-#[test]
-fn test_nested_list_of_u64() {
-    let original: Vec<Vec<u64>> = vec![vec![1]];
-
-    let modified = vec![
-        vec![vec![1]],
-        vec![vec![1], vec![2]],
-        vec![vec![1], vec![3], vec![4]],
-        vec![],
-        vec![vec![1], vec![3], vec![4]],
-        vec![],
-        vec![vec![1, 2], vec![3], vec![4, 5, 6, 7, 8]],
-        vec![],
-        vec![vec![1], vec![2], vec![3]],
-        vec![vec![1, 2, 3, 4, 5, 6], vec![1, 2, 3, 4, 5, 6, 7]],
-        vec![vec![], vec![], vec![]],
-        vec![vec![0, 0, 0], vec![0], vec![0]],
-    ];
-
-    test_routine(original, modified);
-}
-
-#[test]
-fn test_list_of_struct_with_vec() {
+fn test_vec_of_struct_with_vec() {
     let a = StructWithVec {
         a: 42,
         b: Inner {
@@ -211,18 +209,99 @@ fn test_list_of_struct_with_vec() {
     };
     let d = StructWithVec { a: 0, ..a.clone() };
 
-    let original: Vec<StructWithVec> = vec![a.clone(), c.clone()];
+    // let original: Vec<StructWithVec> = vec![a.clone(), c.clone()];
+    let original: Vec<StructWithVec> = vec![a.clone()];
 
     let modified = vec![
         vec![a.clone(), c.clone()],
         vec![a.clone(), b.clone(), c.clone(), d.clone()],
         vec![b.clone(), a.clone(), c.clone(), d.clone()],
         vec![],
+        vec![a.clone()],
+        vec![a.clone(), b.clone(), c.clone(), d.clone()],
     ];
 
     test_routine(original, modified);
 }
 
+#[derive(Clone, Debug, TreeHash, CachedTreeHashSubTree)]
+pub struct StructWithVecOfStructs {
+    pub a: u64,
+    pub b: Inner,
+    pub c: Vec<Inner>,
+}
+
+#[test]
+fn test_struct_with_vec_of_structs() {
+    let inner_a = Inner {
+        a: 12,
+        b: 13,
+        c: 14,
+        d: 15,
+    };
+
+    let inner_b = Inner {
+        a: 99,
+        b: 100,
+        c: 101,
+        d: 102,
+    };
+
+    let inner_c = Inner {
+        a: 255,
+        b: 256,
+        c: 257,
+        d: 0,
+    };
+
+    let a = StructWithVecOfStructs {
+        a: 42,
+        b: inner_a.clone(),
+        c: vec![inner_a.clone(), inner_b.clone(), inner_c.clone()],
+    };
+
+    let b = StructWithVecOfStructs {
+        c: vec![],
+        ..a.clone()
+    };
+
+    let c = StructWithVecOfStructs {
+        a: 800,
+        ..a.clone()
+    };
+
+    let d = StructWithVecOfStructs {
+        b: inner_c.clone(),
+        ..a.clone()
+    };
+
+    let e = StructWithVecOfStructs {
+        c: vec![inner_a.clone(), inner_b.clone()],
+        ..a.clone()
+    };
+
+    let f = StructWithVecOfStructs {
+        c: vec![inner_a.clone()],
+        ..a.clone()
+    };
+
+    let variants = vec![
+        a.clone(),
+        b.clone(),
+        c.clone(),
+        d.clone(),
+        e.clone(),
+        f.clone(),
+    ];
+
+    test_routine(a, variants.clone());
+    test_routine(b, variants.clone());
+    test_routine(c, variants.clone());
+    test_routine(d, variants.clone());
+    test_routine(e, variants.clone());
+    test_routine(f, variants);
+}
+
 #[derive(Clone, Debug, TreeHash, CachedTreeHashSubTree)]
 pub struct Inner {
     pub a: u64,
@@ -231,80 +310,6 @@ pub struct Inner {
     pub d: u64,
 }
 
-/*
-impl TreeHash for Inner {
-    fn tree_hash_type() -> TreeHashType {
-        TreeHashType::Container
-    }
-
-    fn tree_hash_packed_encoding(&self) -> Vec<u8> {
-        unreachable!("Struct should never be packed.")
-    }
-
-    fn tree_hash_packing_factor() -> usize {
-        unreachable!("Struct should never be packed.")
-    }
-
-    fn tree_hash_root(&self) -> Vec<u8> {
-        let mut leaves = Vec::with_capacity(4 * HASHSIZE);
-
-        leaves.append(&mut self.a.tree_hash_root());
-        leaves.append(&mut self.b.tree_hash_root());
-        leaves.append(&mut self.c.tree_hash_root());
-        leaves.append(&mut self.d.tree_hash_root());
-
-        efficient_merkleize(&leaves)[0..32].to_vec()
-    }
-}
-
-impl CachedTreeHashSubTree<Inner> for Inner {
-    fn new_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
-        let tree = TreeHashCache::from_leaves_and_subtrees(
-            self,
-            vec![
-                self.a.new_tree_hash_cache()?,
-                self.b.new_tree_hash_cache()?,
-                self.c.new_tree_hash_cache()?,
-                self.d.new_tree_hash_cache()?,
-            ],
-        )?;
-
-        Ok(tree)
-    }
-
-    fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
-        let mut lengths = vec![];
-
-        lengths.push(BTreeOverlay::new(&self.a, 0)?.num_nodes());
-        lengths.push(BTreeOverlay::new(&self.b, 0)?.num_nodes());
-        lengths.push(BTreeOverlay::new(&self.c, 0)?.num_nodes());
-        lengths.push(BTreeOverlay::new(&self.d, 0)?.num_nodes());
-
-        BTreeOverlay::from_lengths(chunk_offset, 4, lengths)
-    }
-
-    fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
-        let overlay = BTreeOverlay::new(self, cache.chunk_index)?;
-
-        // Skip the chunk index to the first leaf node of this struct.
-        cache.chunk_index = overlay.first_leaf_node();
-        // Skip the overlay index to the first leaf node of this struct.
-        cache.overlay_index += 1;
-
-        // Recurse into the struct items, updating their caches.
-        self.a.update_tree_hash_cache(cache)?;
-        self.b.update_tree_hash_cache(cache)?;
-        self.c.update_tree_hash_cache(cache)?;
-        self.d.update_tree_hash_cache(cache)?;
-
-        // Iterate through the internal nodes, updating them if their children have changed.
-        cache.update_internal_nodes(&overlay)?;
-
-        Ok(())
-    }
-}
-*/
-
 fn generic_test(index: usize) {
     let inner = Inner {
         a: 1,
diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs
index c2b56c19b..38a72f4fa 100644
--- a/eth2/utils/tree_hash_derive/src/lib.rs
+++ b/eth2/utils/tree_hash_derive/src/lib.rs
@@ -77,7 +77,7 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
                 let mut lengths = vec![];
 
                 #(
-                    lengths.push(tree_hash::BTreeOverlay::new(&self.#idents_b, 0, depth)?.num_nodes());
+                    lengths.push(tree_hash::BTreeOverlay::new(&self.#idents_b, 0, depth)?.num_chunks());
                 )*
 
                 tree_hash::BTreeOverlay::from_lengths(chunk_offset, #num_items, depth, lengths)
@@ -97,7 +97,10 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
                 )*
 
                 // Iterate through the internal nodes, updating them if their children have changed.
+                dbg!("START");
+                dbg!(overlay.offset);
                 cache.update_internal_nodes(&overlay)?;
+                dbg!("END");
 
                 Ok(())
             }

From 7563755b155868056e17a311e917f5bb68afad0a Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 24 Apr 2019 18:13:37 +1000
Subject: [PATCH 099/137] Fix mix-in-length issue

---
 eth2/utils/tree_hash/src/cached_tree_hash.rs  | 302 +---------------
 .../tree_hash/src/cached_tree_hash/impls.rs   |  43 ++-
 .../src/cached_tree_hash/impls/vec.rs         |  39 +--
 .../src/cached_tree_hash/tree_hash_cache.rs   | 329 ++++++++++++++++++
 eth2/utils/tree_hash/tests/tests.rs           |   6 +-
 eth2/utils/tree_hash_derive/src/lib.rs        |  11 +-
 6 files changed, 400 insertions(+), 330 deletions(-)
 create mode 100644 eth2/utils/tree_hash/src/cached_tree_hash/tree_hash_cache.rs

diff --git a/eth2/utils/tree_hash/src/cached_tree_hash.rs b/eth2/utils/tree_hash/src/cached_tree_hash.rs
index 46190ff3c..66ccbb680 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash.rs
@@ -6,8 +6,10 @@ use std::ops::Range;
 pub mod btree_overlay;
 pub mod impls;
 pub mod resize;
+pub mod tree_hash_cache;
 
 pub use btree_overlay::BTreeOverlay;
+pub use tree_hash_cache::TreeHashCache;
 
 #[derive(Debug, PartialEq)]
 pub struct CachedTreeHasher {
@@ -79,6 +81,8 @@ pub trait CachedTreeHashSubTree<Item>: TreeHash {
         depth: usize,
     ) -> Result<BTreeOverlay, Error>;
 
+    fn num_tree_hash_cache_chunks(&self) -> usize;
+
     fn new_tree_hash_cache(&self, depth: usize) -> Result<TreeHashCache, Error>;
 
     fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error>;
@@ -158,301 +162,3 @@ fn num_unsanitized_leaves(num_bytes: usize) -> usize {
 fn num_bytes(num_leaves: usize) -> usize {
     num_leaves * HASHSIZE
 }
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct TreeHashCache {
-    cache: Vec<u8>,
-    chunk_modified: Vec<bool>,
-    overlays: Vec<BTreeOverlay>,
-
-    pub chunk_index: usize,
-    pub overlay_index: usize,
-}
-
-impl Into<Vec<u8>> for TreeHashCache {
-    fn into(self) -> Vec<u8> {
-        self.cache
-    }
-}
-
-impl TreeHashCache {
-    pub fn new<T>(item: &T, depth: usize) -> Result<Self, Error>
-    where
-        T: CachedTreeHashSubTree<T>,
-    {
-        item.new_tree_hash_cache(depth)
-    }
-
-    pub fn from_leaves_and_subtrees<T>(
-        item: &T,
-        leaves_and_subtrees: Vec<Self>,
-        depth: usize,
-    ) -> Result<Self, Error>
-    where
-        T: CachedTreeHashSubTree<T>,
-    {
-        let overlay = BTreeOverlay::new(item, 0, depth)?;
-
-        // Note how many leaves were provided. If is not a power-of-two, we'll need to pad it out
-        // later.
-        let num_provided_leaf_nodes = leaves_and_subtrees.len();
-
-        // Allocate enough bytes to store the internal nodes and the leaves and subtrees, then fill
-        // all the to-be-built internal nodes with zeros and append the leaves and subtrees.
-        let internal_node_bytes = overlay.num_internal_nodes() * BYTES_PER_CHUNK;
-        let leaves_and_subtrees_bytes = leaves_and_subtrees
-            .iter()
-            .fold(0, |acc, t| acc + t.bytes_len());
-        let mut cache = Vec::with_capacity(leaves_and_subtrees_bytes + internal_node_bytes);
-        cache.resize(internal_node_bytes, 0);
-
-        // Allocate enough bytes to store all the leaves.
-        let mut leaves = Vec::with_capacity(overlay.num_leaf_nodes() * HASHSIZE);
-        let mut overlays = Vec::with_capacity(leaves_and_subtrees.len());
-
-        if T::tree_hash_type() == TreeHashType::List {
-            overlays.push(overlay);
-        }
-
-        // Iterate through all of the leaves/subtrees, adding their root as a leaf node and then
-        // concatenating their merkle trees.
-        for t in leaves_and_subtrees {
-            leaves.append(&mut t.root()?.to_vec());
-
-            let (mut bytes, _bools, mut t_overlays) = t.into_components();
-            cache.append(&mut bytes);
-            overlays.append(&mut t_overlays);
-        }
-
-        // Pad the leaves to an even power-of-two, using zeros.
-        pad_for_leaf_count(num_provided_leaf_nodes, &mut cache);
-
-        // Merkleize the leaves, then split the leaf nodes off them. Then, replace all-zeros
-        // internal nodes created earlier with the internal nodes generated by `merkleize`.
-        let mut merkleized = merkleize(leaves);
-        merkleized.split_off(internal_node_bytes);
-        cache.splice(0..internal_node_bytes, merkleized);
-
-        Ok(Self {
-            chunk_modified: vec![false; cache.len() / BYTES_PER_CHUNK],
-            cache,
-            overlays,
-            chunk_index: 0,
-            overlay_index: 0,
-        })
-    }
-
-    pub fn from_bytes(
-        bytes: Vec<u8>,
-        initial_modified_state: bool,
-        overlay: Option<BTreeOverlay>,
-    ) -> Result<Self, Error> {
-        if bytes.len() % BYTES_PER_CHUNK > 0 {
-            return Err(Error::BytesAreNotEvenChunks(bytes.len()));
-        }
-
-        let overlays = match overlay {
-            Some(overlay) => vec![overlay],
-            None => vec![],
-        };
-
-        Ok(Self {
-            chunk_modified: vec![initial_modified_state; bytes.len() / BYTES_PER_CHUNK],
-            cache: bytes,
-            overlays,
-            chunk_index: 0,
-            overlay_index: 0,
-        })
-    }
-
-    pub fn get_overlay(
-        &self,
-        overlay_index: usize,
-        chunk_index: usize,
-    ) -> Result<BTreeOverlay, Error> {
-        let mut overlay = self
-            .overlays
-            .get(overlay_index)
-            .ok_or_else(|| Error::NoOverlayForIndex(overlay_index))?
-            .clone();
-
-        overlay.offset = chunk_index;
-
-        Ok(overlay)
-    }
-
-    pub fn reset_modifications(&mut self) {
-        for chunk_modified in &mut self.chunk_modified {
-            *chunk_modified = false;
-        }
-    }
-
-    pub fn replace_overlay(
-        &mut self,
-        overlay_index: usize,
-        chunk_index: usize,
-        new_overlay: BTreeOverlay,
-    ) -> Result<BTreeOverlay, Error> {
-        let old_overlay = self.get_overlay(overlay_index, chunk_index)?;
-
-        // If the merkle tree required to represent the new list is of a different size to the one
-        // required for the previous list, then update our cache.
-        //
-        // This grows/shrinks the bytes to accomodate the new tree, preserving as much of the tree
-        // as possible.
-        if new_overlay.num_leaf_nodes() != old_overlay.num_leaf_nodes() {
-            // Get slices of the exsiting tree from the cache.
-            let (old_bytes, old_flags) = self
-                .slices(old_overlay.chunk_range())
-                .ok_or_else(|| Error::UnableToObtainSlices)?;
-
-            let (new_bytes, new_bools) =
-                if new_overlay.num_leaf_nodes() > old_overlay.num_leaf_nodes() {
-                    resize::grow_merkle_cache(
-                        old_bytes,
-                        old_flags,
-                        old_overlay.height(),
-                        new_overlay.height(),
-                    )
-                    .ok_or_else(|| Error::UnableToGrowMerkleTree)?
-                } else {
-                    resize::shrink_merkle_cache(
-                        old_bytes,
-                        old_flags,
-                        old_overlay.height(),
-                        new_overlay.height(),
-                        new_overlay.num_chunks(),
-                    )
-                    .ok_or_else(|| Error::UnableToShrinkMerkleTree)?
-                };
-
-            // Splice the newly created `TreeHashCache` over the existing elements.
-            self.splice(old_overlay.chunk_range(), new_bytes, new_bools);
-        }
-
-        Ok(std::mem::replace(
-            &mut self.overlays[overlay_index],
-            new_overlay,
-        ))
-    }
-
-    pub fn remove_proceeding_child_overlays(&mut self, overlay_index: usize, depth: usize) {
-        let end = self
-            .overlays
-            .iter()
-            .skip(overlay_index)
-            .position(|o| o.depth <= depth)
-            .unwrap_or_else(|| self.overlays.len());
-
-        self.overlays.splice(overlay_index..end, vec![]);
-    }
-
-    pub fn update_internal_nodes(&mut self, overlay: &BTreeOverlay) -> Result<(), Error> {
-        for (parent, children) in overlay.internal_parents_and_children().into_iter().rev() {
-            if self.either_modified(children)? {
-                self.modify_chunk(parent, &self.hash_children(children)?)?;
-            }
-        }
-
-        Ok(())
-    }
-
-    fn bytes_len(&self) -> usize {
-        self.cache.len()
-    }
-
-    pub fn root(&self) -> Result<&[u8], Error> {
-        self.cache
-            .get(0..HASHSIZE)
-            .ok_or_else(|| Error::NoBytesForRoot)
-    }
-
-    fn splice(&mut self, chunk_range: Range<usize>, bytes: Vec<u8>, bools: Vec<bool>) {
-        // Update the `chunk_modified` vec, marking all spliced-in nodes as changed.
-        self.chunk_modified.splice(chunk_range.clone(), bools);
-        self.cache
-            .splice(node_range_to_byte_range(&chunk_range), bytes);
-    }
-
-    pub fn maybe_update_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> {
-        let start = chunk * BYTES_PER_CHUNK;
-        let end = start + BYTES_PER_CHUNK;
-
-        if !self.chunk_equals(chunk, to)? {
-            self.cache
-                .get_mut(start..end)
-                .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))?
-                .copy_from_slice(to);
-            self.chunk_modified[chunk] = true;
-        }
-
-        Ok(())
-    }
-
-    fn slices(&self, chunk_range: Range<usize>) -> Option<(&[u8], &[bool])> {
-        Some((
-            self.cache.get(node_range_to_byte_range(&chunk_range))?,
-            self.chunk_modified.get(chunk_range)?,
-        ))
-    }
-
-    fn modify_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> {
-        let start = chunk * BYTES_PER_CHUNK;
-        let end = start + BYTES_PER_CHUNK;
-
-        self.cache
-            .get_mut(start..end)
-            .ok_or_else(|| Error::NoBytesForChunk(chunk))?
-            .copy_from_slice(to);
-
-        self.chunk_modified[chunk] = true;
-
-        Ok(())
-    }
-
-    fn get_chunk(&self, chunk: usize) -> Result<&[u8], Error> {
-        let start = chunk * BYTES_PER_CHUNK;
-        let end = start + BYTES_PER_CHUNK;
-
-        Ok(self
-            .cache
-            .get(start..end)
-            .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))?)
-    }
-
-    fn chunk_equals(&mut self, chunk: usize, other: &[u8]) -> Result<bool, Error> {
-        Ok(self.get_chunk(chunk)? == other)
-    }
-
-    pub fn changed(&self, chunk: usize) -> Result<bool, Error> {
-        self.chunk_modified
-            .get(chunk)
-            .cloned()
-            .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))
-    }
-
-    fn either_modified(&self, children: (usize, usize)) -> Result<bool, Error> {
-        Ok(self.changed(children.0)? | self.changed(children.1)?)
-    }
-
-    fn hash_children(&self, children: (usize, usize)) -> Result<Vec<u8>, Error> {
-        let mut child_bytes = Vec::with_capacity(BYTES_PER_CHUNK * 2);
-        child_bytes.append(&mut self.get_chunk(children.0)?.to_vec());
-        child_bytes.append(&mut self.get_chunk(children.1)?.to_vec());
-
-        Ok(hash(&child_bytes))
-    }
-
-    pub fn mix_in_length(&self, chunk: usize, length: usize) -> Result<Vec<u8>, Error> {
-        let mut bytes = Vec::with_capacity(2 * BYTES_PER_CHUNK);
-
-        bytes.append(&mut self.get_chunk(chunk)?.to_vec());
-        bytes.append(&mut int_to_bytes32(length as u64));
-
-        Ok(hash(&bytes))
-    }
-
-    pub fn into_components(self) -> (Vec<u8>, Vec<bool>, Vec<BTreeOverlay>) {
-        (self.cache, self.chunk_modified, self.overlays)
-    }
-}
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
index 42d77d11d..34902f062 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
@@ -3,21 +3,58 @@ use super::*;
 mod vec;
 
 impl CachedTreeHashSubTree<u64> for u64 {
-    fn new_tree_hash_cache(&self, depth: usize) -> Result<TreeHashCache, Error> {
+    fn new_tree_hash_cache(&self, _depth: usize) -> Result<TreeHashCache, Error> {
         Ok(TreeHashCache::from_bytes(
             merkleize(self.to_le_bytes().to_vec()),
             false,
-            // self.tree_hash_cache_overlay(0, depth)?,
             None,
         )?)
     }
 
+    fn num_tree_hash_cache_chunks(&self) -> usize {
+        1
+    }
+
     fn tree_hash_cache_overlay(
         &self,
         chunk_offset: usize,
         depth: usize,
     ) -> Result<BTreeOverlay, Error> {
-        BTreeOverlay::from_lengths(chunk_offset, 1, depth, vec![1])
+        panic!("Basic should not produce overlay");
+        // BTreeOverlay::from_lengths(chunk_offset, 1, depth, vec![1])
+    }
+
+    fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
+        let leaf = merkleize(self.to_le_bytes().to_vec());
+        cache.maybe_update_chunk(cache.chunk_index, &leaf)?;
+
+        cache.chunk_index += 1;
+        // cache.overlay_index += 1;
+
+        Ok(())
+    }
+}
+
+impl CachedTreeHashSubTree<usize> for usize {
+    fn new_tree_hash_cache(&self, _depth: usize) -> Result<TreeHashCache, Error> {
+        Ok(TreeHashCache::from_bytes(
+            merkleize(self.to_le_bytes().to_vec()),
+            false,
+            None,
+        )?)
+    }
+
+    fn num_tree_hash_cache_chunks(&self) -> usize {
+        1
+    }
+
+    fn tree_hash_cache_overlay(
+        &self,
+        chunk_offset: usize,
+        depth: usize,
+    ) -> Result<BTreeOverlay, Error> {
+        panic!("Basic should not produce overlay");
+        // BTreeOverlay::from_lengths(chunk_offset, 1, depth, vec![1])
     }
 
     fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
index 5ab7d06e4..fc43cc9b8 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
@@ -5,7 +5,7 @@ where
     T: CachedTreeHashSubTree<T> + TreeHash,
 {
     fn new_tree_hash_cache(&self, depth: usize) -> Result<TreeHashCache, Error> {
-        let overlay = self.tree_hash_cache_overlay(0, depth)?;
+        let mut overlay = self.tree_hash_cache_overlay(0, depth)?;
 
         let mut cache = match T::tree_hash_type() {
             TreeHashType::Basic => TreeHashCache::from_bytes(
@@ -23,13 +23,18 @@ where
             }
         }?;
 
-        // Mix in the length of the list.
-        let root_node = overlay.root();
-        cache.modify_chunk(root_node, &cache.mix_in_length(root_node, self.len())?)?;
+        cache.add_length_nodes(overlay.chunk_range(), self.len())?;
 
         Ok(cache)
     }
 
+    fn num_tree_hash_cache_chunks(&self) -> usize {
+        BTreeOverlay::new(self, 0, 0)
+            .and_then(|o| Ok(o.num_chunks()))
+            .unwrap_or_else(|_| 1)
+            + 2
+    }
+
     fn tree_hash_cache_overlay(
         &self,
         chunk_offset: usize,
@@ -48,7 +53,7 @@ where
                 let mut lengths = vec![];
 
                 for item in self {
-                    lengths.push(BTreeOverlay::new(item, 0, depth)?.num_chunks())
+                    lengths.push(item.num_tree_hash_cache_chunks())
                 }
 
                 // Disallow zero-length as an empty list still has one all-padding node.
@@ -64,6 +69,9 @@ where
     }
 
     fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
+        // Skip the length-mixed-in root node.
+        cache.chunk_index += 1;
+
         let old_overlay = cache.get_overlay(cache.overlay_index, cache.chunk_index)?;
         let new_overlay = BTreeOverlay::new(self, cache.chunk_index, old_overlay.depth)?;
 
@@ -113,9 +121,6 @@ where
                         // The item existed in the previous list and exists in the current list.
                         (Some(_old), Some(new)) => {
                             cache.chunk_index = new.start;
-                            if cache.chunk_index + 1 < cache.chunk_modified.len() {
-                                cache.chunk_modified[cache.chunk_index + 1] = true;
-                            }
 
                             self[i].update_tree_hash_cache(cache)?;
                         }
@@ -169,21 +174,11 @@ where
 
         cache.update_internal_nodes(&new_overlay)?;
 
-        // Mix in length.
-        let root_node = new_overlay.root();
-        if cache.changed(root_node)? {
-            cache.modify_chunk(root_node, &cache.mix_in_length(root_node, self.len())?)?;
-        } else if old_overlay.num_items != new_overlay.num_items {
-            if new_overlay.num_internal_nodes() == 0 {
-                cache.modify_chunk(root_node, &cache.mix_in_length(root_node, self.len())?)?;
-            } else {
-                let children = new_overlay.child_chunks(0);
-                cache.modify_chunk(root_node, &cache.hash_children(children)?)?;
-                cache.modify_chunk(root_node, &cache.mix_in_length(root_node, self.len())?)?;
-            }
-        }
+        // Mix in length
+        cache.mix_in_length(new_overlay.chunk_range(), self.len())?;
 
-        cache.chunk_index = new_overlay.next_node();
+        // Skip an extra node to clear the length node.
+        cache.chunk_index = new_overlay.next_node() + 1;
 
         Ok(())
     }
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/tree_hash_cache.rs b/eth2/utils/tree_hash/src/cached_tree_hash/tree_hash_cache.rs
new file mode 100644
index 000000000..8fa08e306
--- /dev/null
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/tree_hash_cache.rs
@@ -0,0 +1,329 @@
+use super::*;
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct TreeHashCache {
+    pub cache: Vec<u8>,
+    pub chunk_modified: Vec<bool>,
+    pub overlays: Vec<BTreeOverlay>,
+
+    pub chunk_index: usize,
+    pub overlay_index: usize,
+}
+
+impl Into<Vec<u8>> for TreeHashCache {
+    fn into(self) -> Vec<u8> {
+        self.cache
+    }
+}
+
+impl TreeHashCache {
+    pub fn new<T>(item: &T, depth: usize) -> Result<Self, Error>
+    where
+        T: CachedTreeHashSubTree<T>,
+    {
+        item.new_tree_hash_cache(depth)
+    }
+
+    pub fn from_leaves_and_subtrees<T>(
+        item: &T,
+        leaves_and_subtrees: Vec<Self>,
+        depth: usize,
+    ) -> Result<Self, Error>
+    where
+        T: CachedTreeHashSubTree<T>,
+    {
+        let overlay = BTreeOverlay::new(item, 0, depth)?;
+
+        // Note how many leaves were provided. If is not a power-of-two, we'll need to pad it out
+        // later.
+        let num_provided_leaf_nodes = leaves_and_subtrees.len();
+
+        // Allocate enough bytes to store the internal nodes and the leaves and subtrees, then fill
+        // all the to-be-built internal nodes with zeros and append the leaves and subtrees.
+        let internal_node_bytes = overlay.num_internal_nodes() * BYTES_PER_CHUNK;
+        let leaves_and_subtrees_bytes = leaves_and_subtrees
+            .iter()
+            .fold(0, |acc, t| acc + t.bytes_len());
+        let mut cache = Vec::with_capacity(leaves_and_subtrees_bytes + internal_node_bytes);
+        cache.resize(internal_node_bytes, 0);
+
+        // Allocate enough bytes to store all the leaves.
+        let mut leaves = Vec::with_capacity(overlay.num_leaf_nodes() * HASHSIZE);
+        let mut overlays = Vec::with_capacity(leaves_and_subtrees.len());
+
+        if T::tree_hash_type() == TreeHashType::List {
+            overlays.push(overlay);
+        }
+
+        // Iterate through all of the leaves/subtrees, adding their root as a leaf node and then
+        // concatenating their merkle trees.
+        for t in leaves_and_subtrees {
+            leaves.append(&mut t.root()?.to_vec());
+
+            let (mut bytes, _bools, mut t_overlays) = t.into_components();
+            cache.append(&mut bytes);
+            overlays.append(&mut t_overlays);
+        }
+
+        // Pad the leaves to an even power-of-two, using zeros.
+        pad_for_leaf_count(num_provided_leaf_nodes, &mut cache);
+
+        // Merkleize the leaves, then split the leaf nodes off them. Then, replace all-zeros
+        // internal nodes created earlier with the internal nodes generated by `merkleize`.
+        let mut merkleized = merkleize(leaves);
+        merkleized.split_off(internal_node_bytes);
+        cache.splice(0..internal_node_bytes, merkleized);
+
+        Ok(Self {
+            chunk_modified: vec![false; cache.len() / BYTES_PER_CHUNK],
+            cache,
+            overlays,
+            chunk_index: 0,
+            overlay_index: 0,
+        })
+    }
+
+    pub fn from_bytes(
+        bytes: Vec<u8>,
+        initial_modified_state: bool,
+        overlay: Option<BTreeOverlay>,
+    ) -> Result<Self, Error> {
+        if bytes.len() % BYTES_PER_CHUNK > 0 {
+            return Err(Error::BytesAreNotEvenChunks(bytes.len()));
+        }
+
+        let overlays = match overlay {
+            Some(overlay) => vec![overlay],
+            None => vec![],
+        };
+
+        Ok(Self {
+            chunk_modified: vec![initial_modified_state; bytes.len() / BYTES_PER_CHUNK],
+            cache: bytes,
+            overlays,
+            chunk_index: 0,
+            overlay_index: 0,
+        })
+    }
+
+    pub fn get_overlay(
+        &self,
+        overlay_index: usize,
+        chunk_index: usize,
+    ) -> Result<BTreeOverlay, Error> {
+        let mut overlay = self
+            .overlays
+            .get(overlay_index)
+            .ok_or_else(|| Error::NoOverlayForIndex(overlay_index))?
+            .clone();
+
+        overlay.offset = chunk_index;
+
+        Ok(overlay)
+    }
+
+    pub fn reset_modifications(&mut self) {
+        for chunk_modified in &mut self.chunk_modified {
+            *chunk_modified = false;
+        }
+    }
+
+    pub fn replace_overlay(
+        &mut self,
+        overlay_index: usize,
+        chunk_index: usize,
+        new_overlay: BTreeOverlay,
+    ) -> Result<BTreeOverlay, Error> {
+        let old_overlay = self.get_overlay(overlay_index, chunk_index)?;
+
+        // If the merkle tree required to represent the new list is of a different size to the one
+        // required for the previous list, then update our cache.
+        //
+        // This grows/shrinks the bytes to accomodate the new tree, preserving as much of the tree
+        // as possible.
+        if new_overlay.num_leaf_nodes() != old_overlay.num_leaf_nodes() {
+            // Get slices of the exsiting tree from the cache.
+            let (old_bytes, old_flags) = self
+                .slices(old_overlay.chunk_range())
+                .ok_or_else(|| Error::UnableToObtainSlices)?;
+
+            let (new_bytes, new_bools) =
+                if new_overlay.num_leaf_nodes() > old_overlay.num_leaf_nodes() {
+                    resize::grow_merkle_cache(
+                        old_bytes,
+                        old_flags,
+                        old_overlay.height(),
+                        new_overlay.height(),
+                    )
+                    .ok_or_else(|| Error::UnableToGrowMerkleTree)?
+                } else {
+                    resize::shrink_merkle_cache(
+                        old_bytes,
+                        old_flags,
+                        old_overlay.height(),
+                        new_overlay.height(),
+                        new_overlay.num_chunks(),
+                    )
+                    .ok_or_else(|| Error::UnableToShrinkMerkleTree)?
+                };
+
+            // Splice the newly created `TreeHashCache` over the existing elements.
+            self.splice(old_overlay.chunk_range(), new_bytes, new_bools);
+        }
+
+        Ok(std::mem::replace(
+            &mut self.overlays[overlay_index],
+            new_overlay,
+        ))
+    }
+
+    pub fn remove_proceeding_child_overlays(&mut self, overlay_index: usize, depth: usize) {
+        let end = self
+            .overlays
+            .iter()
+            .skip(overlay_index)
+            .position(|o| o.depth <= depth)
+            .unwrap_or_else(|| self.overlays.len());
+
+        self.overlays.splice(overlay_index..end, vec![]);
+    }
+
+    pub fn update_internal_nodes(&mut self, overlay: &BTreeOverlay) -> Result<(), Error> {
+        for (parent, children) in overlay.internal_parents_and_children().into_iter().rev() {
+            if self.either_modified(children)? {
+                self.modify_chunk(parent, &self.hash_children(children)?)?;
+            }
+        }
+
+        Ok(())
+    }
+
+    fn bytes_len(&self) -> usize {
+        self.cache.len()
+    }
+
+    pub fn root(&self) -> Result<&[u8], Error> {
+        self.cache
+            .get(0..HASHSIZE)
+            .ok_or_else(|| Error::NoBytesForRoot)
+    }
+
+    pub fn splice(&mut self, chunk_range: Range<usize>, bytes: Vec<u8>, bools: Vec<bool>) {
+        // Update the `chunk_modified` vec, marking all spliced-in nodes as changed.
+        self.chunk_modified.splice(chunk_range.clone(), bools);
+        self.cache
+            .splice(node_range_to_byte_range(&chunk_range), bytes);
+    }
+
+    pub fn maybe_update_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> {
+        let start = chunk * BYTES_PER_CHUNK;
+        let end = start + BYTES_PER_CHUNK;
+
+        if !self.chunk_equals(chunk, to)? {
+            self.cache
+                .get_mut(start..end)
+                .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))?
+                .copy_from_slice(to);
+            self.chunk_modified[chunk] = true;
+        }
+
+        Ok(())
+    }
+
+    fn slices(&self, chunk_range: Range<usize>) -> Option<(&[u8], &[bool])> {
+        Some((
+            self.cache.get(node_range_to_byte_range(&chunk_range))?,
+            self.chunk_modified.get(chunk_range)?,
+        ))
+    }
+
+    pub fn modify_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> {
+        let start = chunk * BYTES_PER_CHUNK;
+        let end = start + BYTES_PER_CHUNK;
+
+        self.cache
+            .get_mut(start..end)
+            .ok_or_else(|| Error::NoBytesForChunk(chunk))?
+            .copy_from_slice(to);
+
+        self.chunk_modified[chunk] = true;
+
+        Ok(())
+    }
+
+    fn get_chunk(&self, chunk: usize) -> Result<&[u8], Error> {
+        let start = chunk * BYTES_PER_CHUNK;
+        let end = start + BYTES_PER_CHUNK;
+
+        Ok(self
+            .cache
+            .get(start..end)
+            .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))?)
+    }
+
+    fn chunk_equals(&mut self, chunk: usize, other: &[u8]) -> Result<bool, Error> {
+        Ok(self.get_chunk(chunk)? == other)
+    }
+
+    pub fn changed(&self, chunk: usize) -> Result<bool, Error> {
+        self.chunk_modified
+            .get(chunk)
+            .cloned()
+            .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))
+    }
+
+    fn either_modified(&self, children: (usize, usize)) -> Result<bool, Error> {
+        Ok(self.changed(children.0)? | self.changed(children.1)?)
+    }
+
+    pub fn hash_children(&self, children: (usize, usize)) -> Result<Vec<u8>, Error> {
+        let mut child_bytes = Vec::with_capacity(BYTES_PER_CHUNK * 2);
+        child_bytes.append(&mut self.get_chunk(children.0)?.to_vec());
+        child_bytes.append(&mut self.get_chunk(children.1)?.to_vec());
+
+        Ok(hash(&child_bytes))
+    }
+
+    pub fn add_length_nodes(
+        &mut self,
+        chunk_range: Range<usize>,
+        length: usize,
+    ) -> Result<(), Error> {
+        self.chunk_modified[chunk_range.start] = true;
+
+        let byte_range = node_range_to_byte_range(&chunk_range);
+
+        // Add the last node.
+        self.cache
+            .splice(byte_range.end..byte_range.end, vec![0; HASHSIZE]);
+        self.chunk_modified
+            .splice(chunk_range.end..chunk_range.end, vec![false]);
+
+        // Add the first node.
+        self.cache
+            .splice(byte_range.start..byte_range.start, vec![0; HASHSIZE]);
+        self.chunk_modified
+            .splice(chunk_range.start..chunk_range.start, vec![false]);
+
+        self.mix_in_length(chunk_range.start + 1..chunk_range.end + 1, length)?;
+
+        Ok(())
+    }
+
+    pub fn mix_in_length(&mut self, chunk_range: Range<usize>, length: usize) -> Result<(), Error> {
+        // Update the length chunk.
+        self.maybe_update_chunk(chunk_range.end, &int_to_bytes32(length as u64))?;
+
+        // Update the mixed-in root if the main root or the length have changed.
+        let children = (chunk_range.start, chunk_range.end);
+        if self.either_modified(children)? {
+            self.modify_chunk(chunk_range.start - 1, &self.hash_children(children)?)?;
+        }
+
+        Ok(())
+    }
+
+    pub fn into_components(self) -> (Vec<u8>, Vec<bool>, Vec<BTreeOverlay>) {
+        (self.cache, self.chunk_modified, self.overlays)
+    }
+}
diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
index bc3c5538f..cef366da4 100644
--- a/eth2/utils/tree_hash/tests/tests.rs
+++ b/eth2/utils/tree_hash/tests/tests.rs
@@ -74,9 +74,9 @@ fn test_inner() {
 
 #[test]
 fn test_vec() {
-    let original = vec![1, 2, 3, 4, 5];
+    let original: Vec<u64> = vec![1, 2, 3, 4, 5];
 
-    let modified = vec![
+    let modified: Vec<Vec<u64>> = vec![
         vec![1, 2, 3, 4, 42],
         vec![1, 2, 3, 4],
         vec![],
@@ -93,7 +93,7 @@ fn test_vec() {
 
 #[test]
 fn test_nested_list_of_u64() {
-    let original: Vec<Vec<u64>> = vec![vec![1]];
+    let original: Vec<Vec<u64>> = vec![vec![42]];
 
     let modified = vec![
         vec![vec![1]],
diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs
index 38a72f4fa..272ea7e96 100644
--- a/eth2/utils/tree_hash_derive/src/lib.rs
+++ b/eth2/utils/tree_hash_derive/src/lib.rs
@@ -73,11 +73,17 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
                 Ok(tree)
             }
 
+            fn num_tree_hash_cache_chunks(&self) -> usize {
+                tree_hash::BTreeOverlay::new(self, 0, 0)
+                    .and_then(|o| Ok(o.num_chunks()))
+                    .unwrap_or_else(|_| 1)
+            }
+
             fn tree_hash_cache_overlay(&self, chunk_offset: usize, depth: usize) -> Result<tree_hash::BTreeOverlay, tree_hash::Error> {
                 let mut lengths = vec![];
 
                 #(
-                    lengths.push(tree_hash::BTreeOverlay::new(&self.#idents_b, 0, depth)?.num_chunks());
+                    lengths.push(self.#idents_b.num_tree_hash_cache_chunks());
                 )*
 
                 tree_hash::BTreeOverlay::from_lengths(chunk_offset, #num_items, depth, lengths)
@@ -97,10 +103,7 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
                 )*
 
                 // Iterate through the internal nodes, updating them if their children have changed.
-                dbg!("START");
-                dbg!(overlay.offset);
                 cache.update_internal_nodes(&overlay)?;
-                dbg!("END");
 
                 Ok(())
             }

From cab5e59a6fb2ca6931de741e1aa9c77355176f01 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 24 Apr 2019 18:23:58 +1000
Subject: [PATCH 100/137] Rename `CachedTreeHash` trait, add readme

---
 eth2/utils/tree_hash/README.md                | 76 +++++++++++++++++++
 eth2/utils/tree_hash/src/cached_tree_hash.rs  | 14 +---
 .../src/cached_tree_hash/btree_overlay.rs     |  2 +-
 .../tree_hash/src/cached_tree_hash/impls.rs   |  4 +-
 .../src/cached_tree_hash/impls/vec.rs         |  6 +-
 .../src/cached_tree_hash/tree_hash_cache.rs   |  4 +-
 eth2/utils/tree_hash/src/lib.rs               |  2 +-
 eth2/utils/tree_hash/tests/tests.rs           | 18 ++---
 eth2/utils/tree_hash_derive/src/lib.rs        |  6 +-
 eth2/utils/tree_hash_derive/tests/tests.rs    | 10 +--
 10 files changed, 105 insertions(+), 37 deletions(-)
 create mode 100644 eth2/utils/tree_hash/README.md

diff --git a/eth2/utils/tree_hash/README.md b/eth2/utils/tree_hash/README.md
new file mode 100644
index 000000000..3c6e13a2e
--- /dev/null
+++ b/eth2/utils/tree_hash/README.md
@@ -0,0 +1,76 @@
+# Tree hashing
+
+Provides both cached and non-cached tree hashing methods.
+
+## Standard Tree Hash
+
+```rust
+use tree_hash_derive::TreeHash;
+
+#[derive(TreeHash)]
+struct Foo {
+	a: u64,
+	b: Vec<u64>,
+}
+
+fn main() {
+	let foo = Foo {
+		a: 42,
+		b: vec![1, 2, 3]
+	}
+
+	println!("root: {}", foo.tree_hash_root());
+}
+```
+
+## Cached Tree Hash
+
+
+```rust
+use tree_hash_derive::{TreeHash, CachedTreeHash};
+
+#[derive(TreeHash, CachedTreeHash)]
+struct Foo {
+	a: u64,
+	b: Vec<u64>,
+}
+
+#[derive(TreeHash, CachedTreeHash)]
+struct Bar {
+	a: Vec<Foo>,
+	b: u64,
+}
+
+fn main() {
+	let bar = Bar {
+		a: vec![
+			Foo {
+				a: 42,
+				b: vec![1, 2, 3]
+			}
+		],
+		b: 42
+	};
+
+	let modified_bar = Bar {
+		a: vec![
+			Foo {
+				a: 100,
+				b: vec![1, 2, 3, 4, 5, 6]
+			}
+			Foo {
+				a: 42,
+				b: vec![]
+			}
+		],
+		b: 99
+	};
+
+
+    let mut hasher = CachedTreeHasher::new(&bar).unwrap();
+	hasher.update(&modified_bar);
+
+	// Assert that the cached tree hash matches a standard tree hash.
+	assert_eq!(hasher.tree_hash_root(), modified_bar.tree_hash_root());
+}
+```
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash.rs b/eth2/utils/tree_hash/src/cached_tree_hash.rs
index 66ccbb680..0183f5c84 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash.rs
@@ -19,7 +19,7 @@ pub struct CachedTreeHasher {
 impl CachedTreeHasher {
     pub fn new<T>(item: &T) -> Result<Self, Error>
     where
-        T: CachedTreeHashSubTree<T>,
+        T: CachedTreeHash<T>,
     {
         Ok(Self {
             cache: TreeHashCache::new(item, 0)?,
@@ -28,7 +28,7 @@ impl CachedTreeHasher {
 
     pub fn update<T>(&mut self, item: &T) -> Result<(), Error>
     where
-        T: CachedTreeHashSubTree<T>,
+        T: CachedTreeHash<T>,
     {
         // Reset the per-hash counters.
         self.cache.chunk_index = 0;
@@ -66,15 +66,7 @@ pub enum Error {
     NotLeafNode(usize),
 }
 
-pub trait CachedTreeHash<T>: CachedTreeHashSubTree<T> + Sized {
-    fn update_internal_tree_hash_cache(self, old: T) -> Result<(Self, Self), Error>;
-
-    fn cached_tree_hash_root(&self) -> Option<Vec<u8>>;
-
-    fn clone_without_tree_hash_cache(&self) -> Self;
-}
-
-pub trait CachedTreeHashSubTree<Item>: TreeHash {
+pub trait CachedTreeHash<Item>: TreeHash {
     fn tree_hash_cache_overlay(
         &self,
         chunk_offset: usize,
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs b/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
index 463586d40..9fd1251d7 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
@@ -11,7 +11,7 @@ pub struct BTreeOverlay {
 impl BTreeOverlay {
     pub fn new<T>(item: &T, initial_offset: usize, depth: usize) -> Result<Self, Error>
     where
-        T: CachedTreeHashSubTree<T>,
+        T: CachedTreeHash<T>,
     {
         item.tree_hash_cache_overlay(initial_offset, depth)
     }
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
index 34902f062..74ab986cb 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
@@ -2,7 +2,7 @@ use super::*;
 
 mod vec;
 
-impl CachedTreeHashSubTree<u64> for u64 {
+impl CachedTreeHash<u64> for u64 {
     fn new_tree_hash_cache(&self, _depth: usize) -> Result<TreeHashCache, Error> {
         Ok(TreeHashCache::from_bytes(
             merkleize(self.to_le_bytes().to_vec()),
@@ -35,7 +35,7 @@ impl CachedTreeHashSubTree<u64> for u64 {
     }
 }
 
-impl CachedTreeHashSubTree<usize> for usize {
+impl CachedTreeHash<usize> for usize {
     fn new_tree_hash_cache(&self, _depth: usize) -> Result<TreeHashCache, Error> {
         Ok(TreeHashCache::from_bytes(
             merkleize(self.to_le_bytes().to_vec()),
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
index fc43cc9b8..6a0770681 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
@@ -1,8 +1,8 @@
 use super::*;
 
-impl<T> CachedTreeHashSubTree<Vec<T>> for Vec<T>
+impl<T> CachedTreeHash<Vec<T>> for Vec<T>
 where
-    T: CachedTreeHashSubTree<T> + TreeHash,
+    T: CachedTreeHash<T> + TreeHash,
 {
     fn new_tree_hash_cache(&self, depth: usize) -> Result<TreeHashCache, Error> {
         let mut overlay = self.tree_hash_cache_overlay(0, depth)?;
@@ -186,7 +186,7 @@ where
 
 fn get_packed_leaves<T>(vec: &Vec<T>) -> Result<Vec<u8>, Error>
 where
-    T: CachedTreeHashSubTree<T>,
+    T: CachedTreeHash<T>,
 {
     let num_packed_bytes = (BYTES_PER_CHUNK / T::tree_hash_packing_factor()) * vec.len();
     let num_leaves = num_sanitized_leaves(num_packed_bytes);
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/tree_hash_cache.rs b/eth2/utils/tree_hash/src/cached_tree_hash/tree_hash_cache.rs
index 8fa08e306..336d28028 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/tree_hash_cache.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/tree_hash_cache.rs
@@ -19,7 +19,7 @@ impl Into<Vec<u8>> for TreeHashCache {
 impl TreeHashCache {
     pub fn new<T>(item: &T, depth: usize) -> Result<Self, Error>
     where
-        T: CachedTreeHashSubTree<T>,
+        T: CachedTreeHash<T>,
     {
         item.new_tree_hash_cache(depth)
     }
@@ -30,7 +30,7 @@ impl TreeHashCache {
         depth: usize,
     ) -> Result<Self, Error>
     where
-        T: CachedTreeHashSubTree<T>,
+        T: CachedTreeHash<T>,
     {
         let overlay = BTreeOverlay::new(item, 0, depth)?;
 
diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs
index ed60079c8..5aaf2d585 100644
--- a/eth2/utils/tree_hash/src/lib.rs
+++ b/eth2/utils/tree_hash/src/lib.rs
@@ -6,7 +6,7 @@ pub const BYTES_PER_CHUNK: usize = 32;
 pub const HASHSIZE: usize = 32;
 pub const MERKLE_HASH_CHUNCK: usize = 2 * BYTES_PER_CHUNK;
 
-pub use cached_tree_hash::{BTreeOverlay, CachedTreeHashSubTree, Error, TreeHashCache};
+pub use cached_tree_hash::{BTreeOverlay, CachedTreeHash, Error, TreeHashCache};
 pub use signed_root::SignedRoot;
 pub use standard_tree_hash::{merkle_root, TreeHash};
 
diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
index cef366da4..6f339b8f2 100644
--- a/eth2/utils/tree_hash/tests/tests.rs
+++ b/eth2/utils/tree_hash/tests/tests.rs
@@ -1,8 +1,8 @@
 use int_to_bytes::int_to_bytes32;
 use tree_hash::cached_tree_hash::*;
-use tree_hash_derive::{CachedTreeHashSubTree, TreeHash};
+use tree_hash_derive::{CachedTreeHash, TreeHash};
 
-#[derive(Clone, Debug, TreeHash, CachedTreeHashSubTree)]
+#[derive(Clone, Debug, TreeHash, CachedTreeHash)]
 pub struct NestedStruct {
     pub a: u64,
     pub b: Inner,
@@ -10,7 +10,7 @@ pub struct NestedStruct {
 
 fn test_routine<T>(original: T, modified: Vec<T>)
 where
-    T: CachedTreeHashSubTree<T>,
+    T: CachedTreeHash<T>,
 {
     let mut hasher = CachedTreeHasher::new(&original).unwrap();
 
@@ -113,7 +113,7 @@ fn test_nested_list_of_u64() {
     test_routine(original, modified);
 }
 
-#[derive(Clone, Debug, TreeHash, CachedTreeHashSubTree)]
+#[derive(Clone, Debug, TreeHash, CachedTreeHash)]
 pub struct StructWithVec {
     pub a: u64,
     pub b: Inner,
@@ -224,7 +224,7 @@ fn test_vec_of_struct_with_vec() {
     test_routine(original, modified);
 }
 
-#[derive(Clone, Debug, TreeHash, CachedTreeHashSubTree)]
+#[derive(Clone, Debug, TreeHash, CachedTreeHash)]
 pub struct StructWithVecOfStructs {
     pub a: u64,
     pub b: Inner,
@@ -302,7 +302,7 @@ fn test_struct_with_vec_of_structs() {
     test_routine(f, variants);
 }
 
-#[derive(Clone, Debug, TreeHash, CachedTreeHashSubTree)]
+#[derive(Clone, Debug, TreeHash, CachedTreeHash)]
 pub struct Inner {
     pub a: u64,
     pub b: u64,
@@ -481,7 +481,7 @@ fn works_when_embedded() {
     assert_eq!(&merkle[0..32], &root[..]);
 }
 
-impl CachedTreeHashSubTree<InternalCache> for InternalCache {
+impl CachedTreeHash<InternalCache> for InternalCache {
     fn new_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
         let tree = TreeHashCache::from_leaves_and_subtrees(
             self,
@@ -562,7 +562,7 @@ impl TreeHash for Inner {
     }
 }
 
-impl CachedTreeHashSubTree<Inner> for Inner {
+impl CachedTreeHash<Inner> for Inner {
     fn new_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
         let tree = TreeHashCache::from_leaves_and_subtrees(
             self,
@@ -646,7 +646,7 @@ impl TreeHash for Outer {
     }
 }
 
-impl CachedTreeHashSubTree<Outer> for Outer {
+impl CachedTreeHash<Outer> for Outer {
     fn new_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
         let tree = TreeHashCache::from_leaves_and_subtrees(
             self,
diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs
index 272ea7e96..9b35512a9 100644
--- a/eth2/utils/tree_hash_derive/src/lib.rs
+++ b/eth2/utils/tree_hash_derive/src/lib.rs
@@ -37,10 +37,10 @@ fn should_skip_hashing(field: &syn::Field) -> bool {
         .any(|attr| attr.into_token_stream().to_string() == "# [ tree_hash ( skip_hashing ) ]")
 }
 
-/// Implements `tree_hash::CachedTreeHashSubTree` for some `struct`.
+/// Implements `tree_hash::CachedTreeHash` for some `struct`.
 ///
 /// Fields are hashed in the order they are defined.
-#[proc_macro_derive(CachedTreeHashSubTree, attributes(tree_hash))]
+#[proc_macro_derive(CachedTreeHash, attributes(tree_hash))]
 pub fn subtree_derive(input: TokenStream) -> TokenStream {
     let item = parse_macro_input!(input as DeriveInput);
 
@@ -58,7 +58,7 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
     let num_items = idents_a.len();
 
     let output = quote! {
-        impl tree_hash::CachedTreeHashSubTree<#name> for #name {
+        impl tree_hash::CachedTreeHash<#name> for #name {
             fn new_tree_hash_cache(&self, depth: usize) -> Result<tree_hash::TreeHashCache, tree_hash::Error> {
                 let tree = tree_hash::TreeHashCache::from_leaves_and_subtrees(
                     self,
diff --git a/eth2/utils/tree_hash_derive/tests/tests.rs b/eth2/utils/tree_hash_derive/tests/tests.rs
index a7c74b23e..10d0aa853 100644
--- a/eth2/utils/tree_hash_derive/tests/tests.rs
+++ b/eth2/utils/tree_hash_derive/tests/tests.rs
@@ -1,7 +1,7 @@
-use tree_hash::{CachedTreeHashSubTree, SignedRoot, TreeHash};
-use tree_hash_derive::{CachedTreeHashSubTree, SignedRoot, TreeHash};
+use tree_hash::{CachedTreeHash, SignedRoot, TreeHash};
+use tree_hash_derive::{CachedTreeHash, SignedRoot, TreeHash};
 
-#[derive(Clone, Debug, TreeHash, CachedTreeHashSubTree)]
+#[derive(Clone, Debug, TreeHash, CachedTreeHash)]
 pub struct Inner {
     pub a: u64,
     pub b: u64,
@@ -11,7 +11,7 @@ pub struct Inner {
 
 fn test_standard_and_cached<T>(original: &T, modified: &T)
 where
-    T: CachedTreeHashSubTree<T>,
+    T: CachedTreeHash<T>,
 {
     let mut cache = original.new_tree_hash_cache().unwrap();
 
@@ -44,7 +44,7 @@ fn inner_standard_vs_cached() {
     test_standard_and_cached(&original, &modified);
 }
 
-#[derive(Clone, Debug, TreeHash, CachedTreeHashSubTree)]
+#[derive(Clone, Debug, TreeHash, CachedTreeHash)]
 pub struct Uneven {
     pub a: u64,
     pub b: u64,

From 1f6a54c2baaf1fd7ad28312e89292aba0793ee9d Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 24 Apr 2019 18:27:03 +1000
Subject: [PATCH 101/137] Update readme

---
 eth2/utils/tree_hash/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eth2/utils/tree_hash/README.md b/eth2/utils/tree_hash/README.md
index 3c6e13a2e..55b6cea37 100644
--- a/eth2/utils/tree_hash/README.md
+++ b/eth2/utils/tree_hash/README.md
@@ -17,7 +17,7 @@ fn main() {
 	let foo = Foo {
 		a: 42,
 		b: vec![1, 2, 3]
-	}
+	};
 
 	println!("root: {}", foo.tree_hash_root());
 }
@@ -68,7 +68,7 @@ fn main() {
 
 
     let mut hasher = CachedTreeHasher::new(&bar).unwrap();
-	hasher.update(&modified_bar);
+	hasher.update(&modified_bar).unwrap();
 
 	// Assert that the cached tree hash matches a standard tree hash.
 	assert_eq!(hasher.tree_hash_root(), modified_bar.tree_hash_root());

From 58b69e9ba6afbfc53a978ecf707f46c25e379441 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 24 Apr 2019 18:30:59 +1000
Subject: [PATCH 102/137] Add comma to readme

---
 eth2/utils/tree_hash/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eth2/utils/tree_hash/README.md b/eth2/utils/tree_hash/README.md
index 55b6cea37..0498bfc3e 100644
--- a/eth2/utils/tree_hash/README.md
+++ b/eth2/utils/tree_hash/README.md
@@ -57,7 +57,7 @@ fn main() {
 			Foo {
 				a: 100,
 				b: vec![1, 2, 3, 4, 5, 6]
-			}
+			},
 			Foo {
 				a: 42,
 				b: vec![]

From 827e1c62d9e517ae3a45413b68bcde5d356e65bd Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Thu, 25 Apr 2019 12:00:39 +1000
Subject: [PATCH 103/137] Add extra tests, all passing

---
 .../src/cached_tree_hash/impls/vec.rs         |  2 +-
 .../src/cached_tree_hash/tree_hash_cache.rs   |  1 +
 eth2/utils/tree_hash/tests/tests.rs           | 68 +++++++++++++++----
 3 files changed, 56 insertions(+), 15 deletions(-)

diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
index 6a0770681..c92077e94 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
@@ -5,7 +5,7 @@ where
     T: CachedTreeHash<T> + TreeHash,
 {
     fn new_tree_hash_cache(&self, depth: usize) -> Result<TreeHashCache, Error> {
-        let mut overlay = self.tree_hash_cache_overlay(0, depth)?;
+        let overlay = self.tree_hash_cache_overlay(0, depth)?;
 
         let mut cache = match T::tree_hash_type() {
             TreeHashType::Basic => TreeHashCache::from_bytes(
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/tree_hash_cache.rs b/eth2/utils/tree_hash/src/cached_tree_hash/tree_hash_cache.rs
index 336d28028..169edb4d1 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/tree_hash_cache.rs
+++ b/eth2/utils/tree_hash/src/cached_tree_hash/tree_hash_cache.rs
@@ -183,6 +183,7 @@ impl TreeHashCache {
             .iter()
             .skip(overlay_index)
             .position(|o| o.depth <= depth)
+            .and_then(|i| Some(i + overlay_index))
             .unwrap_or_else(|| self.overlays.len());
 
         self.overlays.splice(overlay_index..end, vec![]);
diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
index 6f339b8f2..e3d8701bd 100644
--- a/eth2/utils/tree_hash/tests/tests.rs
+++ b/eth2/utils/tree_hash/tests/tests.rs
@@ -231,8 +231,7 @@ pub struct StructWithVecOfStructs {
     pub c: Vec<Inner>,
 }
 
-#[test]
-fn test_struct_with_vec_of_structs() {
+fn get_struct_with_vec_of_structs() -> Vec<StructWithVecOfStructs> {
     let inner_a = Inner {
         a: 12,
         b: 13,
@@ -285,21 +284,62 @@ fn test_struct_with_vec_of_structs() {
         ..a.clone()
     };
 
+    vec![a, b, c, d, e, f]
+}
+
+#[test]
+fn test_struct_with_vec_of_structs() {
+    let variants = get_struct_with_vec_of_structs();
+
+    test_routine(variants[0].clone(), variants.clone());
+    test_routine(variants[1].clone(), variants.clone());
+    test_routine(variants[2].clone(), variants.clone());
+    test_routine(variants[3].clone(), variants.clone());
+    test_routine(variants[4].clone(), variants.clone());
+    test_routine(variants[5].clone(), variants.clone());
+}
+
+#[derive(Clone, Debug, TreeHash, CachedTreeHash)]
+pub struct StructWithVecOfStructWithVecOfStructs {
+    pub a: Vec<StructWithVecOfStructs>,
+    pub b: u64,
+}
+
+#[test]
+fn test_struct_with_vec_of_struct_with_vec_of_structs() {
+    let structs = get_struct_with_vec_of_structs();
+
     let variants = vec![
-        a.clone(),
-        b.clone(),
-        c.clone(),
-        d.clone(),
-        e.clone(),
-        f.clone(),
+        StructWithVecOfStructWithVecOfStructs {
+            a: structs[..].to_vec(),
+            b: 99,
+        },
+        StructWithVecOfStructWithVecOfStructs { a: vec![], b: 99 },
+        StructWithVecOfStructWithVecOfStructs {
+            a: structs[0..2].to_vec(),
+            b: 99,
+        },
+        StructWithVecOfStructWithVecOfStructs {
+            a: structs[0..2].to_vec(),
+            b: 100,
+        },
+        StructWithVecOfStructWithVecOfStructs {
+            a: structs[0..1].to_vec(),
+            b: 100,
+        },
+        StructWithVecOfStructWithVecOfStructs {
+            a: structs[0..4].to_vec(),
+            b: 100,
+        },
+        StructWithVecOfStructWithVecOfStructs {
+            a: structs[0..5].to_vec(),
+            b: 8,
+        },
     ];
 
-    test_routine(a, variants.clone());
-    test_routine(b, variants.clone());
-    test_routine(c, variants.clone());
-    test_routine(d, variants.clone());
-    test_routine(e, variants.clone());
-    test_routine(f, variants);
+    for v in &variants {
+        test_routine(v.clone(), variants.clone());
+    }
 }
 
 #[derive(Clone, Debug, TreeHash, CachedTreeHash)]

From 0bb9c59b4764f95ab07bbc4cacb0f2b95fae57b7 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Thu, 25 Apr 2019 12:24:45 +1000
Subject: [PATCH 104/137] Add ignored and non-ignored state-trans tests

---
 Jenkinsfile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 845cd357f..11cbf0abe 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -24,7 +24,9 @@ pipeline {
 				sh 'cargo test --verbose --all'
 				sh 'cargo test --verbose --all --release'
                 sh 'cargo test --manifest-path eth2/state_processing/Cargo.toml --verbose \
-                               --release --features fake_crypto -- --include-ignored'
+                               --release --features fake_crypto'
+                sh 'cargo test --manifest-path eth2/state_processing/Cargo.toml --verbose \
+                               --release --features fake_crypto -- --ignored'
 
 			}
 		}

From a76b24e274a30694004f919372698e707cfb9dff Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Thu, 25 Apr 2019 12:25:01 +1000
Subject: [PATCH 105/137] Disable running docs example for test harness

---
 beacon_node/beacon_chain/test_harness/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/beacon_node/beacon_chain/test_harness/src/lib.rs b/beacon_node/beacon_chain/test_harness/src/lib.rs
index 0703fd4a5..e93fa7003 100644
--- a/beacon_node/beacon_chain/test_harness/src/lib.rs
+++ b/beacon_node/beacon_chain/test_harness/src/lib.rs
@@ -8,7 +8,7 @@
 //! producing blocks and attestations.
 //!
 //! Example:
-//! ```
+//! ```rust,no_run
 //! use test_harness::BeaconChainHarness;
 //! use types::ChainSpec;
 //!

From b213a5ade4799db678afff5190a6adc678ea08b9 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Fri, 26 Apr 2019 09:55:03 +1000
Subject: [PATCH 106/137] Create `cached_tree_hash` crate.

---
 Cargo.toml                                    |    1 +
 eth2/types/src/tree_hash_vector.rs            |    2 +-
 eth2/utils/cached_tree_hash/Cargo.toml        |   14 +
 eth2/utils/cached_tree_hash/README.md         |   76 +
 .../src}/btree_overlay.rs                     |    4 +
 eth2/utils/cached_tree_hash/src/errors.rs     |   18 +
 .../src}/impls.rs                             |    9 +-
 .../src}/impls/vec.rs                         |    1 +
 eth2/utils/cached_tree_hash/src/lib.rs        |   66 +
 eth2/utils/cached_tree_hash/src/merkleize.rs  |   78 +
 .../src}/resize.rs                            |    0
 .../src}/tree_hash_cache.rs                   |    6 +
 eth2/utils/cached_tree_hash/tests/tests.rs    |  437 +++++
 eth2/utils/tree_hash/src/cached_tree_hash.rs  |  156 --
 .../src/{standard_tree_hash => }/impls.rs     |    1 +
 eth2/utils/tree_hash/src/lib.rs               |   84 +-
 eth2/utils/tree_hash/src/signed_root.rs       |    5 -
 .../utils/tree_hash/src/standard_tree_hash.rs |   75 -
 eth2/utils/tree_hash/tests/tests.rs           | 1514 -----------------
 eth2/utils/tree_hash_derive/Cargo.toml        |    1 +
 eth2/utils/tree_hash_derive/src/lib.rs        |   16 +-
 eth2/utils/tree_hash_derive/tests/tests.rs    |   14 +-
 22 files changed, 800 insertions(+), 1778 deletions(-)
 create mode 100644 eth2/utils/cached_tree_hash/Cargo.toml
 create mode 100644 eth2/utils/cached_tree_hash/README.md
 rename eth2/utils/{tree_hash/src/cached_tree_hash => cached_tree_hash/src}/btree_overlay.rs (98%)
 create mode 100644 eth2/utils/cached_tree_hash/src/errors.rs
 rename eth2/utils/{tree_hash/src/cached_tree_hash => cached_tree_hash/src}/impls.rs (92%)
 rename eth2/utils/{tree_hash/src/cached_tree_hash => cached_tree_hash/src}/impls/vec.rs (99%)
 create mode 100644 eth2/utils/cached_tree_hash/src/lib.rs
 create mode 100644 eth2/utils/cached_tree_hash/src/merkleize.rs
 rename eth2/utils/{tree_hash/src/cached_tree_hash => cached_tree_hash/src}/resize.rs (100%)
 rename eth2/utils/{tree_hash/src/cached_tree_hash => cached_tree_hash/src}/tree_hash_cache.rs (98%)
 create mode 100644 eth2/utils/cached_tree_hash/tests/tests.rs
 delete mode 100644 eth2/utils/tree_hash/src/cached_tree_hash.rs
 rename eth2/utils/tree_hash/src/{standard_tree_hash => }/impls.rs (99%)
 delete mode 100644 eth2/utils/tree_hash/src/signed_root.rs
 delete mode 100644 eth2/utils/tree_hash/src/standard_tree_hash.rs
 delete mode 100644 eth2/utils/tree_hash/tests/tests.rs

diff --git a/Cargo.toml b/Cargo.toml
index b419d32e4..c05e22286 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,6 +9,7 @@ members = [
 	"eth2/types",
 	"eth2/utils/bls",
 	"eth2/utils/boolean-bitfield",
+	"eth2/utils/cached_tree_hash",
 	"eth2/utils/hashing",
 	"eth2/utils/honey-badger-split",
 	"eth2/utils/merkle_proof",
diff --git a/eth2/types/src/tree_hash_vector.rs b/eth2/types/src/tree_hash_vector.rs
index 1cc8e40a5..c90a77c8d 100644
--- a/eth2/types/src/tree_hash_vector.rs
+++ b/eth2/types/src/tree_hash_vector.rs
@@ -50,7 +50,7 @@ where
     }
 
     fn tree_hash_root(&self) -> Vec<u8> {
-        tree_hash::standard_tree_hash::vec_tree_hash_root(self)
+        tree_hash::impls::vec_tree_hash_root(self)
     }
 }
 
diff --git a/eth2/utils/cached_tree_hash/Cargo.toml b/eth2/utils/cached_tree_hash/Cargo.toml
new file mode 100644
index 000000000..c8881eb0f
--- /dev/null
+++ b/eth2/utils/cached_tree_hash/Cargo.toml
@@ -0,0 +1,14 @@
+[package]
+name = "cached_tree_hash"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+[dev-dependencies]
+tree_hash_derive = { path = "../tree_hash_derive" }
+
+[dependencies]
+tree_hash = { path = "../tree_hash" }
+ethereum-types = "0.5"
+hashing = { path = "../hashing" }
+int_to_bytes = { path = "../int_to_bytes" }
diff --git a/eth2/utils/cached_tree_hash/README.md b/eth2/utils/cached_tree_hash/README.md
new file mode 100644
index 000000000..0498bfc3e
--- /dev/null
+++ b/eth2/utils/cached_tree_hash/README.md
@@ -0,0 +1,76 @@
+# Tree hashing
+
+Provides both cached and non-cached tree hashing methods.
+
+## Standard Tree Hash
+
+```rust
+use tree_hash_derive::TreeHash;
+
+#[derive(TreeHash)]
+struct Foo {
+	a: u64,
+	b: Vec<u64>,
+}
+
+fn main() {
+	let foo = Foo {
+		a: 42,
+		b: vec![1, 2, 3]
+	};
+
+	println!("root: {}", foo.tree_hash_root());
+}
+```
+
+## Cached Tree Hash
+
+
+```rust
+use tree_hash_derive::{TreeHash, CachedTreeHash};
+
+#[derive(TreeHash, CachedTreeHash)]
+struct Foo {
+	a: u64,
+	b: Vec<u64>,
+}
+
+#[derive(TreeHash, CachedTreeHash)]
+struct Bar {
+	a: Vec<Foo>,
+	b: u64,
+}
+
+fn main() {
+	let bar = Bar {
+		a: vec![
+			Foo {
+				a: 42,
+				b: vec![1, 2, 3]
+			}
+		],
+		b: 42
+	};
+
+	let modified_bar = Bar {
+		a: vec![
+			Foo {
+				a: 100,
+				b: vec![1, 2, 3, 4, 5, 6]
+			},
+			Foo {
+				a: 42,
+				b: vec![]
+			}
+		],
+		b: 99
+	};
+
+
+    let mut hasher = CachedTreeHasher::new(&bar).unwrap();
+	hasher.update(&modified_bar).unwrap();
+
+	// Assert that the cached tree hash matches a standard tree hash.
+	assert_eq!(hasher.tree_hash_root(), modified_bar.tree_hash_root());
+}
+```
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs b/eth2/utils/cached_tree_hash/src/btree_overlay.rs
similarity index 98%
rename from eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
rename to eth2/utils/cached_tree_hash/src/btree_overlay.rs
index 9fd1251d7..1a8fde3c1 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs
+++ b/eth2/utils/cached_tree_hash/src/btree_overlay.rs
@@ -168,6 +168,10 @@ impl BTreeOverlay {
     }
 }
 
+fn children(parent: usize) -> (usize, usize) {
+    ((2 * parent + 1), (2 * parent + 2))
+}
+
 #[cfg(test)]
 mod test {
     use super::*;
diff --git a/eth2/utils/cached_tree_hash/src/errors.rs b/eth2/utils/cached_tree_hash/src/errors.rs
new file mode 100644
index 000000000..9045d0409
--- /dev/null
+++ b/eth2/utils/cached_tree_hash/src/errors.rs
@@ -0,0 +1,18 @@
+use tree_hash::TreeHashType;
+
+#[derive(Debug, PartialEq, Clone)]
+pub enum Error {
+    ShouldNotProduceBTreeOverlay,
+    NoFirstNode,
+    NoBytesForRoot,
+    UnableToObtainSlices,
+    UnableToGrowMerkleTree,
+    UnableToShrinkMerkleTree,
+    TreeCannotHaveZeroNodes,
+    ShouldNeverBePacked(TreeHashType),
+    BytesAreNotEvenChunks(usize),
+    NoModifiedFieldForChunk(usize),
+    NoBytesForChunk(usize),
+    NoOverlayForIndex(usize),
+    NotLeafNode(usize),
+}
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs b/eth2/utils/cached_tree_hash/src/impls.rs
similarity index 92%
rename from eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
rename to eth2/utils/cached_tree_hash/src/impls.rs
index 74ab986cb..80259632d 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs
+++ b/eth2/utils/cached_tree_hash/src/impls.rs
@@ -1,4 +1,5 @@
 use super::*;
+use crate::merkleize::merkleize;
 
 mod vec;
 
@@ -17,8 +18,8 @@ impl CachedTreeHash<u64> for u64 {
 
     fn tree_hash_cache_overlay(
         &self,
-        chunk_offset: usize,
-        depth: usize,
+        _chunk_offset: usize,
+        _depth: usize,
     ) -> Result<BTreeOverlay, Error> {
         panic!("Basic should not produce overlay");
         // BTreeOverlay::from_lengths(chunk_offset, 1, depth, vec![1])
@@ -50,8 +51,8 @@ impl CachedTreeHash<usize> for usize {
 
     fn tree_hash_cache_overlay(
         &self,
-        chunk_offset: usize,
-        depth: usize,
+        _chunk_offset: usize,
+        _depth: usize,
     ) -> Result<BTreeOverlay, Error> {
         panic!("Basic should not produce overlay");
         // BTreeOverlay::from_lengths(chunk_offset, 1, depth, vec![1])
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs b/eth2/utils/cached_tree_hash/src/impls/vec.rs
similarity index 99%
rename from eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
rename to eth2/utils/cached_tree_hash/src/impls/vec.rs
index c92077e94..8c58e022a 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs
+++ b/eth2/utils/cached_tree_hash/src/impls/vec.rs
@@ -1,4 +1,5 @@
 use super::*;
+use crate::merkleize::{merkleize, num_sanitized_leaves, sanitise_bytes};
 
 impl<T> CachedTreeHash<Vec<T>> for Vec<T>
 where
diff --git a/eth2/utils/cached_tree_hash/src/lib.rs b/eth2/utils/cached_tree_hash/src/lib.rs
new file mode 100644
index 000000000..539519611
--- /dev/null
+++ b/eth2/utils/cached_tree_hash/src/lib.rs
@@ -0,0 +1,66 @@
+use hashing::hash;
+use std::ops::Range;
+use tree_hash::{TreeHash, TreeHashType, BYTES_PER_CHUNK, HASHSIZE};
+
+mod btree_overlay;
+mod errors;
+mod impls;
+pub mod merkleize;
+mod resize;
+mod tree_hash_cache;
+
+pub use btree_overlay::BTreeOverlay;
+pub use errors::Error;
+pub use tree_hash_cache::TreeHashCache;
+
+pub trait CachedTreeHash<Item>: TreeHash {
+    fn tree_hash_cache_overlay(
+        &self,
+        chunk_offset: usize,
+        depth: usize,
+    ) -> Result<BTreeOverlay, Error>;
+
+    fn num_tree_hash_cache_chunks(&self) -> usize;
+
+    fn new_tree_hash_cache(&self, depth: usize) -> Result<TreeHashCache, Error>;
+
+    fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error>;
+}
+
+#[derive(Debug, PartialEq)]
+pub struct CachedTreeHasher {
+    cache: TreeHashCache,
+}
+
+impl CachedTreeHasher {
+    pub fn new<T>(item: &T) -> Result<Self, Error>
+    where
+        T: CachedTreeHash<T>,
+    {
+        Ok(Self {
+            cache: TreeHashCache::new(item, 0)?,
+        })
+    }
+
+    pub fn update<T>(&mut self, item: &T) -> Result<(), Error>
+    where
+        T: CachedTreeHash<T>,
+    {
+        // Reset the per-hash counters.
+        self.cache.chunk_index = 0;
+        self.cache.overlay_index = 0;
+
+        // Reset the "modified" flags for the cache.
+        self.cache.reset_modifications();
+
+        // Update the cache with the (maybe) changed object.
+        item.update_tree_hash_cache(&mut self.cache)?;
+
+        Ok(())
+    }
+
+    pub fn tree_hash_root(&self) -> Result<Vec<u8>, Error> {
+        // Return the root of the cache -- the merkle root.
+        Ok(self.cache.root()?.to_vec())
+    }
+}
diff --git a/eth2/utils/cached_tree_hash/src/merkleize.rs b/eth2/utils/cached_tree_hash/src/merkleize.rs
new file mode 100644
index 000000000..6bfa73888
--- /dev/null
+++ b/eth2/utils/cached_tree_hash/src/merkleize.rs
@@ -0,0 +1,78 @@
+use hashing::hash;
+use tree_hash::{BYTES_PER_CHUNK, HASHSIZE, MERKLE_HASH_CHUNK};
+
+/// Split `values` into a power-of-two, identical-length chunks (padding with `0`) and merkleize
+/// them, returning the entire merkle tree.
+///
+/// The root hash is `merkleize(values)[0..BYTES_PER_CHUNK]`.
+pub fn merkleize(values: Vec<u8>) -> Vec<u8> {
+    let values = sanitise_bytes(values);
+
+    let leaves = values.len() / HASHSIZE;
+
+    if leaves == 0 {
+        panic!("No full leaves");
+    }
+
+    if !leaves.is_power_of_two() {
+        panic!("leaves is not power of two");
+    }
+
+    let mut o: Vec<u8> = vec![0; (num_nodes(leaves) - leaves) * HASHSIZE];
+    o.append(&mut values.to_vec());
+
+    let mut i = o.len();
+    let mut j = o.len() - values.len();
+
+    while i >= MERKLE_HASH_CHUNK {
+        i -= MERKLE_HASH_CHUNK;
+        let hash = hash(&o[i..i + MERKLE_HASH_CHUNK]);
+
+        j -= HASHSIZE;
+        o[j..j + HASHSIZE].copy_from_slice(&hash);
+    }
+
+    o
+}
+
+pub fn sanitise_bytes(mut bytes: Vec<u8>) -> Vec<u8> {
+    let present_leaves = num_unsanitized_leaves(bytes.len());
+    let required_leaves = present_leaves.next_power_of_two();
+
+    if (present_leaves != required_leaves) | last_leaf_needs_padding(bytes.len()) {
+        bytes.resize(num_bytes(required_leaves), 0);
+    }
+
+    bytes
+}
+
+pub fn pad_for_leaf_count(num_leaves: usize, bytes: &mut Vec<u8>) {
+    let required_leaves = num_leaves.next_power_of_two();
+
+    bytes.resize(
+        bytes.len() + (required_leaves - num_leaves) * BYTES_PER_CHUNK,
+        0,
+    );
+}
+
+fn last_leaf_needs_padding(num_bytes: usize) -> bool {
+    num_bytes % HASHSIZE != 0
+}
+
+/// Rounds up
+fn num_unsanitized_leaves(num_bytes: usize) -> usize {
+    (num_bytes + HASHSIZE - 1) / HASHSIZE
+}
+
+fn num_bytes(num_leaves: usize) -> usize {
+    num_leaves * HASHSIZE
+}
+
+fn num_nodes(num_leaves: usize) -> usize {
+    2 * num_leaves - 1
+}
+
+pub fn num_sanitized_leaves(num_bytes: usize) -> usize {
+    let leaves = (num_bytes + HASHSIZE - 1) / HASHSIZE;
+    leaves.next_power_of_two()
+}
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/resize.rs b/eth2/utils/cached_tree_hash/src/resize.rs
similarity index 100%
rename from eth2/utils/tree_hash/src/cached_tree_hash/resize.rs
rename to eth2/utils/cached_tree_hash/src/resize.rs
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/tree_hash_cache.rs b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
similarity index 98%
rename from eth2/utils/tree_hash/src/cached_tree_hash/tree_hash_cache.rs
rename to eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
index 169edb4d1..d93278d30 100644
--- a/eth2/utils/tree_hash/src/cached_tree_hash/tree_hash_cache.rs
+++ b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
@@ -1,4 +1,6 @@
 use super::*;
+use crate::merkleize::{merkleize, pad_for_leaf_count};
+use int_to_bytes::int_to_bytes32;
 
 #[derive(Debug, PartialEq, Clone)]
 pub struct TreeHashCache {
@@ -328,3 +330,7 @@ impl TreeHashCache {
         (self.cache, self.chunk_modified, self.overlays)
     }
 }
+
+fn node_range_to_byte_range(node_range: &Range<usize>) -> Range<usize> {
+    node_range.start * HASHSIZE..node_range.end * HASHSIZE
+}
diff --git a/eth2/utils/cached_tree_hash/tests/tests.rs b/eth2/utils/cached_tree_hash/tests/tests.rs
new file mode 100644
index 000000000..8837fa1da
--- /dev/null
+++ b/eth2/utils/cached_tree_hash/tests/tests.rs
@@ -0,0 +1,437 @@
+use cached_tree_hash::{merkleize::merkleize, *};
+use int_to_bytes::int_to_bytes32;
+use tree_hash_derive::{CachedTreeHash, TreeHash};
+
+#[derive(Clone, Debug, TreeHash, CachedTreeHash)]
+pub struct NestedStruct {
+    pub a: u64,
+    pub b: Inner,
+}
+
+fn test_routine<T>(original: T, modified: Vec<T>)
+where
+    T: CachedTreeHash<T>,
+{
+    let mut hasher = CachedTreeHasher::new(&original).unwrap();
+
+    let standard_root = original.tree_hash_root();
+    let cached_root = hasher.tree_hash_root().unwrap();
+    assert_eq!(standard_root, cached_root, "Initial cache build failed.");
+
+    for (i, modified) in modified.iter().enumerate() {
+        println!("-- Start of modification {} --", i);
+        // Test after a modification
+        hasher
+            .update(modified)
+            .expect(&format!("Modification {}", i));
+        let standard_root = modified.tree_hash_root();
+        let cached_root = hasher
+            .tree_hash_root()
+            .expect(&format!("Modification {}", i));
+        assert_eq!(
+            standard_root, cached_root,
+            "Modification {} failed. \n Cache: {:?}",
+            i, hasher
+        );
+    }
+}
+
+#[test]
+fn test_nested_struct() {
+    let original = NestedStruct {
+        a: 42,
+        b: Inner {
+            a: 12,
+            b: 13,
+            c: 14,
+            d: 15,
+        },
+    };
+    let modified = vec![NestedStruct {
+        a: 99,
+        ..original.clone()
+    }];
+
+    test_routine(original, modified);
+}
+
+#[test]
+fn test_inner() {
+    let original = Inner {
+        a: 12,
+        b: 13,
+        c: 14,
+        d: 15,
+    };
+
+    let modified = vec![Inner {
+        a: 99,
+        ..original.clone()
+    }];
+
+    test_routine(original, modified);
+}
+
+#[test]
+fn test_vec() {
+    let original: Vec<u64> = vec![1, 2, 3, 4, 5];
+
+    let modified: Vec<Vec<u64>> = vec![
+        vec![1, 2, 3, 4, 42],
+        vec![1, 2, 3, 4],
+        vec![],
+        vec![42; 2_usize.pow(4)],
+        vec![],
+        vec![],
+        vec![1, 2, 3, 4, 42],
+        vec![1, 2, 3],
+        vec![1],
+    ];
+
+    test_routine(original, modified);
+}
+
+#[test]
+fn test_nested_list_of_u64() {
+    let original: Vec<Vec<u64>> = vec![vec![42]];
+
+    let modified = vec![
+        vec![vec![1]],
+        vec![vec![1], vec![2]],
+        vec![vec![1], vec![3], vec![4]],
+        vec![],
+        vec![vec![1], vec![3], vec![4]],
+        vec![],
+        vec![vec![1, 2], vec![3], vec![4, 5, 6, 7, 8]],
+        vec![],
+        vec![vec![1], vec![2], vec![3]],
+        vec![vec![1, 2, 3, 4, 5, 6], vec![1, 2, 3, 4, 5, 6, 7]],
+        vec![vec![], vec![], vec![]],
+        vec![vec![0, 0, 0], vec![0], vec![0]],
+    ];
+
+    test_routine(original, modified);
+}
+
+#[derive(Clone, Debug, TreeHash, CachedTreeHash)]
+pub struct StructWithVec {
+    pub a: u64,
+    pub b: Inner,
+    pub c: Vec<u64>,
+}
+
+#[test]
+fn test_struct_with_vec() {
+    let original = StructWithVec {
+        a: 42,
+        b: Inner {
+            a: 12,
+            b: 13,
+            c: 14,
+            d: 15,
+        },
+        c: vec![1, 2, 3, 4, 5],
+    };
+
+    let modified = vec![
+        StructWithVec {
+            a: 99,
+            ..original.clone()
+        },
+        StructWithVec {
+            a: 100,
+            ..original.clone()
+        },
+        StructWithVec {
+            c: vec![1, 2, 3, 4, 5],
+            ..original.clone()
+        },
+        StructWithVec {
+            c: vec![1, 3, 4, 5, 6],
+            ..original.clone()
+        },
+        StructWithVec {
+            c: vec![1, 3, 4, 5, 6, 7, 8, 9],
+            ..original.clone()
+        },
+        StructWithVec {
+            c: vec![1, 3, 4, 5],
+            ..original.clone()
+        },
+        StructWithVec {
+            b: Inner {
+                a: u64::max_value(),
+                b: u64::max_value(),
+                c: u64::max_value(),
+                d: u64::max_value(),
+            },
+            c: vec![],
+            ..original.clone()
+        },
+        StructWithVec {
+            b: Inner {
+                a: 0,
+                b: 1,
+                c: 2,
+                d: 3,
+            },
+            ..original.clone()
+        },
+    ];
+
+    test_routine(original, modified);
+}
+
+#[test]
+fn test_vec_of_struct_with_vec() {
+    let a = StructWithVec {
+        a: 42,
+        b: Inner {
+            a: 12,
+            b: 13,
+            c: 14,
+            d: 15,
+        },
+        c: vec![1, 2, 3, 4, 5],
+    };
+    let b = StructWithVec {
+        c: vec![],
+        ..a.clone()
+    };
+    let c = StructWithVec {
+        b: Inner {
+            a: 99,
+            b: 100,
+            c: 101,
+            d: 102,
+        },
+        ..a.clone()
+    };
+    let d = StructWithVec { a: 0, ..a.clone() };
+
+    // let original: Vec<StructWithVec> = vec![a.clone(), c.clone()];
+    let original: Vec<StructWithVec> = vec![a.clone()];
+
+    let modified = vec![
+        vec![a.clone(), c.clone()],
+        vec![a.clone(), b.clone(), c.clone(), d.clone()],
+        vec![b.clone(), a.clone(), c.clone(), d.clone()],
+        vec![],
+        vec![a.clone()],
+        vec![a.clone(), b.clone(), c.clone(), d.clone()],
+    ];
+
+    test_routine(original, modified);
+}
+
+#[derive(Clone, Debug, TreeHash, CachedTreeHash)]
+pub struct StructWithVecOfStructs {
+    pub a: u64,
+    pub b: Inner,
+    pub c: Vec<Inner>,
+}
+
+fn get_struct_with_vec_of_structs() -> Vec<StructWithVecOfStructs> {
+    let inner_a = Inner {
+        a: 12,
+        b: 13,
+        c: 14,
+        d: 15,
+    };
+
+    let inner_b = Inner {
+        a: 99,
+        b: 100,
+        c: 101,
+        d: 102,
+    };
+
+    let inner_c = Inner {
+        a: 255,
+        b: 256,
+        c: 257,
+        d: 0,
+    };
+
+    let a = StructWithVecOfStructs {
+        a: 42,
+        b: inner_a.clone(),
+        c: vec![inner_a.clone(), inner_b.clone(), inner_c.clone()],
+    };
+
+    let b = StructWithVecOfStructs {
+        c: vec![],
+        ..a.clone()
+    };
+
+    let c = StructWithVecOfStructs {
+        a: 800,
+        ..a.clone()
+    };
+
+    let d = StructWithVecOfStructs {
+        b: inner_c.clone(),
+        ..a.clone()
+    };
+
+    let e = StructWithVecOfStructs {
+        c: vec![inner_a.clone(), inner_b.clone()],
+        ..a.clone()
+    };
+
+    let f = StructWithVecOfStructs {
+        c: vec![inner_a.clone()],
+        ..a.clone()
+    };
+
+    vec![a, b, c, d, e, f]
+}
+
+#[test]
+fn test_struct_with_vec_of_structs() {
+    let variants = get_struct_with_vec_of_structs();
+
+    test_routine(variants[0].clone(), variants.clone());
+    test_routine(variants[1].clone(), variants.clone());
+    test_routine(variants[2].clone(), variants.clone());
+    test_routine(variants[3].clone(), variants.clone());
+    test_routine(variants[4].clone(), variants.clone());
+    test_routine(variants[5].clone(), variants.clone());
+}
+
+#[derive(Clone, Debug, TreeHash, CachedTreeHash)]
+pub struct StructWithVecOfStructWithVecOfStructs {
+    pub a: Vec<StructWithVecOfStructs>,
+    pub b: u64,
+}
+
+#[test]
+fn test_struct_with_vec_of_struct_with_vec_of_structs() {
+    let structs = get_struct_with_vec_of_structs();
+
+    let variants = vec![
+        StructWithVecOfStructWithVecOfStructs {
+            a: structs[..].to_vec(),
+            b: 99,
+        },
+        StructWithVecOfStructWithVecOfStructs { a: vec![], b: 99 },
+        StructWithVecOfStructWithVecOfStructs {
+            a: structs[0..2].to_vec(),
+            b: 99,
+        },
+        StructWithVecOfStructWithVecOfStructs {
+            a: structs[0..2].to_vec(),
+            b: 100,
+        },
+        StructWithVecOfStructWithVecOfStructs {
+            a: structs[0..1].to_vec(),
+            b: 100,
+        },
+        StructWithVecOfStructWithVecOfStructs {
+            a: structs[0..4].to_vec(),
+            b: 100,
+        },
+        StructWithVecOfStructWithVecOfStructs {
+            a: structs[0..5].to_vec(),
+            b: 8,
+        },
+    ];
+
+    for v in &variants {
+        test_routine(v.clone(), variants.clone());
+    }
+}
+
+#[derive(Clone, Debug, TreeHash, CachedTreeHash)]
+pub struct Inner {
+    pub a: u64,
+    pub b: u64,
+    pub c: u64,
+    pub d: u64,
+}
+
+fn generic_test(index: usize) {
+    let inner = Inner {
+        a: 1,
+        b: 2,
+        c: 3,
+        d: 4,
+    };
+
+    let mut cache = TreeHashCache::new(&inner, 0).unwrap();
+
+    let changed_inner = match index {
+        0 => Inner {
+            a: 42,
+            ..inner.clone()
+        },
+        1 => Inner {
+            b: 42,
+            ..inner.clone()
+        },
+        2 => Inner {
+            c: 42,
+            ..inner.clone()
+        },
+        3 => Inner {
+            d: 42,
+            ..inner.clone()
+        },
+        _ => panic!("bad index"),
+    };
+
+    changed_inner.update_tree_hash_cache(&mut cache).unwrap();
+
+    let data1 = int_to_bytes32(1);
+    let data2 = int_to_bytes32(2);
+    let data3 = int_to_bytes32(3);
+    let data4 = int_to_bytes32(4);
+
+    let mut data = vec![data1, data2, data3, data4];
+
+    data[index] = int_to_bytes32(42);
+
+    let expected = merkleize(join(data));
+
+    let cache_bytes: Vec<u8> = cache.into();
+
+    assert_eq!(expected, cache_bytes);
+}
+
+#[test]
+fn cached_hash_on_inner() {
+    generic_test(0);
+    generic_test(1);
+    generic_test(2);
+    generic_test(3);
+}
+
+#[test]
+fn inner_builds() {
+    let data1 = int_to_bytes32(1);
+    let data2 = int_to_bytes32(2);
+    let data3 = int_to_bytes32(3);
+    let data4 = int_to_bytes32(4);
+
+    let data = join(vec![data1, data2, data3, data4]);
+    let expected = merkleize(data);
+
+    let inner = Inner {
+        a: 1,
+        b: 2,
+        c: 3,
+        d: 4,
+    };
+
+    let cache: Vec<u8> = TreeHashCache::new(&inner, 0).unwrap().into();
+
+    assert_eq!(expected, cache);
+}
+
+fn join(many: Vec<Vec<u8>>) -> Vec<u8> {
+    let mut all = vec![];
+    for one in many {
+        all.extend_from_slice(&mut one.clone())
+    }
+    all
+}
diff --git a/eth2/utils/tree_hash/src/cached_tree_hash.rs b/eth2/utils/tree_hash/src/cached_tree_hash.rs
deleted file mode 100644
index 0183f5c84..000000000
--- a/eth2/utils/tree_hash/src/cached_tree_hash.rs
+++ /dev/null
@@ -1,156 +0,0 @@
-use super::*;
-use hashing::hash;
-use int_to_bytes::int_to_bytes32;
-use std::ops::Range;
-
-pub mod btree_overlay;
-pub mod impls;
-pub mod resize;
-pub mod tree_hash_cache;
-
-pub use btree_overlay::BTreeOverlay;
-pub use tree_hash_cache::TreeHashCache;
-
-#[derive(Debug, PartialEq)]
-pub struct CachedTreeHasher {
-    cache: TreeHashCache,
-}
-
-impl CachedTreeHasher {
-    pub fn new<T>(item: &T) -> Result<Self, Error>
-    where
-        T: CachedTreeHash<T>,
-    {
-        Ok(Self {
-            cache: TreeHashCache::new(item, 0)?,
-        })
-    }
-
-    pub fn update<T>(&mut self, item: &T) -> Result<(), Error>
-    where
-        T: CachedTreeHash<T>,
-    {
-        // Reset the per-hash counters.
-        self.cache.chunk_index = 0;
-        self.cache.overlay_index = 0;
-
-        // Reset the "modified" flags for the cache.
-        self.cache.reset_modifications();
-
-        // Update the cache with the (maybe) changed object.
-        item.update_tree_hash_cache(&mut self.cache)?;
-
-        Ok(())
-    }
-
-    pub fn tree_hash_root(&self) -> Result<Vec<u8>, Error> {
-        // Return the root of the cache -- the merkle root.
-        Ok(self.cache.root()?.to_vec())
-    }
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub enum Error {
-    ShouldNotProduceBTreeOverlay,
-    NoFirstNode,
-    NoBytesForRoot,
-    UnableToObtainSlices,
-    UnableToGrowMerkleTree,
-    UnableToShrinkMerkleTree,
-    TreeCannotHaveZeroNodes,
-    ShouldNeverBePacked(TreeHashType),
-    BytesAreNotEvenChunks(usize),
-    NoModifiedFieldForChunk(usize),
-    NoBytesForChunk(usize),
-    NoOverlayForIndex(usize),
-    NotLeafNode(usize),
-}
-
-pub trait CachedTreeHash<Item>: TreeHash {
-    fn tree_hash_cache_overlay(
-        &self,
-        chunk_offset: usize,
-        depth: usize,
-    ) -> Result<BTreeOverlay, Error>;
-
-    fn num_tree_hash_cache_chunks(&self) -> usize;
-
-    fn new_tree_hash_cache(&self, depth: usize) -> Result<TreeHashCache, Error>;
-
-    fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error>;
-}
-
-fn children(parent: usize) -> (usize, usize) {
-    ((2 * parent + 1), (2 * parent + 2))
-}
-
-fn node_range_to_byte_range(node_range: &Range<usize>) -> Range<usize> {
-    node_range.start * HASHSIZE..node_range.end * HASHSIZE
-}
-
-/// Split `values` into a power-of-two, identical-length chunks (padding with `0`) and merkleize
-/// them, returning the entire merkle tree.
-///
-/// The root hash is `merkleize(values)[0..BYTES_PER_CHUNK]`.
-pub fn merkleize(values: Vec<u8>) -> Vec<u8> {
-    let values = sanitise_bytes(values);
-
-    let leaves = values.len() / HASHSIZE;
-
-    if leaves == 0 {
-        panic!("No full leaves");
-    }
-
-    if !leaves.is_power_of_two() {
-        panic!("leaves is not power of two");
-    }
-
-    let mut o: Vec<u8> = vec![0; (num_nodes(leaves) - leaves) * HASHSIZE];
-    o.append(&mut values.to_vec());
-
-    let mut i = o.len();
-    let mut j = o.len() - values.len();
-
-    while i >= MERKLE_HASH_CHUNCK {
-        i -= MERKLE_HASH_CHUNCK;
-        let hash = hash(&o[i..i + MERKLE_HASH_CHUNCK]);
-
-        j -= HASHSIZE;
-        o[j..j + HASHSIZE].copy_from_slice(&hash);
-    }
-
-    o
-}
-
-pub fn sanitise_bytes(mut bytes: Vec<u8>) -> Vec<u8> {
-    let present_leaves = num_unsanitized_leaves(bytes.len());
-    let required_leaves = present_leaves.next_power_of_two();
-
-    if (present_leaves != required_leaves) | last_leaf_needs_padding(bytes.len()) {
-        bytes.resize(num_bytes(required_leaves), 0);
-    }
-
-    bytes
-}
-
-fn pad_for_leaf_count(num_leaves: usize, bytes: &mut Vec<u8>) {
-    let required_leaves = num_leaves.next_power_of_two();
-
-    bytes.resize(
-        bytes.len() + (required_leaves - num_leaves) * BYTES_PER_CHUNK,
-        0,
-    );
-}
-
-fn last_leaf_needs_padding(num_bytes: usize) -> bool {
-    num_bytes % HASHSIZE != 0
-}
-
-/// Rounds up
-fn num_unsanitized_leaves(num_bytes: usize) -> usize {
-    (num_bytes + HASHSIZE - 1) / HASHSIZE
-}
-
-fn num_bytes(num_leaves: usize) -> usize {
-    num_leaves * HASHSIZE
-}
diff --git a/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs b/eth2/utils/tree_hash/src/impls.rs
similarity index 99%
rename from eth2/utils/tree_hash/src/standard_tree_hash/impls.rs
rename to eth2/utils/tree_hash/src/impls.rs
index be6b4ba07..01b165150 100644
--- a/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs
+++ b/eth2/utils/tree_hash/src/impls.rs
@@ -1,5 +1,6 @@
 use super::*;
 use ethereum_types::H256;
+use int_to_bytes::int_to_bytes32;
 
 macro_rules! impl_for_bitsize {
     ($type: ident, $bit_size: expr) => {
diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs
index 5aaf2d585..6ed0247f1 100644
--- a/eth2/utils/tree_hash/src/lib.rs
+++ b/eth2/utils/tree_hash/src/lib.rs
@@ -1,14 +1,10 @@
-pub mod cached_tree_hash;
-pub mod signed_root;
-pub mod standard_tree_hash;
+use hashing::hash;
+
+pub mod impls;
 
 pub const BYTES_PER_CHUNK: usize = 32;
 pub const HASHSIZE: usize = 32;
-pub const MERKLE_HASH_CHUNCK: usize = 2 * BYTES_PER_CHUNK;
-
-pub use cached_tree_hash::{BTreeOverlay, CachedTreeHash, Error, TreeHashCache};
-pub use signed_root::SignedRoot;
-pub use standard_tree_hash::{merkle_root, TreeHash};
+pub const MERKLE_HASH_CHUNK: usize = 2 * BYTES_PER_CHUNK;
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum TreeHashType {
@@ -18,6 +14,78 @@ pub enum TreeHashType {
     Container,
 }
 
+pub trait TreeHash {
+    fn tree_hash_type() -> TreeHashType;
+
+    fn tree_hash_packed_encoding(&self) -> Vec<u8>;
+
+    fn tree_hash_packing_factor() -> usize;
+
+    fn tree_hash_root(&self) -> Vec<u8>;
+}
+
+pub trait SignedRoot: TreeHash {
+    fn signed_root(&self) -> Vec<u8>;
+}
+
+pub fn merkle_root(bytes: &[u8]) -> Vec<u8> {
+    // TODO: replace this with a more memory efficient method.
+    efficient_merkleize(&bytes)[0..32].to_vec()
+}
+
+pub fn efficient_merkleize(bytes: &[u8]) -> Vec<u8> {
+    // If the bytes are just one chunk (or less than one chunk) just return them.
+    if bytes.len() <= HASHSIZE {
+        let mut o = bytes.to_vec();
+        o.resize(HASHSIZE, 0);
+        return o;
+    }
+
+    let leaves = num_sanitized_leaves(bytes.len());
+    let nodes = num_nodes(leaves);
+    let internal_nodes = nodes - leaves;
+
+    let num_bytes = std::cmp::max(internal_nodes, 1) * HASHSIZE + bytes.len();
+
+    let mut o: Vec<u8> = vec![0; internal_nodes * HASHSIZE];
+
+    o.append(&mut bytes.to_vec());
+
+    assert_eq!(o.len(), num_bytes);
+
+    let empty_chunk_hash = hash(&[0; MERKLE_HASH_CHUNK]);
+
+    let mut i = nodes * HASHSIZE;
+    let mut j = internal_nodes * HASHSIZE;
+
+    while i >= MERKLE_HASH_CHUNK {
+        i -= MERKLE_HASH_CHUNK;
+
+        j -= HASHSIZE;
+        let hash = match o.get(i..i + MERKLE_HASH_CHUNK) {
+            // All bytes are available, hash as ususal.
+            Some(slice) => hash(slice),
+            // Unable to get all the bytes.
+            None => {
+                match o.get(i..) {
+                    // Able to get some of the bytes, pad them out.
+                    Some(slice) => {
+                        let mut bytes = slice.to_vec();
+                        bytes.resize(MERKLE_HASH_CHUNK, 0);
+                        hash(&bytes)
+                    }
+                    // Unable to get any bytes, use the empty-chunk hash.
+                    None => empty_chunk_hash.clone(),
+                }
+            }
+        };
+
+        o[j..j + HASHSIZE].copy_from_slice(&hash);
+    }
+
+    o
+}
+
 fn num_sanitized_leaves(num_bytes: usize) -> usize {
     let leaves = (num_bytes + HASHSIZE - 1) / HASHSIZE;
     leaves.next_power_of_two()
diff --git a/eth2/utils/tree_hash/src/signed_root.rs b/eth2/utils/tree_hash/src/signed_root.rs
deleted file mode 100644
index f7aeca4af..000000000
--- a/eth2/utils/tree_hash/src/signed_root.rs
+++ /dev/null
@@ -1,5 +0,0 @@
-use crate::TreeHash;
-
-pub trait SignedRoot: TreeHash {
-    fn signed_root(&self) -> Vec<u8>;
-}
diff --git a/eth2/utils/tree_hash/src/standard_tree_hash.rs b/eth2/utils/tree_hash/src/standard_tree_hash.rs
deleted file mode 100644
index 812a2c352..000000000
--- a/eth2/utils/tree_hash/src/standard_tree_hash.rs
+++ /dev/null
@@ -1,75 +0,0 @@
-use super::*;
-use hashing::hash;
-use int_to_bytes::int_to_bytes32;
-
-pub use impls::vec_tree_hash_root;
-
-mod impls;
-
-pub trait TreeHash {
-    fn tree_hash_type() -> TreeHashType;
-
-    fn tree_hash_packed_encoding(&self) -> Vec<u8>;
-
-    fn tree_hash_packing_factor() -> usize;
-
-    fn tree_hash_root(&self) -> Vec<u8>;
-}
-
-pub fn merkle_root(bytes: &[u8]) -> Vec<u8> {
-    // TODO: replace this with a more memory efficient method.
-    efficient_merkleize(&bytes)[0..32].to_vec()
-}
-
-pub fn efficient_merkleize(bytes: &[u8]) -> Vec<u8> {
-    // If the bytes are just one chunk (or less than one chunk) just return them.
-    if bytes.len() <= HASHSIZE {
-        let mut o = bytes.to_vec();
-        o.resize(HASHSIZE, 0);
-        return o;
-    }
-
-    let leaves = num_sanitized_leaves(bytes.len());
-    let nodes = num_nodes(leaves);
-    let internal_nodes = nodes - leaves;
-
-    let num_bytes = std::cmp::max(internal_nodes, 1) * HASHSIZE + bytes.len();
-
-    let mut o: Vec<u8> = vec![0; internal_nodes * HASHSIZE];
-
-    o.append(&mut bytes.to_vec());
-
-    assert_eq!(o.len(), num_bytes);
-
-    let empty_chunk_hash = hash(&[0; MERKLE_HASH_CHUNCK]);
-
-    let mut i = nodes * HASHSIZE;
-    let mut j = internal_nodes * HASHSIZE;
-
-    while i >= MERKLE_HASH_CHUNCK {
-        i -= MERKLE_HASH_CHUNCK;
-
-        j -= HASHSIZE;
-        let hash = match o.get(i..i + MERKLE_HASH_CHUNCK) {
-            // All bytes are available, hash as ususal.
-            Some(slice) => hash(slice),
-            // Unable to get all the bytes.
-            None => {
-                match o.get(i..) {
-                    // Able to get some of the bytes, pad them out.
-                    Some(slice) => {
-                        let mut bytes = slice.to_vec();
-                        bytes.resize(MERKLE_HASH_CHUNCK, 0);
-                        hash(&bytes)
-                    }
-                    // Unable to get any bytes, use the empty-chunk hash.
-                    None => empty_chunk_hash.clone(),
-                }
-            }
-        };
-
-        o[j..j + HASHSIZE].copy_from_slice(&hash);
-    }
-
-    o
-}
diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs
deleted file mode 100644
index e3d8701bd..000000000
--- a/eth2/utils/tree_hash/tests/tests.rs
+++ /dev/null
@@ -1,1514 +0,0 @@
-use int_to_bytes::int_to_bytes32;
-use tree_hash::cached_tree_hash::*;
-use tree_hash_derive::{CachedTreeHash, TreeHash};
-
-#[derive(Clone, Debug, TreeHash, CachedTreeHash)]
-pub struct NestedStruct {
-    pub a: u64,
-    pub b: Inner,
-}
-
-fn test_routine<T>(original: T, modified: Vec<T>)
-where
-    T: CachedTreeHash<T>,
-{
-    let mut hasher = CachedTreeHasher::new(&original).unwrap();
-
-    let standard_root = original.tree_hash_root();
-    let cached_root = hasher.tree_hash_root().unwrap();
-    assert_eq!(standard_root, cached_root, "Initial cache build failed.");
-
-    for (i, modified) in modified.iter().enumerate() {
-        println!("-- Start of modification {} --", i);
-        // Test after a modification
-        hasher
-            .update(modified)
-            .expect(&format!("Modification {}", i));
-        let standard_root = modified.tree_hash_root();
-        let cached_root = hasher
-            .tree_hash_root()
-            .expect(&format!("Modification {}", i));
-        assert_eq!(
-            standard_root, cached_root,
-            "Modification {} failed. \n Cache: {:?}",
-            i, hasher
-        );
-    }
-}
-
-#[test]
-fn test_nested_struct() {
-    let original = NestedStruct {
-        a: 42,
-        b: Inner {
-            a: 12,
-            b: 13,
-            c: 14,
-            d: 15,
-        },
-    };
-    let modified = vec![NestedStruct {
-        a: 99,
-        ..original.clone()
-    }];
-
-    test_routine(original, modified);
-}
-
-#[test]
-fn test_inner() {
-    let original = Inner {
-        a: 12,
-        b: 13,
-        c: 14,
-        d: 15,
-    };
-
-    let modified = vec![Inner {
-        a: 99,
-        ..original.clone()
-    }];
-
-    test_routine(original, modified);
-}
-
-#[test]
-fn test_vec() {
-    let original: Vec<u64> = vec![1, 2, 3, 4, 5];
-
-    let modified: Vec<Vec<u64>> = vec![
-        vec![1, 2, 3, 4, 42],
-        vec![1, 2, 3, 4],
-        vec![],
-        vec![42; 2_usize.pow(4)],
-        vec![],
-        vec![],
-        vec![1, 2, 3, 4, 42],
-        vec![1, 2, 3],
-        vec![1],
-    ];
-
-    test_routine(original, modified);
-}
-
-#[test]
-fn test_nested_list_of_u64() {
-    let original: Vec<Vec<u64>> = vec![vec![42]];
-
-    let modified = vec![
-        vec![vec![1]],
-        vec![vec![1], vec![2]],
-        vec![vec![1], vec![3], vec![4]],
-        vec![],
-        vec![vec![1], vec![3], vec![4]],
-        vec![],
-        vec![vec![1, 2], vec![3], vec![4, 5, 6, 7, 8]],
-        vec![],
-        vec![vec![1], vec![2], vec![3]],
-        vec![vec![1, 2, 3, 4, 5, 6], vec![1, 2, 3, 4, 5, 6, 7]],
-        vec![vec![], vec![], vec![]],
-        vec![vec![0, 0, 0], vec![0], vec![0]],
-    ];
-
-    test_routine(original, modified);
-}
-
-#[derive(Clone, Debug, TreeHash, CachedTreeHash)]
-pub struct StructWithVec {
-    pub a: u64,
-    pub b: Inner,
-    pub c: Vec<u64>,
-}
-
-#[test]
-fn test_struct_with_vec() {
-    let original = StructWithVec {
-        a: 42,
-        b: Inner {
-            a: 12,
-            b: 13,
-            c: 14,
-            d: 15,
-        },
-        c: vec![1, 2, 3, 4, 5],
-    };
-
-    let modified = vec![
-        StructWithVec {
-            a: 99,
-            ..original.clone()
-        },
-        StructWithVec {
-            a: 100,
-            ..original.clone()
-        },
-        StructWithVec {
-            c: vec![1, 2, 3, 4, 5],
-            ..original.clone()
-        },
-        StructWithVec {
-            c: vec![1, 3, 4, 5, 6],
-            ..original.clone()
-        },
-        StructWithVec {
-            c: vec![1, 3, 4, 5, 6, 7, 8, 9],
-            ..original.clone()
-        },
-        StructWithVec {
-            c: vec![1, 3, 4, 5],
-            ..original.clone()
-        },
-        StructWithVec {
-            b: Inner {
-                a: u64::max_value(),
-                b: u64::max_value(),
-                c: u64::max_value(),
-                d: u64::max_value(),
-            },
-            c: vec![],
-            ..original.clone()
-        },
-        StructWithVec {
-            b: Inner {
-                a: 0,
-                b: 1,
-                c: 2,
-                d: 3,
-            },
-            ..original.clone()
-        },
-    ];
-
-    test_routine(original, modified);
-}
-
-#[test]
-fn test_vec_of_struct_with_vec() {
-    let a = StructWithVec {
-        a: 42,
-        b: Inner {
-            a: 12,
-            b: 13,
-            c: 14,
-            d: 15,
-        },
-        c: vec![1, 2, 3, 4, 5],
-    };
-    let b = StructWithVec {
-        c: vec![],
-        ..a.clone()
-    };
-    let c = StructWithVec {
-        b: Inner {
-            a: 99,
-            b: 100,
-            c: 101,
-            d: 102,
-        },
-        ..a.clone()
-    };
-    let d = StructWithVec { a: 0, ..a.clone() };
-
-    // let original: Vec<StructWithVec> = vec![a.clone(), c.clone()];
-    let original: Vec<StructWithVec> = vec![a.clone()];
-
-    let modified = vec![
-        vec![a.clone(), c.clone()],
-        vec![a.clone(), b.clone(), c.clone(), d.clone()],
-        vec![b.clone(), a.clone(), c.clone(), d.clone()],
-        vec![],
-        vec![a.clone()],
-        vec![a.clone(), b.clone(), c.clone(), d.clone()],
-    ];
-
-    test_routine(original, modified);
-}
-
-#[derive(Clone, Debug, TreeHash, CachedTreeHash)]
-pub struct StructWithVecOfStructs {
-    pub a: u64,
-    pub b: Inner,
-    pub c: Vec<Inner>,
-}
-
-fn get_struct_with_vec_of_structs() -> Vec<StructWithVecOfStructs> {
-    let inner_a = Inner {
-        a: 12,
-        b: 13,
-        c: 14,
-        d: 15,
-    };
-
-    let inner_b = Inner {
-        a: 99,
-        b: 100,
-        c: 101,
-        d: 102,
-    };
-
-    let inner_c = Inner {
-        a: 255,
-        b: 256,
-        c: 257,
-        d: 0,
-    };
-
-    let a = StructWithVecOfStructs {
-        a: 42,
-        b: inner_a.clone(),
-        c: vec![inner_a.clone(), inner_b.clone(), inner_c.clone()],
-    };
-
-    let b = StructWithVecOfStructs {
-        c: vec![],
-        ..a.clone()
-    };
-
-    let c = StructWithVecOfStructs {
-        a: 800,
-        ..a.clone()
-    };
-
-    let d = StructWithVecOfStructs {
-        b: inner_c.clone(),
-        ..a.clone()
-    };
-
-    let e = StructWithVecOfStructs {
-        c: vec![inner_a.clone(), inner_b.clone()],
-        ..a.clone()
-    };
-
-    let f = StructWithVecOfStructs {
-        c: vec![inner_a.clone()],
-        ..a.clone()
-    };
-
-    vec![a, b, c, d, e, f]
-}
-
-#[test]
-fn test_struct_with_vec_of_structs() {
-    let variants = get_struct_with_vec_of_structs();
-
-    test_routine(variants[0].clone(), variants.clone());
-    test_routine(variants[1].clone(), variants.clone());
-    test_routine(variants[2].clone(), variants.clone());
-    test_routine(variants[3].clone(), variants.clone());
-    test_routine(variants[4].clone(), variants.clone());
-    test_routine(variants[5].clone(), variants.clone());
-}
-
-#[derive(Clone, Debug, TreeHash, CachedTreeHash)]
-pub struct StructWithVecOfStructWithVecOfStructs {
-    pub a: Vec<StructWithVecOfStructs>,
-    pub b: u64,
-}
-
-#[test]
-fn test_struct_with_vec_of_struct_with_vec_of_structs() {
-    let structs = get_struct_with_vec_of_structs();
-
-    let variants = vec![
-        StructWithVecOfStructWithVecOfStructs {
-            a: structs[..].to_vec(),
-            b: 99,
-        },
-        StructWithVecOfStructWithVecOfStructs { a: vec![], b: 99 },
-        StructWithVecOfStructWithVecOfStructs {
-            a: structs[0..2].to_vec(),
-            b: 99,
-        },
-        StructWithVecOfStructWithVecOfStructs {
-            a: structs[0..2].to_vec(),
-            b: 100,
-        },
-        StructWithVecOfStructWithVecOfStructs {
-            a: structs[0..1].to_vec(),
-            b: 100,
-        },
-        StructWithVecOfStructWithVecOfStructs {
-            a: structs[0..4].to_vec(),
-            b: 100,
-        },
-        StructWithVecOfStructWithVecOfStructs {
-            a: structs[0..5].to_vec(),
-            b: 8,
-        },
-    ];
-
-    for v in &variants {
-        test_routine(v.clone(), variants.clone());
-    }
-}
-
-#[derive(Clone, Debug, TreeHash, CachedTreeHash)]
-pub struct Inner {
-    pub a: u64,
-    pub b: u64,
-    pub c: u64,
-    pub d: u64,
-}
-
-fn generic_test(index: usize) {
-    let inner = Inner {
-        a: 1,
-        b: 2,
-        c: 3,
-        d: 4,
-    };
-
-    let mut cache = TreeHashCache::new(&inner, 0).unwrap();
-
-    let changed_inner = match index {
-        0 => Inner {
-            a: 42,
-            ..inner.clone()
-        },
-        1 => Inner {
-            b: 42,
-            ..inner.clone()
-        },
-        2 => Inner {
-            c: 42,
-            ..inner.clone()
-        },
-        3 => Inner {
-            d: 42,
-            ..inner.clone()
-        },
-        _ => panic!("bad index"),
-    };
-
-    changed_inner.update_tree_hash_cache(&mut cache).unwrap();
-
-    let data1 = int_to_bytes32(1);
-    let data2 = int_to_bytes32(2);
-    let data3 = int_to_bytes32(3);
-    let data4 = int_to_bytes32(4);
-
-    let mut data = vec![data1, data2, data3, data4];
-
-    data[index] = int_to_bytes32(42);
-
-    let expected = merkleize(join(data));
-
-    let cache_bytes: Vec<u8> = cache.into();
-
-    assert_eq!(expected, cache_bytes);
-}
-
-#[test]
-fn cached_hash_on_inner() {
-    generic_test(0);
-    generic_test(1);
-    generic_test(2);
-    generic_test(3);
-}
-
-#[test]
-fn inner_builds() {
-    let data1 = int_to_bytes32(1);
-    let data2 = int_to_bytes32(2);
-    let data3 = int_to_bytes32(3);
-    let data4 = int_to_bytes32(4);
-
-    let data = join(vec![data1, data2, data3, data4]);
-    let expected = merkleize(data);
-
-    let inner = Inner {
-        a: 1,
-        b: 2,
-        c: 3,
-        d: 4,
-    };
-
-    let cache: Vec<u8> = TreeHashCache::new(&inner, 0).unwrap().into();
-
-    assert_eq!(expected, cache);
-}
-
-fn join(many: Vec<Vec<u8>>) -> Vec<u8> {
-    let mut all = vec![];
-    for one in many {
-        all.extend_from_slice(&mut one.clone())
-    }
-    all
-}
-
-/*
-#[derive(Clone, Debug)]
-pub struct InternalCache {
-    pub a: u64,
-    pub b: u64,
-    pub cache: Option<TreeHashCache>,
-}
-
-impl TreeHash for InternalCache {
-    fn tree_hash_type() -> TreeHashType {
-        TreeHashType::Container
-    }
-
-    fn tree_hash_packed_encoding(&self) -> Vec<u8> {
-        unreachable!("Struct should never be packed.")
-    }
-
-    fn tree_hash_packing_factor() -> usize {
-        unreachable!("Struct should never be packed.")
-    }
-
-    fn tree_hash_root(&self) -> Vec<u8> {
-        let mut leaves = Vec::with_capacity(4 * HASHSIZE);
-
-        leaves.append(&mut self.a.tree_hash_root());
-        leaves.append(&mut self.b.tree_hash_root());
-
-        efficient_merkleize(&leaves)[0..32].to_vec()
-    }
-}
-
-impl CachedTreeHash<InternalCache> for InternalCache {
-    fn update_internal_tree_hash_cache(mut self, mut old: Self) -> Result<(Self, Self), Error> {
-        let mut local_cache = old.cache;
-        old.cache = None;
-
-        if let Some(ref mut local_cache) = local_cache {
-            self.update_tree_hash_cache(&old, local_cache, 0)?;
-        } else {
-            local_cache = Some(self.new_tree_hash_cache()?)
-        }
-
-        self.cache = local_cache;
-
-        Ok((old, self))
-    }
-
-    fn cached_tree_hash_root(&self) -> Option<Vec<u8>> {
-        match &self.cache {
-            None => None,
-            Some(c) => Some(c.root()?.to_vec()),
-        }
-    }
-
-    fn clone_without_tree_hash_cache(&self) -> Self {
-        Self {
-            a: self.a,
-            b: self.b,
-            cache: None,
-        }
-    }
-}
-
-#[test]
-fn works_when_embedded() {
-    let old = InternalCache {
-        a: 99,
-        b: 99,
-        cache: None,
-    };
-
-    let mut new = old.clone_without_tree_hash_cache();
-    new.a = 1;
-    new.b = 2;
-
-    let (_old, new) = new.update_internal_tree_hash_cache(old).unwrap();
-
-    let root = new.cached_tree_hash_root().unwrap();
-
-    let leaves = vec![int_to_bytes32(1), int_to_bytes32(2)];
-    let merkle = merkleize(join(leaves));
-
-    assert_eq!(&merkle[0..32], &root[..]);
-}
-
-impl CachedTreeHash<InternalCache> for InternalCache {
-    fn new_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
-        let tree = TreeHashCache::from_leaves_and_subtrees(
-            self,
-            vec![self.a.new_tree_hash_cache()?, self.b.new_tree_hash_cache()?],
-        )?;
-
-        Ok(tree)
-    }
-
-    fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
-        let mut lengths = vec![];
-
-        lengths.push(BTreeOverlay::new(&self.a, 0)?.num_nodes());
-        lengths.push(BTreeOverlay::new(&self.b, 0)?.num_nodes());
-
-        BTreeOverlay::from_lengths(chunk_offset, lengths)
-    }
-
-    fn update_tree_hash_cache(
-        &self,
-        other: &Self,
-        cache: &mut TreeHashCache,
-        chunk: usize,
-    ) -> Result<usize, Error> {
-        let offset_handler = BTreeOverlay::new(self, chunk)?;
-
-        // Skip past the internal nodes and update any changed leaf nodes.
-        {
-            let chunk = offset_handler.first_leaf_node()?;
-            let chunk = self.a.update_tree_hash_cache(&other.a, cache, chunk)?;
-            let _chunk = self.b.update_tree_hash_cache(&other.b, cache, chunk)?;
-        }
-
-        for (&parent, children) in offset_handler.iter_internal_nodes().rev() {
-            if cache.either_modified(children)? {
-                cache.modify_chunk(parent, &cache.hash_children(children)?)?;
-            }
-        }
-
-        Ok(offset_handler.next_node)
-    }
-}
-
-fn num_nodes(num_leaves: usize) -> usize {
-    2 * num_leaves - 1
-}
-
-#[derive(Clone, Debug)]
-pub struct Inner {
-    pub a: u64,
-    pub b: u64,
-    pub c: u64,
-    pub d: u64,
-}
-
-impl TreeHash for Inner {
-    fn tree_hash_type() -> TreeHashType {
-        TreeHashType::Container
-    }
-
-    fn tree_hash_packed_encoding(&self) -> Vec<u8> {
-        unreachable!("Struct should never be packed.")
-    }
-
-    fn tree_hash_packing_factor() -> usize {
-        unreachable!("Struct should never be packed.")
-    }
-
-    fn tree_hash_root(&self) -> Vec<u8> {
-        let mut leaves = Vec::with_capacity(4 * HASHSIZE);
-
-        leaves.append(&mut self.a.tree_hash_root());
-        leaves.append(&mut self.b.tree_hash_root());
-        leaves.append(&mut self.c.tree_hash_root());
-        leaves.append(&mut self.d.tree_hash_root());
-
-        efficient_merkleize(&leaves)[0..32].to_vec()
-    }
-}
-
-impl CachedTreeHash<Inner> for Inner {
-    fn new_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
-        let tree = TreeHashCache::from_leaves_and_subtrees(
-            self,
-            vec![
-                self.a.new_tree_hash_cache()?,
-                self.b.new_tree_hash_cache()?,
-                self.c.new_tree_hash_cache()?,
-                self.d.new_tree_hash_cache()?,
-            ],
-        )?;
-
-        Ok(tree)
-    }
-
-    fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
-        let mut lengths = vec![];
-
-        lengths.push(BTreeOverlay::new(&self.a, 0)?.num_nodes());
-        lengths.push(BTreeOverlay::new(&self.b, 0)?.num_nodes());
-        lengths.push(BTreeOverlay::new(&self.c, 0)?.num_nodes());
-        lengths.push(BTreeOverlay::new(&self.d, 0)?.num_nodes());
-
-        BTreeOverlay::from_lengths(chunk_offset, lengths)
-    }
-
-    fn update_tree_hash_cache(
-        &self,
-        other: &Self,
-        cache: &mut TreeHashCache,
-        chunk: usize,
-    ) -> Result<usize, Error> {
-        let offset_handler = BTreeOverlay::new(self, chunk)?;
-
-        // Skip past the internal nodes and update any changed leaf nodes.
-        {
-            let chunk = offset_handler.first_leaf_node()?;
-            let chunk = self.a.update_tree_hash_cache(&other.a, cache, chunk)?;
-            let chunk = self.b.update_tree_hash_cache(&other.b, cache, chunk)?;
-            let chunk = self.c.update_tree_hash_cache(&other.c, cache, chunk)?;
-            let _chunk = self.d.update_tree_hash_cache(&other.d, cache, chunk)?;
-        }
-
-        for (&parent, children) in offset_handler.iter_internal_nodes().rev() {
-            if cache.either_modified(children)? {
-                cache.modify_chunk(parent, &cache.hash_children(children)?)?;
-            }
-        }
-
-        Ok(offset_handler.next_node)
-    }
-}
-
-#[derive(Clone, Debug)]
-pub struct Outer {
-    pub a: u64,
-    pub b: Inner,
-    pub c: u64,
-}
-
-impl TreeHash for Outer {
-    fn tree_hash_type() -> TreeHashType {
-        TreeHashType::Container
-    }
-
-    fn tree_hash_packed_encoding(&self) -> Vec<u8> {
-        unreachable!("Struct should never be packed.")
-    }
-
-    fn tree_hash_packing_factor() -> usize {
-        unreachable!("Struct should never be packed.")
-    }
-
-    fn tree_hash_root(&self) -> Vec<u8> {
-        let mut leaves = Vec::with_capacity(4 * HASHSIZE);
-
-        leaves.append(&mut self.a.tree_hash_root());
-        leaves.append(&mut self.b.tree_hash_root());
-        leaves.append(&mut self.c.tree_hash_root());
-
-        efficient_merkleize(&leaves)[0..32].to_vec()
-    }
-}
-
-impl CachedTreeHash<Outer> for Outer {
-    fn new_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
-        let tree = TreeHashCache::from_leaves_and_subtrees(
-            self,
-            vec![
-                self.a.new_tree_hash_cache()?,
-                self.b.new_tree_hash_cache()?,
-                self.c.new_tree_hash_cache()?,
-            ],
-        )?;
-
-        Ok(tree)
-    }
-
-    fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result<BTreeOverlay, Error> {
-        let mut lengths = vec![];
-
-        lengths.push(BTreeOverlay::new(&self.a, 0)?.num_nodes());
-        lengths.push(BTreeOverlay::new(&self.b, 0)?.num_nodes());
-        lengths.push(BTreeOverlay::new(&self.c, 0)?.num_nodes());
-
-        BTreeOverlay::from_lengths(chunk_offset, lengths)
-    }
-
-    fn update_tree_hash_cache(
-        &self,
-        other: &Self,
-        cache: &mut TreeHashCache,
-        chunk: usize,
-    ) -> Result<usize, Error> {
-        let offset_handler = BTreeOverlay::new(self, chunk)?;
-
-        // Skip past the internal nodes and update any changed leaf nodes.
-        {
-            let chunk = offset_handler.first_leaf_node()?;
-            let chunk = self.a.update_tree_hash_cache(&other.a, cache, chunk)?;
-            let chunk = self.b.update_tree_hash_cache(&other.b, cache, chunk)?;
-            let _chunk = self.c.update_tree_hash_cache(&other.c, cache, chunk)?;
-        }
-
-        for (&parent, children) in offset_handler.iter_internal_nodes().rev() {
-            if cache.either_modified(children)? {
-                cache.modify_chunk(parent, &cache.hash_children(children)?)?;
-            }
-        }
-
-        Ok(offset_handler.next_node)
-    }
-}
-
-fn join(many: Vec<Vec<u8>>) -> Vec<u8> {
-    let mut all = vec![];
-    for one in many {
-        all.extend_from_slice(&mut one.clone())
-    }
-    all
-}
-
-#[test]
-fn partial_modification_to_inner_struct() {
-    let original_inner = Inner {
-        a: 1,
-        b: 2,
-        c: 3,
-        d: 4,
-    };
-
-    let original_outer = Outer {
-        a: 0,
-        b: original_inner.clone(),
-        c: 5,
-    };
-
-    let modified_inner = Inner {
-        a: 42,
-        ..original_inner.clone()
-    };
-
-    // Modify outer
-    let modified_outer = Outer {
-        b: modified_inner.clone(),
-        ..original_outer.clone()
-    };
-
-    // Perform a differential hash
-    let mut cache_struct = TreeHashCache::new(&original_outer).unwrap();
-
-    modified_outer
-        .update_tree_hash_cache(&original_outer, &mut cache_struct, 0)
-        .unwrap();
-
-    let modified_cache: Vec<u8> = cache_struct.into();
-
-    // Generate reference data.
-    let mut data = vec![];
-    data.append(&mut int_to_bytes32(0));
-    let inner_bytes: Vec<u8> = TreeHashCache::new(&modified_inner).unwrap().into();
-    data.append(&mut int_to_bytes32(5));
-
-    let leaves = vec![
-        int_to_bytes32(0),
-        inner_bytes[0..32].to_vec(),
-        int_to_bytes32(5),
-        vec![0; 32], // padding
-    ];
-    let mut merkle = merkleize(join(leaves));
-    merkle.splice(4 * 32..5 * 32, inner_bytes);
-
-    assert_eq!(merkle.len() / HASHSIZE, 13);
-    assert_eq!(modified_cache.len() / HASHSIZE, 13);
-
-    assert_eq!(merkle, modified_cache);
-}
-
-#[test]
-fn partial_modification_to_outer() {
-    let inner = Inner {
-        a: 1,
-        b: 2,
-        c: 3,
-        d: 4,
-    };
-
-    let original_outer = Outer {
-        a: 0,
-        b: inner.clone(),
-        c: 5,
-    };
-
-    // Build the initial cache.
-    // let original_cache = original_outer.build_cache_bytes();
-
-    // Modify outer
-    let modified_outer = Outer {
-        c: 42,
-        ..original_outer.clone()
-    };
-
-    // Perform a differential hash
-    let mut cache_struct = TreeHashCache::new(&original_outer).unwrap();
-
-    modified_outer
-        .update_tree_hash_cache(&original_outer, &mut cache_struct, 0)
-        .unwrap();
-
-    let modified_cache: Vec<u8> = cache_struct.into();
-
-    // Generate reference data.
-    let mut data = vec![];
-    data.append(&mut int_to_bytes32(0));
-    let inner_bytes: Vec<u8> = TreeHashCache::new(&inner).unwrap().into();
-    data.append(&mut int_to_bytes32(5));
-
-    let leaves = vec![
-        int_to_bytes32(0),
-        inner_bytes[0..32].to_vec(),
-        int_to_bytes32(42),
-        vec![0; 32], // padding
-    ];
-    let mut merkle = merkleize(join(leaves));
-    merkle.splice(4 * 32..5 * 32, inner_bytes);
-
-    assert_eq!(merkle.len() / HASHSIZE, 13);
-    assert_eq!(modified_cache.len() / HASHSIZE, 13);
-
-    assert_eq!(merkle, modified_cache);
-}
-
-#[test]
-fn outer_builds() {
-    let inner = Inner {
-        a: 1,
-        b: 2,
-        c: 3,
-        d: 4,
-    };
-
-    let outer = Outer {
-        a: 0,
-        b: inner.clone(),
-        c: 5,
-    };
-
-    // Build the function output.
-    let cache: Vec<u8> = TreeHashCache::new(&outer).unwrap().into();
-
-    // Generate reference data.
-    let mut data = vec![];
-    data.append(&mut int_to_bytes32(0));
-    let inner_bytes: Vec<u8> = TreeHashCache::new(&inner).unwrap().into();
-    data.append(&mut int_to_bytes32(5));
-
-    let leaves = vec![
-        int_to_bytes32(0),
-        inner_bytes[0..32].to_vec(),
-        int_to_bytes32(5),
-        vec![0; 32], // padding
-    ];
-    let mut merkle = merkleize(join(leaves));
-    merkle.splice(4 * 32..5 * 32, inner_bytes);
-
-    assert_eq!(merkle.len() / HASHSIZE, 13);
-    assert_eq!(cache.len() / HASHSIZE, 13);
-
-    assert_eq!(merkle, cache);
-}
-
-fn mix_in_length(root: &mut [u8], len: usize) {
-    let mut bytes = root.to_vec();
-    bytes.append(&mut int_to_bytes32(len as u64));
-
-    root.copy_from_slice(&hash(&bytes));
-}
-
-/// Generic test that covers:
-///
-/// 1. Produce a new cache from `original`.
-/// 2. Do a differential hash between `original` and `modified`.
-/// 3. Test that the cache generated matches the one we generate manually.
-///
-/// In effect it ensures that we can do a differential hash between two `Vec<u64>`.
-fn test_u64_vec_modifications(original: Vec<u64>, modified: Vec<u64>) {
-    // Generate initial cache.
-    let original_cache: Vec<u8> = TreeHashCache::new(&original).unwrap().into();
-
-    // Perform a differential hash
-    let mut cache_struct = TreeHashCache::from_bytes(original_cache.clone(), false).unwrap();
-    modified
-        .update_tree_hash_cache(&original, &mut cache_struct, 0)
-        .unwrap();
-    let modified_cache: Vec<u8> = cache_struct.into();
-
-    // Generate reference data.
-    let mut data = vec![];
-    for i in &modified {
-        data.append(&mut int_to_bytes8(*i));
-    }
-    let data = sanitise_bytes(data);
-    let mut expected = merkleize(data);
-
-    mix_in_length(&mut expected[0..HASHSIZE], modified.len());
-
-    assert_eq!(expected, modified_cache);
-    assert_eq!(&expected[0..32], &modified.tree_hash_root()[..]);
-}
-
-#[test]
-fn partial_modification_u64_vec() {
-    let n: u64 = 2_u64.pow(5);
-
-    let original_vec: Vec<u64> = (0..n).collect();
-
-    let mut modified_vec = original_vec.clone();
-    modified_vec[n as usize - 1] = 42;
-
-    test_u64_vec_modifications(original_vec, modified_vec);
-}
-
-#[test]
-fn shortened_u64_vec_len_within_pow_2_boundary() {
-    let n: u64 = 2_u64.pow(5) - 1;
-
-    let original_vec: Vec<u64> = (0..n).collect();
-
-    let mut modified_vec = original_vec.clone();
-    modified_vec.pop();
-
-    test_u64_vec_modifications(original_vec, modified_vec);
-}
-
-#[test]
-fn shortened_u64_vec_len_outside_pow_2_boundary() {
-    let original_vec: Vec<u64> = (0..2_u64.pow(6)).collect();
-
-    let modified_vec: Vec<u64> = (0..2_u64.pow(5)).collect();
-
-    test_u64_vec_modifications(original_vec, modified_vec);
-}
-
-#[test]
-fn extended_u64_vec_len_within_pow_2_boundary() {
-    let n: u64 = 2_u64.pow(5) - 2;
-
-    let original_vec: Vec<u64> = (0..n).collect();
-
-    let mut modified_vec = original_vec.clone();
-    modified_vec.push(42);
-
-    test_u64_vec_modifications(original_vec, modified_vec);
-}
-
-#[test]
-fn extended_u64_vec_len_outside_pow_2_boundary() {
-    let original_vec: Vec<u64> = (0..2_u64.pow(5)).collect();
-
-    let modified_vec: Vec<u64> = (0..2_u64.pow(6)).collect();
-
-    test_u64_vec_modifications(original_vec, modified_vec);
-}
-
-#[test]
-fn large_vec_of_u64_builds() {
-    let n: u64 = 50;
-
-    let my_vec: Vec<u64> = (0..n).collect();
-
-    // Generate function output.
-    let cache: Vec<u8> = TreeHashCache::new(&my_vec).unwrap().into();
-
-    // Generate reference data.
-    let mut data = vec![];
-    for i in &my_vec {
-        data.append(&mut int_to_bytes8(*i));
-    }
-    let data = sanitise_bytes(data);
-    let expected = merkleize(data);
-
-    assert_eq!(expected, cache);
-}
-
-/// Generic test that covers:
-///
-/// 1. Produce a new cache from `original`.
-/// 2. Do a differential hash between `original` and `modified`.
-/// 3. Test that the cache generated matches the one we generate manually.
-///
-/// The `reference` vec is used to build the tree hash cache manually. `Inner` is just 4x `u64`, so
-/// you can represent 2x `Inner` with a `reference` vec of len 8.
-///
-/// In effect it ensures that we can do a differential hash between two `Vec<Inner>`.
-fn test_inner_vec_modifications(original: Vec<Inner>, modified: Vec<Inner>, reference: Vec<u64>) {
-    let mut cache = TreeHashCache::new(&original).unwrap();
-
-    modified
-        .update_tree_hash_cache(&original, &mut cache, 0)
-        .unwrap();
-    let modified_cache: Vec<u8> = cache.into();
-
-    // Build the reference vec.
-
-    let mut leaves = vec![];
-    let mut full_bytes = vec![];
-
-    for n in reference.chunks(4) {
-        let mut merkle = merkleize(join(vec![
-            int_to_bytes32(n[0]),
-            int_to_bytes32(n[1]),
-            int_to_bytes32(n[2]),
-            int_to_bytes32(n[3]),
-        ]));
-        leaves.append(&mut merkle[0..HASHSIZE].to_vec());
-        full_bytes.append(&mut merkle);
-    }
-
-    let num_leaves = leaves.len() / HASHSIZE;
-    let mut expected = merkleize(leaves);
-
-    let num_internal_nodes = num_leaves.next_power_of_two() - 1;
-    expected.splice(num_internal_nodes * HASHSIZE.., full_bytes);
-
-    for _ in num_leaves..num_leaves.next_power_of_two() {
-        expected.append(&mut vec![0; HASHSIZE]);
-    }
-
-    mix_in_length(&mut expected[0..HASHSIZE], modified.len());
-
-    // Compare the cached tree to the reference tree.
-    assert_trees_eq(&expected, &modified_cache);
-    assert_eq!(&expected[0..32], &modified.tree_hash_root()[..]);
-}
-
-#[test]
-fn partial_modification_of_vec_of_inner() {
-    let original = vec![
-        Inner {
-            a: 0,
-            b: 1,
-            c: 2,
-            d: 3,
-        },
-        Inner {
-            a: 4,
-            b: 5,
-            c: 6,
-            d: 7,
-        },
-        Inner {
-            a: 8,
-            b: 9,
-            c: 10,
-            d: 11,
-        },
-    ];
-
-    let mut modified = original.clone();
-    modified[1].a = 42;
-
-    let mut reference_vec: Vec<u64> = (0..12).collect();
-    reference_vec[4] = 42;
-
-    test_inner_vec_modifications(original, modified, reference_vec);
-}
-
-#[test]
-fn shortened_vec_of_inner_within_power_of_two_boundary() {
-    let original = vec![
-        Inner {
-            a: 0,
-            b: 1,
-            c: 2,
-            d: 3,
-        },
-        Inner {
-            a: 4,
-            b: 5,
-            c: 6,
-            d: 7,
-        },
-        Inner {
-            a: 8,
-            b: 9,
-            c: 10,
-            d: 11,
-        },
-        Inner {
-            a: 12,
-            b: 13,
-            c: 14,
-            d: 15,
-        },
-    ];
-
-    let mut modified = original.clone();
-    modified.pop(); // remove the last element from the list.
-
-    let reference_vec: Vec<u64> = (0..12).collect();
-
-    test_inner_vec_modifications(original, modified, reference_vec);
-}
-
-#[test]
-fn shortened_vec_of_inner_outside_power_of_two_boundary() {
-    let original = vec![
-        Inner {
-            a: 0,
-            b: 1,
-            c: 2,
-            d: 3,
-        },
-        Inner {
-            a: 4,
-            b: 5,
-            c: 6,
-            d: 7,
-        },
-        Inner {
-            a: 8,
-            b: 9,
-            c: 10,
-            d: 11,
-        },
-        Inner {
-            a: 12,
-            b: 13,
-            c: 14,
-            d: 15,
-        },
-        Inner {
-            a: 16,
-            b: 17,
-            c: 18,
-            d: 19,
-        },
-    ];
-
-    let mut modified = original.clone();
-    modified.pop(); // remove the last element from the list.
-
-    let reference_vec: Vec<u64> = (0..16).collect();
-
-    test_inner_vec_modifications(original, modified, reference_vec);
-}
-
-#[test]
-fn lengthened_vec_of_inner_within_power_of_two_boundary() {
-    let original = vec![
-        Inner {
-            a: 0,
-            b: 1,
-            c: 2,
-            d: 3,
-        },
-        Inner {
-            a: 4,
-            b: 5,
-            c: 6,
-            d: 7,
-        },
-        Inner {
-            a: 8,
-            b: 9,
-            c: 10,
-            d: 11,
-        },
-    ];
-
-    let mut modified = original.clone();
-    modified.push(Inner {
-        a: 12,
-        b: 13,
-        c: 14,
-        d: 15,
-    });
-
-    let reference_vec: Vec<u64> = (0..16).collect();
-
-    test_inner_vec_modifications(original, modified, reference_vec);
-}
-
-#[test]
-fn lengthened_vec_of_inner_outside_power_of_two_boundary() {
-    let original = vec![
-        Inner {
-            a: 0,
-            b: 1,
-            c: 2,
-            d: 3,
-        },
-        Inner {
-            a: 4,
-            b: 5,
-            c: 6,
-            d: 7,
-        },
-        Inner {
-            a: 8,
-            b: 9,
-            c: 10,
-            d: 11,
-        },
-        Inner {
-            a: 12,
-            b: 13,
-            c: 14,
-            d: 15,
-        },
-    ];
-
-    let mut modified = original.clone();
-    modified.push(Inner {
-        a: 16,
-        b: 17,
-        c: 18,
-        d: 19,
-    });
-
-    let reference_vec: Vec<u64> = (0..20).collect();
-
-    test_inner_vec_modifications(original, modified, reference_vec);
-}
-
-#[test]
-fn vec_of_inner_builds() {
-    let numbers: Vec<u64> = (0..12).collect();
-
-    let mut leaves = vec![];
-    let mut full_bytes = vec![];
-
-    for n in numbers.chunks(4) {
-        let mut merkle = merkleize(join(vec![
-            int_to_bytes32(n[0]),
-            int_to_bytes32(n[1]),
-            int_to_bytes32(n[2]),
-            int_to_bytes32(n[3]),
-        ]));
-        leaves.append(&mut merkle[0..HASHSIZE].to_vec());
-        full_bytes.append(&mut merkle);
-    }
-
-    let mut expected = merkleize(leaves);
-    expected.splice(3 * HASHSIZE.., full_bytes);
-    expected.append(&mut vec![0; HASHSIZE]);
-
-    let my_vec = vec![
-        Inner {
-            a: 0,
-            b: 1,
-            c: 2,
-            d: 3,
-        },
-        Inner {
-            a: 4,
-            b: 5,
-            c: 6,
-            d: 7,
-        },
-        Inner {
-            a: 8,
-            b: 9,
-            c: 10,
-            d: 11,
-        },
-    ];
-
-    let cache: Vec<u8> = TreeHashCache::new(&my_vec).unwrap().into();
-
-    assert_trees_eq(&expected, &cache);
-}
-
-/// Provides detailed assertions when comparing merkle trees.
-fn assert_trees_eq(a: &[u8], b: &[u8]) {
-    assert_eq!(a.len(), b.len(), "Byte lens different");
-    for i in (0..a.len() / HASHSIZE).rev() {
-        let range = i * HASHSIZE..(i + 1) * HASHSIZE;
-        assert_eq!(
-            a[range.clone()],
-            b[range],
-            "Chunk {}/{} different \n\n a: {:?} \n\n b: {:?}",
-            i,
-            a.len() / HASHSIZE,
-            a,
-            b,
-        );
-    }
-}
-
-#[test]
-fn vec_of_u64_builds() {
-    let data = join(vec![
-        int_to_bytes8(1),
-        int_to_bytes8(2),
-        int_to_bytes8(3),
-        int_to_bytes8(4),
-        int_to_bytes8(5),
-        vec![0; 32 - 8], // padding
-    ]);
-
-    let expected = merkleize(data);
-
-    let my_vec = vec![1, 2, 3, 4, 5];
-
-    //
-    // Note: the length is not mixed-in in this example. The user must ensure the length is
-    // mixed-in.
-    //
-
-    let cache: Vec<u8> = TreeHashCache::new(&my_vec).unwrap().into();
-
-    assert_eq!(expected, cache);
-}
-
-#[test]
-fn vec_does_mix_in_len() {
-    let data = join(vec![
-        int_to_bytes8(1),
-        int_to_bytes8(2),
-        int_to_bytes8(3),
-        int_to_bytes8(4),
-        int_to_bytes8(5),
-        vec![0; 32 - 8], // padding
-    ]);
-
-    let tree = merkleize(data);
-
-    let my_vec: Vec<u64> = vec![1, 2, 3, 4, 5];
-
-    let mut expected = vec![0; 32];
-    expected.copy_from_slice(&tree[0..HASHSIZE]);
-    expected.append(&mut int_to_bytes32(my_vec.len() as u64));
-    let expected = hash(&expected);
-
-    assert_eq!(&expected[0..HASHSIZE], &my_vec.tree_hash_root()[..]);
-}
-
-#[test]
-fn merkleize_odd() {
-    let data = join(vec![
-        int_to_bytes32(1),
-        int_to_bytes32(2),
-        int_to_bytes32(3),
-        int_to_bytes32(4),
-        int_to_bytes32(5),
-    ]);
-
-    let merkle = merkleize(sanitise_bytes(data));
-
-    let expected_len = num_nodes(8) * BYTES_PER_CHUNK;
-
-    assert_eq!(merkle.len(), expected_len);
-}
-
-fn generic_test(index: usize) {
-    let inner = Inner {
-        a: 1,
-        b: 2,
-        c: 3,
-        d: 4,
-    };
-
-    let cache: Vec<u8> = TreeHashCache::new(&inner).unwrap().into();
-
-    let changed_inner = match index {
-        0 => Inner {
-            a: 42,
-            ..inner.clone()
-        },
-        1 => Inner {
-            b: 42,
-            ..inner.clone()
-        },
-        2 => Inner {
-            c: 42,
-            ..inner.clone()
-        },
-        3 => Inner {
-            d: 42,
-            ..inner.clone()
-        },
-        _ => panic!("bad index"),
-    };
-
-    let mut cache_struct = TreeHashCache::from_bytes(cache.clone(), false).unwrap();
-
-    changed_inner
-        .update_tree_hash_cache(&inner, &mut cache_struct, 0)
-        .unwrap();
-
-    // assert_eq!(*cache_struct.hash_count, 3);
-
-    let new_tree_hash_cache: Vec<u8> = cache_struct.into();
-
-    let data1 = int_to_bytes32(1);
-    let data2 = int_to_bytes32(2);
-    let data3 = int_to_bytes32(3);
-    let data4 = int_to_bytes32(4);
-
-    let mut data = vec![data1, data2, data3, data4];
-
-    data[index] = int_to_bytes32(42);
-
-    let expected = merkleize(join(data));
-
-    assert_eq!(expected, new_tree_hash_cache);
-}
-
-#[test]
-fn cached_hash_on_inner() {
-    generic_test(0);
-    generic_test(1);
-    generic_test(2);
-    generic_test(3);
-}
-
-#[test]
-fn inner_builds() {
-    let data1 = int_to_bytes32(1);
-    let data2 = int_to_bytes32(2);
-    let data3 = int_to_bytes32(3);
-    let data4 = int_to_bytes32(4);
-
-    let data = join(vec![data1, data2, data3, data4]);
-    let expected = merkleize(data);
-
-    let inner = Inner {
-        a: 1,
-        b: 2,
-        c: 3,
-        d: 4,
-    };
-
-    let cache: Vec<u8> = TreeHashCache::new(&inner).unwrap().into();
-
-    assert_eq!(expected, cache);
-}
-
-#[test]
-fn merkleize_4_leaves() {
-    let data1 = hash(&int_to_bytes32(1));
-    let data2 = hash(&int_to_bytes32(2));
-    let data3 = hash(&int_to_bytes32(3));
-    let data4 = hash(&int_to_bytes32(4));
-
-    let data = join(vec![
-        data1.clone(),
-        data2.clone(),
-        data3.clone(),
-        data4.clone(),
-    ]);
-
-    let cache = merkleize(data);
-
-    let hash_12 = {
-        let mut joined = vec![];
-        joined.append(&mut data1.clone());
-        joined.append(&mut data2.clone());
-        hash(&joined)
-    };
-    let hash_34 = {
-        let mut joined = vec![];
-        joined.append(&mut data3.clone());
-        joined.append(&mut data4.clone());
-        hash(&joined)
-    };
-    let hash_hash12_hash_34 = {
-        let mut joined = vec![];
-        joined.append(&mut hash_12.clone());
-        joined.append(&mut hash_34.clone());
-        hash(&joined)
-    };
-
-    for (i, chunk) in cache.chunks(HASHSIZE).enumerate().rev() {
-        let expected = match i {
-            0 => hash_hash12_hash_34.clone(),
-            1 => hash_12.clone(),
-            2 => hash_34.clone(),
-            3 => data1.clone(),
-            4 => data2.clone(),
-            5 => data3.clone(),
-            6 => data4.clone(),
-            _ => vec![],
-        };
-
-        assert_eq!(chunk, &expected[..], "failed at {}", i);
-    }
-}
-*/
diff --git a/eth2/utils/tree_hash_derive/Cargo.toml b/eth2/utils/tree_hash_derive/Cargo.toml
index f227d7954..8544108a7 100644
--- a/eth2/utils/tree_hash_derive/Cargo.toml
+++ b/eth2/utils/tree_hash_derive/Cargo.toml
@@ -10,6 +10,7 @@ proc-macro = true
 
 [dev-dependencies]
 tree_hash = { path = "../tree_hash" }
+cached_tree_hash = { path = "../cached_tree_hash" }
 
 [dependencies]
 syn = "0.15"
diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs
index 9b35512a9..29db95509 100644
--- a/eth2/utils/tree_hash_derive/src/lib.rs
+++ b/eth2/utils/tree_hash_derive/src/lib.rs
@@ -58,9 +58,9 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
     let num_items = idents_a.len();
 
     let output = quote! {
-        impl tree_hash::CachedTreeHash<#name> for #name {
-            fn new_tree_hash_cache(&self, depth: usize) -> Result<tree_hash::TreeHashCache, tree_hash::Error> {
-                let tree = tree_hash::TreeHashCache::from_leaves_and_subtrees(
+        impl cached_tree_hash::CachedTreeHash<#name> for #name {
+            fn new_tree_hash_cache(&self, depth: usize) -> Result<cached_tree_hash::TreeHashCache, cached_tree_hash::Error> {
+                let tree = cached_tree_hash::TreeHashCache::from_leaves_and_subtrees(
                     self,
                     vec![
                         #(
@@ -74,23 +74,23 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
             }
 
             fn num_tree_hash_cache_chunks(&self) -> usize {
-                tree_hash::BTreeOverlay::new(self, 0, 0)
+                cached_tree_hash::BTreeOverlay::new(self, 0, 0)
                     .and_then(|o| Ok(o.num_chunks()))
                     .unwrap_or_else(|_| 1)
             }
 
-            fn tree_hash_cache_overlay(&self, chunk_offset: usize, depth: usize) -> Result<tree_hash::BTreeOverlay, tree_hash::Error> {
+            fn tree_hash_cache_overlay(&self, chunk_offset: usize, depth: usize) -> Result<cached_tree_hash::BTreeOverlay, cached_tree_hash::Error> {
                 let mut lengths = vec![];
 
                 #(
                     lengths.push(self.#idents_b.num_tree_hash_cache_chunks());
                 )*
 
-                tree_hash::BTreeOverlay::from_lengths(chunk_offset, #num_items, depth, lengths)
+                cached_tree_hash::BTreeOverlay::from_lengths(chunk_offset, #num_items, depth, lengths)
             }
 
-            fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
-                let overlay = BTreeOverlay::new(self, cache.chunk_index, 0)?;
+            fn update_tree_hash_cache(&self, cache: &mut cached_tree_hash::TreeHashCache) -> Result<(), cached_tree_hash::Error> {
+                let overlay = cached_tree_hash::BTreeOverlay::new(self, cache.chunk_index, 0)?;
 
                 // Skip the chunk index to the first leaf node of this struct.
                 cache.chunk_index = overlay.first_leaf_node();
diff --git a/eth2/utils/tree_hash_derive/tests/tests.rs b/eth2/utils/tree_hash_derive/tests/tests.rs
index 10d0aa853..5b1bb481f 100644
--- a/eth2/utils/tree_hash_derive/tests/tests.rs
+++ b/eth2/utils/tree_hash_derive/tests/tests.rs
@@ -1,4 +1,5 @@
-use tree_hash::{CachedTreeHash, SignedRoot, TreeHash};
+use cached_tree_hash::{CachedTreeHash, CachedTreeHasher};
+use tree_hash::{SignedRoot, TreeHash};
 use tree_hash_derive::{CachedTreeHash, SignedRoot, TreeHash};
 
 #[derive(Clone, Debug, TreeHash, CachedTreeHash)]
@@ -13,18 +14,17 @@ fn test_standard_and_cached<T>(original: &T, modified: &T)
 where
     T: CachedTreeHash<T>,
 {
-    let mut cache = original.new_tree_hash_cache().unwrap();
+    // let mut cache = original.new_tree_hash_cache().unwrap();
+    let mut hasher = CachedTreeHasher::new(original).unwrap();
 
     let standard_root = original.tree_hash_root();
-    let cached_root = cache.root().unwrap().to_vec();
+    let cached_root = hasher.tree_hash_root().unwrap();
     assert_eq!(standard_root, cached_root);
 
     // Test after a modification
-    modified
-        .update_tree_hash_cache(&original, &mut cache, 0)
-        .unwrap();
+    hasher.update(modified).unwrap();
     let standard_root = modified.tree_hash_root();
-    let cached_root = cache.root().unwrap().to_vec();
+    let cached_root = hasher.tree_hash_root().unwrap();
     assert_eq!(standard_root, cached_root);
 }
 

From f1d8224d89abdf19eec6e6f5a531fb65eb875c50 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Fri, 26 Apr 2019 10:37:50 +1000
Subject: [PATCH 107/137] Further tidy `tree_hash` crate.

---
 eth2/utils/tree_hash/src/impls.rs          |  2 +
 eth2/utils/tree_hash/src/lib.rs            | 72 +---------------------
 eth2/utils/tree_hash/src/merkleize.rs      | 69 +++++++++++++++++++++
 eth2/utils/tree_hash_derive/src/lib.rs     |  4 +-
 eth2/utils/tree_hash_derive/tests/tests.rs |  8 +--
 5 files changed, 79 insertions(+), 76 deletions(-)
 create mode 100644 eth2/utils/tree_hash/src/merkleize.rs

diff --git a/eth2/utils/tree_hash/src/impls.rs b/eth2/utils/tree_hash/src/impls.rs
index 01b165150..a32e3844c 100644
--- a/eth2/utils/tree_hash/src/impls.rs
+++ b/eth2/utils/tree_hash/src/impls.rs
@@ -1,5 +1,7 @@
 use super::*;
+use crate::merkleize::merkle_root;
 use ethereum_types::H256;
+use hashing::hash;
 use int_to_bytes::int_to_bytes32;
 
 macro_rules! impl_for_bitsize {
diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs
index 6ed0247f1..5e8bfdf8e 100644
--- a/eth2/utils/tree_hash/src/lib.rs
+++ b/eth2/utils/tree_hash/src/lib.rs
@@ -1,6 +1,5 @@
-use hashing::hash;
-
 pub mod impls;
+pub mod merkleize;
 
 pub const BYTES_PER_CHUNK: usize = 32;
 pub const HASHSIZE: usize = 32;
@@ -28,73 +27,6 @@ pub trait SignedRoot: TreeHash {
     fn signed_root(&self) -> Vec<u8>;
 }
 
-pub fn merkle_root(bytes: &[u8]) -> Vec<u8> {
-    // TODO: replace this with a more memory efficient method.
-    efficient_merkleize(&bytes)[0..32].to_vec()
-}
-
-pub fn efficient_merkleize(bytes: &[u8]) -> Vec<u8> {
-    // If the bytes are just one chunk (or less than one chunk) just return them.
-    if bytes.len() <= HASHSIZE {
-        let mut o = bytes.to_vec();
-        o.resize(HASHSIZE, 0);
-        return o;
-    }
-
-    let leaves = num_sanitized_leaves(bytes.len());
-    let nodes = num_nodes(leaves);
-    let internal_nodes = nodes - leaves;
-
-    let num_bytes = std::cmp::max(internal_nodes, 1) * HASHSIZE + bytes.len();
-
-    let mut o: Vec<u8> = vec![0; internal_nodes * HASHSIZE];
-
-    o.append(&mut bytes.to_vec());
-
-    assert_eq!(o.len(), num_bytes);
-
-    let empty_chunk_hash = hash(&[0; MERKLE_HASH_CHUNK]);
-
-    let mut i = nodes * HASHSIZE;
-    let mut j = internal_nodes * HASHSIZE;
-
-    while i >= MERKLE_HASH_CHUNK {
-        i -= MERKLE_HASH_CHUNK;
-
-        j -= HASHSIZE;
-        let hash = match o.get(i..i + MERKLE_HASH_CHUNK) {
-            // All bytes are available, hash as ususal.
-            Some(slice) => hash(slice),
-            // Unable to get all the bytes.
-            None => {
-                match o.get(i..) {
-                    // Able to get some of the bytes, pad them out.
-                    Some(slice) => {
-                        let mut bytes = slice.to_vec();
-                        bytes.resize(MERKLE_HASH_CHUNK, 0);
-                        hash(&bytes)
-                    }
-                    // Unable to get any bytes, use the empty-chunk hash.
-                    None => empty_chunk_hash.clone(),
-                }
-            }
-        };
-
-        o[j..j + HASHSIZE].copy_from_slice(&hash);
-    }
-
-    o
-}
-
-fn num_sanitized_leaves(num_bytes: usize) -> usize {
-    let leaves = (num_bytes + HASHSIZE - 1) / HASHSIZE;
-    leaves.next_power_of_two()
-}
-
-fn num_nodes(num_leaves: usize) -> usize {
-    2 * num_leaves - 1
-}
-
 #[macro_export]
 macro_rules! tree_hash_ssz_encoding_as_vector {
     ($type: ident) => {
@@ -112,7 +44,7 @@ macro_rules! tree_hash_ssz_encoding_as_vector {
             }
 
             fn tree_hash_root(&self) -> Vec<u8> {
-                tree_hash::merkle_root(&ssz::ssz_encode(self))
+                tree_hash::merkleize::merkle_root(&ssz::ssz_encode(self))
             }
         }
     };
diff --git a/eth2/utils/tree_hash/src/merkleize.rs b/eth2/utils/tree_hash/src/merkleize.rs
new file mode 100644
index 000000000..9482895ec
--- /dev/null
+++ b/eth2/utils/tree_hash/src/merkleize.rs
@@ -0,0 +1,69 @@
+use super::*;
+use hashing::hash;
+
+pub fn merkle_root(bytes: &[u8]) -> Vec<u8> {
+    // TODO: replace this with a more memory efficient method.
+    efficient_merkleize(&bytes)[0..32].to_vec()
+}
+
+pub fn efficient_merkleize(bytes: &[u8]) -> Vec<u8> {
+    // If the bytes are just one chunk (or less than one chunk) just return them.
+    if bytes.len() <= HASHSIZE {
+        let mut o = bytes.to_vec();
+        o.resize(HASHSIZE, 0);
+        return o;
+    }
+
+    let leaves = num_sanitized_leaves(bytes.len());
+    let nodes = num_nodes(leaves);
+    let internal_nodes = nodes - leaves;
+
+    let num_bytes = std::cmp::max(internal_nodes, 1) * HASHSIZE + bytes.len();
+
+    let mut o: Vec<u8> = vec![0; internal_nodes * HASHSIZE];
+
+    o.append(&mut bytes.to_vec());
+
+    assert_eq!(o.len(), num_bytes);
+
+    let empty_chunk_hash = hash(&[0; MERKLE_HASH_CHUNK]);
+
+    let mut i = nodes * HASHSIZE;
+    let mut j = internal_nodes * HASHSIZE;
+
+    while i >= MERKLE_HASH_CHUNK {
+        i -= MERKLE_HASH_CHUNK;
+
+        j -= HASHSIZE;
+        let hash = match o.get(i..i + MERKLE_HASH_CHUNK) {
+            // All bytes are available, hash as ususal.
+            Some(slice) => hash(slice),
+            // Unable to get all the bytes.
+            None => {
+                match o.get(i..) {
+                    // Able to get some of the bytes, pad them out.
+                    Some(slice) => {
+                        let mut bytes = slice.to_vec();
+                        bytes.resize(MERKLE_HASH_CHUNK, 0);
+                        hash(&bytes)
+                    }
+                    // Unable to get any bytes, use the empty-chunk hash.
+                    None => empty_chunk_hash.clone(),
+                }
+            }
+        };
+
+        o[j..j + HASHSIZE].copy_from_slice(&hash);
+    }
+
+    o
+}
+
+fn num_sanitized_leaves(num_bytes: usize) -> usize {
+    let leaves = (num_bytes + HASHSIZE - 1) / HASHSIZE;
+    leaves.next_power_of_two()
+}
+
+fn num_nodes(num_leaves: usize) -> usize {
+    2 * num_leaves - 1
+}
diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs
index 29db95509..b6ba10e02 100644
--- a/eth2/utils/tree_hash_derive/src/lib.rs
+++ b/eth2/utils/tree_hash_derive/src/lib.rs
@@ -149,7 +149,7 @@ pub fn tree_hash_derive(input: TokenStream) -> TokenStream {
                     leaves.append(&mut self.#idents.tree_hash_root());
                 )*
 
-                tree_hash::merkle_root(&leaves)
+                tree_hash::merkleize::merkle_root(&leaves)
             }
         }
     };
@@ -179,7 +179,7 @@ pub fn tree_hash_signed_root_derive(input: TokenStream) -> TokenStream {
                     leaves.append(&mut self.#idents.tree_hash_root());
                 )*
 
-                tree_hash::merkle_root(&leaves)
+                tree_hash::merkleize::merkle_root(&leaves)
             }
         }
     };
diff --git a/eth2/utils/tree_hash_derive/tests/tests.rs b/eth2/utils/tree_hash_derive/tests/tests.rs
index 5b1bb481f..2166fd146 100644
--- a/eth2/utils/tree_hash_derive/tests/tests.rs
+++ b/eth2/utils/tree_hash_derive/tests/tests.rs
@@ -1,5 +1,5 @@
 use cached_tree_hash::{CachedTreeHash, CachedTreeHasher};
-use tree_hash::{SignedRoot, TreeHash};
+use tree_hash::{merkleize::merkle_root, SignedRoot, TreeHash};
 use tree_hash_derive::{CachedTreeHash, SignedRoot, TreeHash};
 
 #[derive(Clone, Debug, TreeHash, CachedTreeHash)]
@@ -120,7 +120,7 @@ impl CryptoKitties {
         leaves.append(&mut self.best_kitty.tree_hash_root());
         leaves.append(&mut self.worst_kitty.tree_hash_root());
         leaves.append(&mut self.kitties.tree_hash_root());
-        tree_hash::merkle_root(&leaves)
+        merkle_root(&leaves)
     }
 }
 
@@ -158,14 +158,14 @@ impl Casper {
         let mut list = Vec::new();
         list.append(&mut self.friendly.tree_hash_root());
         list.append(&mut self.friends.tree_hash_root());
-        tree_hash::merkle_root(&list)
+        merkle_root(&list)
     }
 
     fn expected_tree_hash(&self) -> Vec<u8> {
         let mut list = Vec::new();
         list.append(&mut self.friendly.tree_hash_root());
         list.append(&mut self.dead.tree_hash_root());
-        tree_hash::merkle_root(&list)
+        merkle_root(&list)
     }
 }
 

From 15f81c090701620a0ff700f70ebb2d2420112526 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Fri, 26 Apr 2019 11:15:17 +1000
Subject: [PATCH 108/137] Implement CachedTreeHash for TreeHashVector

---
 eth2/types/Cargo.toml                        |   1 +
 eth2/types/src/tree_hash_vector.rs           |  38 +++
 eth2/utils/cached_tree_hash/src/impls.rs     |   2 +-
 eth2/utils/cached_tree_hash/src/impls/vec.rs | 316 ++++++++++---------
 eth2/utils/cached_tree_hash/src/lib.rs       |   2 +-
 eth2/utils/tree_hash/src/impls.rs            |   4 +-
 eth2/utils/tree_hash/src/lib.rs              |   1 +
 7 files changed, 215 insertions(+), 149 deletions(-)

diff --git a/eth2/types/Cargo.toml b/eth2/types/Cargo.toml
index b88e1d4cf..36e251d7e 100644
--- a/eth2/types/Cargo.toml
+++ b/eth2/types/Cargo.toml
@@ -7,6 +7,7 @@ edition = "2018"
 [dependencies]
 bls = { path = "../utils/bls" }
 boolean-bitfield = { path = "../utils/boolean-bitfield" }
+cached_tree_hash = { path = "../utils/cached_tree_hash" }
 dirs = "1.0"
 derivative = "1.0"
 ethereum-types = "0.5"
diff --git a/eth2/types/src/tree_hash_vector.rs b/eth2/types/src/tree_hash_vector.rs
index c90a77c8d..a3e63c405 100644
--- a/eth2/types/src/tree_hash_vector.rs
+++ b/eth2/types/src/tree_hash_vector.rs
@@ -1,4 +1,5 @@
 use crate::test_utils::{RngCore, TestRandom};
+use cached_tree_hash::CachedTreeHash;
 use serde_derive::{Deserialize, Serialize};
 use ssz::{Decodable, DecodeError, Encodable, SszStream};
 use std::ops::{Deref, DerefMut};
@@ -54,6 +55,43 @@ where
     }
 }
 
+impl<T> CachedTreeHash<TreeHashVector<T>> for TreeHashVector<T>
+where
+    T: CachedTreeHash<T> + TreeHash,
+{
+    fn new_tree_hash_cache(
+        &self,
+        depth: usize,
+    ) -> Result<cached_tree_hash::TreeHashCache, cached_tree_hash::Error> {
+        let (cache, _overlay) = cached_tree_hash::impls::vec::new_tree_hash_cache(self, depth)?;
+
+        Ok(cache)
+    }
+
+    fn num_tree_hash_cache_chunks(&self) -> usize {
+        cached_tree_hash::BTreeOverlay::new(self, 0, 0)
+            .and_then(|o| Ok(o.num_chunks()))
+            .unwrap_or_else(|_| 1)
+    }
+
+    fn tree_hash_cache_overlay(
+        &self,
+        chunk_offset: usize,
+        depth: usize,
+    ) -> Result<cached_tree_hash::BTreeOverlay, cached_tree_hash::Error> {
+        cached_tree_hash::impls::vec::produce_overlay(self, chunk_offset, depth)
+    }
+
+    fn update_tree_hash_cache(
+        &self,
+        cache: &mut cached_tree_hash::TreeHashCache,
+    ) -> Result<(), cached_tree_hash::Error> {
+        cached_tree_hash::impls::vec::update_tree_hash_cache(self, cache)?;
+
+        Ok(())
+    }
+}
+
 impl<T> Encodable for TreeHashVector<T>
 where
     T: Encodable,
diff --git a/eth2/utils/cached_tree_hash/src/impls.rs b/eth2/utils/cached_tree_hash/src/impls.rs
index 80259632d..827b0fbe9 100644
--- a/eth2/utils/cached_tree_hash/src/impls.rs
+++ b/eth2/utils/cached_tree_hash/src/impls.rs
@@ -1,7 +1,7 @@
 use super::*;
 use crate::merkleize::merkleize;
 
-mod vec;
+pub mod vec;
 
 impl CachedTreeHash<u64> for u64 {
     fn new_tree_hash_cache(&self, _depth: usize) -> Result<TreeHashCache, Error> {
diff --git a/eth2/utils/cached_tree_hash/src/impls/vec.rs b/eth2/utils/cached_tree_hash/src/impls/vec.rs
index 8c58e022a..14802254b 100644
--- a/eth2/utils/cached_tree_hash/src/impls/vec.rs
+++ b/eth2/utils/cached_tree_hash/src/impls/vec.rs
@@ -6,23 +6,7 @@ where
     T: CachedTreeHash<T> + TreeHash,
 {
     fn new_tree_hash_cache(&self, depth: usize) -> Result<TreeHashCache, Error> {
-        let overlay = self.tree_hash_cache_overlay(0, depth)?;
-
-        let mut cache = match T::tree_hash_type() {
-            TreeHashType::Basic => TreeHashCache::from_bytes(
-                merkleize(get_packed_leaves(self)?),
-                false,
-                Some(overlay.clone()),
-            ),
-            TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
-                let subtrees = self
-                    .iter()
-                    .map(|item| TreeHashCache::new(item, depth + 1))
-                    .collect::<Result<Vec<TreeHashCache>, _>>()?;
-
-                TreeHashCache::from_leaves_and_subtrees(self, subtrees, depth)
-            }
-        }?;
+        let (mut cache, overlay) = new_tree_hash_cache(self, depth)?;
 
         cache.add_length_nodes(overlay.chunk_range(), self.len())?;
 
@@ -33,7 +17,7 @@ where
         BTreeOverlay::new(self, 0, 0)
             .and_then(|o| Ok(o.num_chunks()))
             .unwrap_or_else(|_| 1)
-            + 2
+            + 2 // Add two extra nodes to cater for the length.
     }
 
     fn tree_hash_cache_overlay(
@@ -41,139 +25,15 @@ where
         chunk_offset: usize,
         depth: usize,
     ) -> Result<BTreeOverlay, Error> {
-        let lengths = match T::tree_hash_type() {
-            TreeHashType::Basic => {
-                // Ceil division.
-                let num_leaves = (self.len() + T::tree_hash_packing_factor() - 1)
-                    / T::tree_hash_packing_factor();
-
-                // Disallow zero-length as an empty list still has one all-padding node.
-                vec![1; std::cmp::max(1, num_leaves)]
-            }
-            TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
-                let mut lengths = vec![];
-
-                for item in self {
-                    lengths.push(item.num_tree_hash_cache_chunks())
-                }
-
-                // Disallow zero-length as an empty list still has one all-padding node.
-                if lengths.is_empty() {
-                    lengths.push(1);
-                }
-
-                lengths
-            }
-        };
-
-        BTreeOverlay::from_lengths(chunk_offset, self.len(), depth, lengths)
+        produce_overlay(self, chunk_offset, depth)
     }
 
     fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
         // Skip the length-mixed-in root node.
         cache.chunk_index += 1;
 
-        let old_overlay = cache.get_overlay(cache.overlay_index, cache.chunk_index)?;
-        let new_overlay = BTreeOverlay::new(self, cache.chunk_index, old_overlay.depth)?;
-
-        cache.replace_overlay(cache.overlay_index, cache.chunk_index, new_overlay.clone())?;
-
-        cache.overlay_index += 1;
-
-        match T::tree_hash_type() {
-            TreeHashType::Basic => {
-                let mut buf = vec![0; HASHSIZE];
-                let item_bytes = HASHSIZE / T::tree_hash_packing_factor();
-
-                // Iterate through each of the leaf nodes.
-                for i in 0..new_overlay.num_leaf_nodes() {
-                    // Iterate through the number of items that may be packing into the leaf node.
-                    for j in 0..T::tree_hash_packing_factor() {
-                        // Create a mut slice that can be filled with either a serialized item or
-                        // padding.
-                        let buf_slice = &mut buf[j * item_bytes..(j + 1) * item_bytes];
-
-                        // Attempt to get the item for this portion of the chunk. If it exists,
-                        // update `buf` with it's serialized bytes. If it doesn't exist, update
-                        // `buf` with padding.
-                        match self.get(i * T::tree_hash_packing_factor() + j) {
-                            Some(item) => {
-                                buf_slice.copy_from_slice(&item.tree_hash_packed_encoding());
-                            }
-                            None => buf_slice.copy_from_slice(&vec![0; item_bytes]),
-                        }
-                    }
-
-                    // Update the chunk if the generated `buf` is not the same as the cache.
-                    let chunk = new_overlay.first_leaf_node() + i;
-                    cache.maybe_update_chunk(chunk, &buf)?;
-                }
-            }
-            TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
-                for i in 0..new_overlay.num_leaf_nodes() {
-                    // Adjust `i` so it is a leaf node for each of the overlays.
-                    let old_i = i + old_overlay.num_internal_nodes();
-                    let new_i = i + new_overlay.num_internal_nodes();
-
-                    match (
-                        old_overlay.get_leaf_node(old_i)?,
-                        new_overlay.get_leaf_node(new_i)?,
-                    ) {
-                        // The item existed in the previous list and exists in the current list.
-                        (Some(_old), Some(new)) => {
-                            cache.chunk_index = new.start;
-
-                            self[i].update_tree_hash_cache(cache)?;
-                        }
-                        // The item did not exist in the previous list but does exist in this list.
-                        //
-                        // Viz., the list has been lengthened.
-                        (None, Some(new)) => {
-                            let (bytes, mut bools, overlays) =
-                                TreeHashCache::new(&self[i], new_overlay.depth + 1)?
-                                    .into_components();
-
-                            // Record the number of overlays, this will be used later in the fn.
-                            let num_overlays = overlays.len();
-
-                            // Flag the root node of the new tree as dirty.
-                            bools[0] = true;
-
-                            cache.splice(new.start..new.start + 1, bytes, bools);
-                            cache
-                                .overlays
-                                .splice(cache.overlay_index..cache.overlay_index, overlays);
-
-                            cache.overlay_index += num_overlays;
-                        }
-                        // The item existed in the previous list but does not exist in this list.
-                        //
-                        // Viz., the list has been shortened.
-                        (Some(old), None) => {
-                            if new_overlay.num_items == 0 {
-                                // In this case, the list has been made empty and we should make
-                                // this node padding.
-                                cache.maybe_update_chunk(new_overlay.root(), &[0; HASHSIZE])?;
-                            } else {
-                                // In this case, there are some items in the new list and we should
-                                // splice out the entire tree of the removed node, replacing it
-                                // with a single padding node.
-                                cache.splice(old, vec![0; HASHSIZE], vec![true]);
-                            }
-                        }
-                        // The item didn't exist in the old list and doesn't exist in the new list,
-                        // nothing to do.
-                        (None, None) => {}
-                    }
-                }
-
-                // Clean out any excess overlays that may or may not be remaining if the list was
-                // shortened.
-                cache.remove_proceeding_child_overlays(cache.overlay_index, new_overlay.depth);
-            }
-        }
-
-        cache.update_internal_nodes(&new_overlay)?;
+        // Update the cache, returning the new overlay.
+        let new_overlay = update_tree_hash_cache(self, cache)?;
 
         // Mix in length
         cache.mix_in_length(new_overlay.chunk_range(), self.len())?;
@@ -185,6 +45,172 @@ where
     }
 }
 
+pub fn new_tree_hash_cache<T: CachedTreeHash<T>>(
+    vec: &Vec<T>,
+    depth: usize,
+) -> Result<(TreeHashCache, BTreeOverlay), Error> {
+    let overlay = vec.tree_hash_cache_overlay(0, depth)?;
+
+    let cache = match T::tree_hash_type() {
+        TreeHashType::Basic => TreeHashCache::from_bytes(
+            merkleize(get_packed_leaves(vec)?),
+            false,
+            Some(overlay.clone()),
+        ),
+        TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
+            let subtrees = vec
+                .iter()
+                .map(|item| TreeHashCache::new(item, depth + 1))
+                .collect::<Result<Vec<TreeHashCache>, _>>()?;
+
+            TreeHashCache::from_leaves_and_subtrees(vec, subtrees, depth)
+        }
+    }?;
+
+    Ok((cache, overlay))
+}
+
+pub fn produce_overlay<T: CachedTreeHash<T>>(
+    vec: &Vec<T>,
+    chunk_offset: usize,
+    depth: usize,
+) -> Result<BTreeOverlay, Error> {
+    let lengths = match T::tree_hash_type() {
+        TreeHashType::Basic => {
+            // Ceil division.
+            let num_leaves =
+                (vec.len() + T::tree_hash_packing_factor() - 1) / T::tree_hash_packing_factor();
+
+            // Disallow zero-length as an empty list still has one all-padding node.
+            vec![1; std::cmp::max(1, num_leaves)]
+        }
+        TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
+            let mut lengths = vec![];
+
+            for item in vec {
+                lengths.push(item.num_tree_hash_cache_chunks())
+            }
+
+            // Disallow zero-length as an empty list still has one all-padding node.
+            if lengths.is_empty() {
+                lengths.push(1);
+            }
+
+            lengths
+        }
+    };
+
+    BTreeOverlay::from_lengths(chunk_offset, vec.len(), depth, lengths)
+}
+
+pub fn update_tree_hash_cache<T: CachedTreeHash<T>>(
+    vec: &Vec<T>,
+    cache: &mut TreeHashCache,
+) -> Result<BTreeOverlay, Error> {
+    let old_overlay = cache.get_overlay(cache.overlay_index, cache.chunk_index)?;
+    let new_overlay = BTreeOverlay::new(vec, cache.chunk_index, old_overlay.depth)?;
+
+    cache.replace_overlay(cache.overlay_index, cache.chunk_index, new_overlay.clone())?;
+
+    cache.overlay_index += 1;
+
+    match T::tree_hash_type() {
+        TreeHashType::Basic => {
+            let mut buf = vec![0; HASHSIZE];
+            let item_bytes = HASHSIZE / T::tree_hash_packing_factor();
+
+            // Iterate through each of the leaf nodes.
+            for i in 0..new_overlay.num_leaf_nodes() {
+                // Iterate through the number of items that may be packing into the leaf node.
+                for j in 0..T::tree_hash_packing_factor() {
+                    // Create a mut slice that can be filled with either a serialized item or
+                    // padding.
+                    let buf_slice = &mut buf[j * item_bytes..(j + 1) * item_bytes];
+
+                    // Attempt to get the item for this portion of the chunk. If it exists,
+                    // update `buf` with it's serialized bytes. If it doesn't exist, update
+                    // `buf` with padding.
+                    match vec.get(i * T::tree_hash_packing_factor() + j) {
+                        Some(item) => {
+                            buf_slice.copy_from_slice(&item.tree_hash_packed_encoding());
+                        }
+                        None => buf_slice.copy_from_slice(&vec![0; item_bytes]),
+                    }
+                }
+
+                // Update the chunk if the generated `buf` is not the same as the cache.
+                let chunk = new_overlay.first_leaf_node() + i;
+                cache.maybe_update_chunk(chunk, &buf)?;
+            }
+        }
+        TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
+            for i in 0..new_overlay.num_leaf_nodes() {
+                // Adjust `i` so it is a leaf node for each of the overlays.
+                let old_i = i + old_overlay.num_internal_nodes();
+                let new_i = i + new_overlay.num_internal_nodes();
+
+                match (
+                    old_overlay.get_leaf_node(old_i)?,
+                    new_overlay.get_leaf_node(new_i)?,
+                ) {
+                    // The item existed in the previous list and exists in the current list.
+                    (Some(_old), Some(new)) => {
+                        cache.chunk_index = new.start;
+
+                        vec[i].update_tree_hash_cache(cache)?;
+                    }
+                    // The item did not exist in the previous list but does exist in this list.
+                    //
+                    // Viz., the list has been lengthened.
+                    (None, Some(new)) => {
+                        let (bytes, mut bools, overlays) =
+                            TreeHashCache::new(&vec[i], new_overlay.depth + 1)?.into_components();
+
+                        // Record the number of overlays, this will be used later in the fn.
+                        let num_overlays = overlays.len();
+
+                        // Flag the root node of the new tree as dirty.
+                        bools[0] = true;
+
+                        cache.splice(new.start..new.start + 1, bytes, bools);
+                        cache
+                            .overlays
+                            .splice(cache.overlay_index..cache.overlay_index, overlays);
+
+                        cache.overlay_index += num_overlays;
+                    }
+                    // The item existed in the previous list but does not exist in this list.
+                    //
+                    // Viz., the list has been shortened.
+                    (Some(old), None) => {
+                        if new_overlay.num_items == 0 {
+                            // In this case, the list has been made empty and we should make
+                            // this node padding.
+                            cache.maybe_update_chunk(new_overlay.root(), &[0; HASHSIZE])?;
+                        } else {
+                            // In this case, there are some items in the new list and we should
+                            // splice out the entire tree of the removed node, replacing it
+                            // with a single padding node.
+                            cache.splice(old, vec![0; HASHSIZE], vec![true]);
+                        }
+                    }
+                    // The item didn't exist in the old list and doesn't exist in the new list,
+                    // nothing to do.
+                    (None, None) => {}
+                }
+            }
+
+            // Clean out any excess overlays that may or may not be remaining if the list was
+            // shortened.
+            cache.remove_proceeding_child_overlays(cache.overlay_index, new_overlay.depth);
+        }
+    }
+
+    cache.update_internal_nodes(&new_overlay)?;
+
+    Ok(new_overlay)
+}
+
 fn get_packed_leaves<T>(vec: &Vec<T>) -> Result<Vec<u8>, Error>
 where
     T: CachedTreeHash<T>,
diff --git a/eth2/utils/cached_tree_hash/src/lib.rs b/eth2/utils/cached_tree_hash/src/lib.rs
index 539519611..d9f2b6a84 100644
--- a/eth2/utils/cached_tree_hash/src/lib.rs
+++ b/eth2/utils/cached_tree_hash/src/lib.rs
@@ -4,7 +4,7 @@ use tree_hash::{TreeHash, TreeHashType, BYTES_PER_CHUNK, HASHSIZE};
 
 mod btree_overlay;
 mod errors;
-mod impls;
+pub mod impls;
 pub mod merkleize;
 mod resize;
 mod tree_hash_cache;
diff --git a/eth2/utils/tree_hash/src/impls.rs b/eth2/utils/tree_hash/src/impls.rs
index a32e3844c..f2ff8c2bc 100644
--- a/eth2/utils/tree_hash/src/impls.rs
+++ b/eth2/utils/tree_hash/src/impls.rs
@@ -56,11 +56,11 @@ impl TreeHash for [u8; 4] {
     }
 
     fn tree_hash_packed_encoding(&self) -> Vec<u8> {
-        panic!("bytesN should never be packed.")
+        unreachable!("bytesN should never be packed.")
     }
 
     fn tree_hash_packing_factor() -> usize {
-        panic!("bytesN should never be packed.")
+        unreachable!("bytesN should never be packed.")
     }
 
     fn tree_hash_root(&self) -> Vec<u8> {
diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs
index 5e8bfdf8e..2554e70c3 100644
--- a/eth2/utils/tree_hash/src/lib.rs
+++ b/eth2/utils/tree_hash/src/lib.rs
@@ -49,6 +49,7 @@ macro_rules! tree_hash_ssz_encoding_as_vector {
         }
     };
 }
+
 #[macro_export]
 macro_rules! tree_hash_ssz_encoding_as_list {
     ($type: ident) => {

From 794b48078c12cc275505859bf80f3d1b9e55e600 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Fri, 26 Apr 2019 11:34:07 +1000
Subject: [PATCH 109/137] Remove `num_items` from `BTreeOverlay`

---
 .../cached_tree_hash/src/btree_overlay.rs     | 29 ++++++-------------
 eth2/utils/cached_tree_hash/src/impls.rs      | 18 +++---------
 eth2/utils/cached_tree_hash/src/impls/vec.rs  | 27 +++++------------
 eth2/utils/cached_tree_hash/src/lib.rs        |  6 +---
 .../cached_tree_hash/src/tree_hash_cache.rs   |  2 +-
 eth2/utils/tree_hash_derive/src/lib.rs        | 12 +++-----
 6 files changed, 27 insertions(+), 67 deletions(-)

diff --git a/eth2/utils/cached_tree_hash/src/btree_overlay.rs b/eth2/utils/cached_tree_hash/src/btree_overlay.rs
index 1a8fde3c1..866f5c6b7 100644
--- a/eth2/utils/cached_tree_hash/src/btree_overlay.rs
+++ b/eth2/utils/cached_tree_hash/src/btree_overlay.rs
@@ -4,33 +4,22 @@ use super::*;
 pub struct BTreeOverlay {
     pub offset: usize,
     pub depth: usize,
-    pub num_items: usize,
     pub lengths: Vec<usize>,
 }
 
 impl BTreeOverlay {
-    pub fn new<T>(item: &T, initial_offset: usize, depth: usize) -> Result<Self, Error>
+    pub fn new<T>(item: &T, initial_offset: usize, depth: usize) -> Self
     where
         T: CachedTreeHash<T>,
     {
         item.tree_hash_cache_overlay(initial_offset, depth)
     }
 
-    pub fn from_lengths(
-        offset: usize,
-        num_items: usize,
-        depth: usize,
-        lengths: Vec<usize>,
-    ) -> Result<Self, Error> {
-        if lengths.is_empty() {
-            Err(Error::TreeCannotHaveZeroNodes)
-        } else {
-            Ok(Self {
-                offset,
-                num_items,
-                depth,
-                lengths,
-            })
+    pub fn from_lengths(offset: usize, depth: usize, lengths: Vec<usize>) -> Self {
+        Self {
+            offset,
+            depth,
+            lengths,
         }
     }
 
@@ -96,7 +85,7 @@ impl BTreeOverlay {
     pub fn get_leaf_node(&self, i: usize) -> Result<Option<Range<usize>>, Error> {
         if i >= self.num_nodes() - self.num_padding_leaves() {
             Ok(None)
-        } else if (i == self.num_internal_nodes()) && (self.num_items == 0) {
+        } else if (i == self.num_internal_nodes()) && (self.lengths.len() == 0) {
             // If this is the first leaf node and the overlay contains zero items, return `None` as
             // this node must be padding.
             Ok(None)
@@ -177,7 +166,7 @@ mod test {
     use super::*;
 
     fn get_tree_a(n: usize) -> BTreeOverlay {
-        BTreeOverlay::from_lengths(0, n, 0, vec![1; n]).unwrap()
+        BTreeOverlay::from_lengths(0, 0, vec![1; n])
     }
 
     #[test]
@@ -215,7 +204,7 @@ mod test {
         let tree = get_tree_a(2);
         assert_eq!(tree.chunk_range(), 0..3);
 
-        let tree = BTreeOverlay::from_lengths(11, 4, 0, vec![1, 1]).unwrap();
+        let tree = BTreeOverlay::from_lengths(11, 0, vec![1, 1]);
         assert_eq!(tree.chunk_range(), 11..14);
     }
 
diff --git a/eth2/utils/cached_tree_hash/src/impls.rs b/eth2/utils/cached_tree_hash/src/impls.rs
index 827b0fbe9..666461962 100644
--- a/eth2/utils/cached_tree_hash/src/impls.rs
+++ b/eth2/utils/cached_tree_hash/src/impls.rs
@@ -16,13 +16,8 @@ impl CachedTreeHash<u64> for u64 {
         1
     }
 
-    fn tree_hash_cache_overlay(
-        &self,
-        _chunk_offset: usize,
-        _depth: usize,
-    ) -> Result<BTreeOverlay, Error> {
-        panic!("Basic should not produce overlay");
-        // BTreeOverlay::from_lengths(chunk_offset, 1, depth, vec![1])
+    fn tree_hash_cache_overlay(&self, _chunk_offset: usize, _depth: usize) -> BTreeOverlay {
+        unreachable!("Basic should not produce overlay");
     }
 
     fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
@@ -49,13 +44,8 @@ impl CachedTreeHash<usize> for usize {
         1
     }
 
-    fn tree_hash_cache_overlay(
-        &self,
-        _chunk_offset: usize,
-        _depth: usize,
-    ) -> Result<BTreeOverlay, Error> {
-        panic!("Basic should not produce overlay");
-        // BTreeOverlay::from_lengths(chunk_offset, 1, depth, vec![1])
+    fn tree_hash_cache_overlay(&self, _chunk_offset: usize, _depth: usize) -> BTreeOverlay {
+        unreachable!("Basic should not produce overlay");
     }
 
     fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
diff --git a/eth2/utils/cached_tree_hash/src/impls/vec.rs b/eth2/utils/cached_tree_hash/src/impls/vec.rs
index 14802254b..c37354404 100644
--- a/eth2/utils/cached_tree_hash/src/impls/vec.rs
+++ b/eth2/utils/cached_tree_hash/src/impls/vec.rs
@@ -14,17 +14,11 @@ where
     }
 
     fn num_tree_hash_cache_chunks(&self) -> usize {
-        BTreeOverlay::new(self, 0, 0)
-            .and_then(|o| Ok(o.num_chunks()))
-            .unwrap_or_else(|_| 1)
-            + 2 // Add two extra nodes to cater for the length.
+        // Add two extra nodes to cater for the node before and after to allow mixing-in length.
+        BTreeOverlay::new(self, 0, 0).num_chunks() + 2
     }
 
-    fn tree_hash_cache_overlay(
-        &self,
-        chunk_offset: usize,
-        depth: usize,
-    ) -> Result<BTreeOverlay, Error> {
+    fn tree_hash_cache_overlay(&self, chunk_offset: usize, depth: usize) -> BTreeOverlay {
         produce_overlay(self, chunk_offset, depth)
     }
 
@@ -49,7 +43,7 @@ pub fn new_tree_hash_cache<T: CachedTreeHash<T>>(
     vec: &Vec<T>,
     depth: usize,
 ) -> Result<(TreeHashCache, BTreeOverlay), Error> {
-    let overlay = vec.tree_hash_cache_overlay(0, depth)?;
+    let overlay = vec.tree_hash_cache_overlay(0, depth);
 
     let cache = match T::tree_hash_type() {
         TreeHashType::Basic => TreeHashCache::from_bytes(
@@ -74,7 +68,7 @@ pub fn produce_overlay<T: CachedTreeHash<T>>(
     vec: &Vec<T>,
     chunk_offset: usize,
     depth: usize,
-) -> Result<BTreeOverlay, Error> {
+) -> BTreeOverlay {
     let lengths = match T::tree_hash_type() {
         TreeHashType::Basic => {
             // Ceil division.
@@ -91,16 +85,11 @@ pub fn produce_overlay<T: CachedTreeHash<T>>(
                 lengths.push(item.num_tree_hash_cache_chunks())
             }
 
-            // Disallow zero-length as an empty list still has one all-padding node.
-            if lengths.is_empty() {
-                lengths.push(1);
-            }
-
             lengths
         }
     };
 
-    BTreeOverlay::from_lengths(chunk_offset, vec.len(), depth, lengths)
+    BTreeOverlay::from_lengths(chunk_offset, depth, lengths)
 }
 
 pub fn update_tree_hash_cache<T: CachedTreeHash<T>>(
@@ -108,7 +97,7 @@ pub fn update_tree_hash_cache<T: CachedTreeHash<T>>(
     cache: &mut TreeHashCache,
 ) -> Result<BTreeOverlay, Error> {
     let old_overlay = cache.get_overlay(cache.overlay_index, cache.chunk_index)?;
-    let new_overlay = BTreeOverlay::new(vec, cache.chunk_index, old_overlay.depth)?;
+    let new_overlay = BTreeOverlay::new(vec, cache.chunk_index, old_overlay.depth);
 
     cache.replace_overlay(cache.overlay_index, cache.chunk_index, new_overlay.clone())?;
 
@@ -183,7 +172,7 @@ pub fn update_tree_hash_cache<T: CachedTreeHash<T>>(
                     //
                     // Viz., the list has been shortened.
                     (Some(old), None) => {
-                        if new_overlay.num_items == 0 {
+                        if vec.len() == 0 {
                             // In this case, the list has been made empty and we should make
                             // this node padding.
                             cache.maybe_update_chunk(new_overlay.root(), &[0; HASHSIZE])?;
diff --git a/eth2/utils/cached_tree_hash/src/lib.rs b/eth2/utils/cached_tree_hash/src/lib.rs
index d9f2b6a84..2cf3c38bd 100644
--- a/eth2/utils/cached_tree_hash/src/lib.rs
+++ b/eth2/utils/cached_tree_hash/src/lib.rs
@@ -14,11 +14,7 @@ pub use errors::Error;
 pub use tree_hash_cache::TreeHashCache;
 
 pub trait CachedTreeHash<Item>: TreeHash {
-    fn tree_hash_cache_overlay(
-        &self,
-        chunk_offset: usize,
-        depth: usize,
-    ) -> Result<BTreeOverlay, Error>;
+    fn tree_hash_cache_overlay(&self, chunk_offset: usize, depth: usize) -> BTreeOverlay;
 
     fn num_tree_hash_cache_chunks(&self) -> usize;
 
diff --git a/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
index d93278d30..2a9bae5a3 100644
--- a/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
+++ b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
@@ -34,7 +34,7 @@ impl TreeHashCache {
     where
         T: CachedTreeHash<T>,
     {
-        let overlay = BTreeOverlay::new(item, 0, depth)?;
+        let overlay = BTreeOverlay::new(item, 0, depth);
 
         // Note how many leaves were provided. If is not a power-of-two, we'll need to pad it out
         // later.
diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs
index b6ba10e02..104cb8b32 100644
--- a/eth2/utils/tree_hash_derive/src/lib.rs
+++ b/eth2/utils/tree_hash_derive/src/lib.rs
@@ -55,8 +55,6 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
     let idents_b = idents_a.clone();
     let idents_c = idents_a.clone();
 
-    let num_items = idents_a.len();
-
     let output = quote! {
         impl cached_tree_hash::CachedTreeHash<#name> for #name {
             fn new_tree_hash_cache(&self, depth: usize) -> Result<cached_tree_hash::TreeHashCache, cached_tree_hash::Error> {
@@ -74,23 +72,21 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
             }
 
             fn num_tree_hash_cache_chunks(&self) -> usize {
-                cached_tree_hash::BTreeOverlay::new(self, 0, 0)
-                    .and_then(|o| Ok(o.num_chunks()))
-                    .unwrap_or_else(|_| 1)
+                cached_tree_hash::BTreeOverlay::new(self, 0, 0).num_chunks()
             }
 
-            fn tree_hash_cache_overlay(&self, chunk_offset: usize, depth: usize) -> Result<cached_tree_hash::BTreeOverlay, cached_tree_hash::Error> {
+            fn tree_hash_cache_overlay(&self, chunk_offset: usize, depth: usize) ->cached_tree_hash::BTreeOverlay {
                 let mut lengths = vec![];
 
                 #(
                     lengths.push(self.#idents_b.num_tree_hash_cache_chunks());
                 )*
 
-                cached_tree_hash::BTreeOverlay::from_lengths(chunk_offset, #num_items, depth, lengths)
+                cached_tree_hash::BTreeOverlay::from_lengths(chunk_offset, depth, lengths)
             }
 
             fn update_tree_hash_cache(&self, cache: &mut cached_tree_hash::TreeHashCache) -> Result<(), cached_tree_hash::Error> {
-                let overlay = cached_tree_hash::BTreeOverlay::new(self, cache.chunk_index, 0)?;
+                let overlay = cached_tree_hash::BTreeOverlay::new(self, cache.chunk_index, 0);
 
                 // Skip the chunk index to the first leaf node of this struct.
                 cache.chunk_index = overlay.first_leaf_node();

From 8976e652d29b552e8093c550c046dd15910945d2 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Fri, 26 Apr 2019 11:55:26 +1000
Subject: [PATCH 110/137] Introduce `BTreeSchema`

---
 eth2/utils/cached_tree_hash/src/errors.rs     |  2 +-
 eth2/utils/cached_tree_hash/src/impls/vec.rs  | 22 ++---
 eth2/utils/cached_tree_hash/src/lib.rs        |  2 +-
 .../cached_tree_hash/src/tree_hash_cache.rs   | 93 ++++++++++++-------
 4 files changed, 70 insertions(+), 49 deletions(-)

diff --git a/eth2/utils/cached_tree_hash/src/errors.rs b/eth2/utils/cached_tree_hash/src/errors.rs
index 9045d0409..cd387fa47 100644
--- a/eth2/utils/cached_tree_hash/src/errors.rs
+++ b/eth2/utils/cached_tree_hash/src/errors.rs
@@ -13,6 +13,6 @@ pub enum Error {
     BytesAreNotEvenChunks(usize),
     NoModifiedFieldForChunk(usize),
     NoBytesForChunk(usize),
-    NoOverlayForIndex(usize),
+    NoSchemaForIndex(usize),
     NotLeafNode(usize),
 }
diff --git a/eth2/utils/cached_tree_hash/src/impls/vec.rs b/eth2/utils/cached_tree_hash/src/impls/vec.rs
index c37354404..4d50a7b46 100644
--- a/eth2/utils/cached_tree_hash/src/impls/vec.rs
+++ b/eth2/utils/cached_tree_hash/src/impls/vec.rs
@@ -96,12 +96,12 @@ pub fn update_tree_hash_cache<T: CachedTreeHash<T>>(
     vec: &Vec<T>,
     cache: &mut TreeHashCache,
 ) -> Result<BTreeOverlay, Error> {
-    let old_overlay = cache.get_overlay(cache.overlay_index, cache.chunk_index)?;
+    let old_overlay = cache.get_overlay(cache.schema_index, cache.chunk_index)?;
     let new_overlay = BTreeOverlay::new(vec, cache.chunk_index, old_overlay.depth);
 
-    cache.replace_overlay(cache.overlay_index, cache.chunk_index, new_overlay.clone())?;
+    cache.replace_overlay(cache.schema_index, cache.chunk_index, new_overlay.clone())?;
 
-    cache.overlay_index += 1;
+    cache.schema_index += 1;
 
     match T::tree_hash_type() {
         TreeHashType::Basic => {
@@ -152,21 +152,21 @@ pub fn update_tree_hash_cache<T: CachedTreeHash<T>>(
                     //
                     // Viz., the list has been lengthened.
                     (None, Some(new)) => {
-                        let (bytes, mut bools, overlays) =
+                        let (bytes, mut bools, schemas) =
                             TreeHashCache::new(&vec[i], new_overlay.depth + 1)?.into_components();
 
-                        // Record the number of overlays, this will be used later in the fn.
-                        let num_overlays = overlays.len();
+                        // Record the number of schemas, this will be used later in the fn.
+                        let num_schemas = schemas.len();
 
                         // Flag the root node of the new tree as dirty.
                         bools[0] = true;
 
                         cache.splice(new.start..new.start + 1, bytes, bools);
                         cache
-                            .overlays
-                            .splice(cache.overlay_index..cache.overlay_index, overlays);
+                            .schemas
+                            .splice(cache.schema_index..cache.schema_index, schemas);
 
-                        cache.overlay_index += num_overlays;
+                        cache.schema_index += num_schemas;
                     }
                     // The item existed in the previous list but does not exist in this list.
                     //
@@ -189,9 +189,9 @@ pub fn update_tree_hash_cache<T: CachedTreeHash<T>>(
                 }
             }
 
-            // Clean out any excess overlays that may or may not be remaining if the list was
+            // Clean out any excess schemas that may or may not be remaining if the list was
             // shortened.
-            cache.remove_proceeding_child_overlays(cache.overlay_index, new_overlay.depth);
+            cache.remove_proceeding_child_schemas(cache.schema_index, new_overlay.depth);
         }
     }
 
diff --git a/eth2/utils/cached_tree_hash/src/lib.rs b/eth2/utils/cached_tree_hash/src/lib.rs
index 2cf3c38bd..38142aaa8 100644
--- a/eth2/utils/cached_tree_hash/src/lib.rs
+++ b/eth2/utils/cached_tree_hash/src/lib.rs
@@ -44,7 +44,7 @@ impl CachedTreeHasher {
     {
         // Reset the per-hash counters.
         self.cache.chunk_index = 0;
-        self.cache.overlay_index = 0;
+        self.cache.schema_index = 0;
 
         // Reset the "modified" flags for the cache.
         self.cache.reset_modifications();
diff --git a/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
index 2a9bae5a3..225286079 100644
--- a/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
+++ b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
@@ -2,14 +2,39 @@ use super::*;
 use crate::merkleize::{merkleize, pad_for_leaf_count};
 use int_to_bytes::int_to_bytes32;
 
+#[derive(Debug, PartialEq, Clone)]
+pub struct BTreeSchema {
+    pub depth: usize,
+    pub lengths: Vec<usize>,
+}
+
+impl BTreeSchema {
+    pub fn into_overlay(self, offset: usize) -> BTreeOverlay {
+        BTreeOverlay {
+            offset,
+            depth: self.depth,
+            lengths: self.lengths,
+        }
+    }
+}
+
+impl Into<BTreeSchema> for BTreeOverlay {
+    fn into(self) -> BTreeSchema {
+        BTreeSchema {
+            depth: self.depth,
+            lengths: self.lengths,
+        }
+    }
+}
+
 #[derive(Debug, PartialEq, Clone)]
 pub struct TreeHashCache {
     pub cache: Vec<u8>,
     pub chunk_modified: Vec<bool>,
-    pub overlays: Vec<BTreeOverlay>,
+    pub schemas: Vec<BTreeSchema>,
 
     pub chunk_index: usize,
-    pub overlay_index: usize,
+    pub schema_index: usize,
 }
 
 impl Into<Vec<u8>> for TreeHashCache {
@@ -51,10 +76,10 @@ impl TreeHashCache {
 
         // Allocate enough bytes to store all the leaves.
         let mut leaves = Vec::with_capacity(overlay.num_leaf_nodes() * HASHSIZE);
-        let mut overlays = Vec::with_capacity(leaves_and_subtrees.len());
+        let mut schemas = Vec::with_capacity(leaves_and_subtrees.len());
 
         if T::tree_hash_type() == TreeHashType::List {
-            overlays.push(overlay);
+            schemas.push(overlay.into());
         }
 
         // Iterate through all of the leaves/subtrees, adding their root as a leaf node and then
@@ -62,9 +87,9 @@ impl TreeHashCache {
         for t in leaves_and_subtrees {
             leaves.append(&mut t.root()?.to_vec());
 
-            let (mut bytes, _bools, mut t_overlays) = t.into_components();
+            let (mut bytes, _bools, mut t_schemas) = t.into_components();
             cache.append(&mut bytes);
-            overlays.append(&mut t_overlays);
+            schemas.append(&mut t_schemas);
         }
 
         // Pad the leaves to an even power-of-two, using zeros.
@@ -79,9 +104,9 @@ impl TreeHashCache {
         Ok(Self {
             chunk_modified: vec![false; cache.len() / BYTES_PER_CHUNK],
             cache,
-            overlays,
+            schemas,
             chunk_index: 0,
-            overlay_index: 0,
+            schema_index: 0,
         })
     }
 
@@ -94,34 +119,31 @@ impl TreeHashCache {
             return Err(Error::BytesAreNotEvenChunks(bytes.len()));
         }
 
-        let overlays = match overlay {
-            Some(overlay) => vec![overlay],
+        let schemas = match overlay {
+            Some(overlay) => vec![overlay.into()],
             None => vec![],
         };
 
         Ok(Self {
             chunk_modified: vec![initial_modified_state; bytes.len() / BYTES_PER_CHUNK],
             cache: bytes,
-            overlays,
+            schemas,
             chunk_index: 0,
-            overlay_index: 0,
+            schema_index: 0,
         })
     }
 
     pub fn get_overlay(
         &self,
-        overlay_index: usize,
+        schema_index: usize,
         chunk_index: usize,
     ) -> Result<BTreeOverlay, Error> {
-        let mut overlay = self
-            .overlays
-            .get(overlay_index)
-            .ok_or_else(|| Error::NoOverlayForIndex(overlay_index))?
-            .clone();
-
-        overlay.offset = chunk_index;
-
-        Ok(overlay)
+        Ok(self
+            .schemas
+            .get(schema_index)
+            .ok_or_else(|| Error::NoSchemaForIndex(schema_index))?
+            .clone()
+            .into_overlay(chunk_index))
     }
 
     pub fn reset_modifications(&mut self) {
@@ -132,11 +154,11 @@ impl TreeHashCache {
 
     pub fn replace_overlay(
         &mut self,
-        overlay_index: usize,
+        schema_index: usize,
         chunk_index: usize,
         new_overlay: BTreeOverlay,
     ) -> Result<BTreeOverlay, Error> {
-        let old_overlay = self.get_overlay(overlay_index, chunk_index)?;
+        let old_overlay = self.get_overlay(schema_index, chunk_index)?;
 
         // If the merkle tree required to represent the new list is of a different size to the one
         // required for the previous list, then update our cache.
@@ -173,22 +195,21 @@ impl TreeHashCache {
             self.splice(old_overlay.chunk_range(), new_bytes, new_bools);
         }
 
-        Ok(std::mem::replace(
-            &mut self.overlays[overlay_index],
-            new_overlay,
-        ))
+        let old_schema = std::mem::replace(&mut self.schemas[schema_index], new_overlay.into());
+
+        Ok(old_schema.into_overlay(chunk_index))
     }
 
-    pub fn remove_proceeding_child_overlays(&mut self, overlay_index: usize, depth: usize) {
+    pub fn remove_proceeding_child_schemas(&mut self, schema_index: usize, depth: usize) {
         let end = self
-            .overlays
+            .schemas
             .iter()
-            .skip(overlay_index)
+            .skip(schema_index)
             .position(|o| o.depth <= depth)
-            .and_then(|i| Some(i + overlay_index))
-            .unwrap_or_else(|| self.overlays.len());
+            .and_then(|i| Some(i + schema_index))
+            .unwrap_or_else(|| self.schemas.len());
 
-        self.overlays.splice(overlay_index..end, vec![]);
+        self.schemas.splice(schema_index..end, vec![]);
     }
 
     pub fn update_internal_nodes(&mut self, overlay: &BTreeOverlay) -> Result<(), Error> {
@@ -326,8 +347,8 @@ impl TreeHashCache {
         Ok(())
     }
 
-    pub fn into_components(self) -> (Vec<u8>, Vec<bool>, Vec<BTreeOverlay>) {
-        (self.cache, self.chunk_modified, self.overlays)
+    pub fn into_components(self) -> (Vec<u8>, Vec<bool>, Vec<BTreeSchema>) {
+        (self.cache, self.chunk_modified, self.schemas)
     }
 }
 

From ecff8f0007650a9e57b8be83e0c390892822fd49 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Fri, 26 Apr 2019 12:27:04 +1000
Subject: [PATCH 111/137] Update `CachedTreeHash` trait to return schema

---
 eth2/types/src/tree_hash_vector.rs            | 14 +-----
 .../cached_tree_hash/src/btree_overlay.rs     | 50 +++++++++++++------
 eth2/utils/cached_tree_hash/src/impls.rs      | 16 ++----
 eth2/utils/cached_tree_hash/src/impls/vec.rs  | 24 ++++-----
 eth2/utils/cached_tree_hash/src/lib.rs        |  8 +--
 .../cached_tree_hash/src/tree_hash_cache.rs   | 31 ++----------
 eth2/utils/tree_hash_derive/src/lib.rs        |  4 +-
 7 files changed, 62 insertions(+), 85 deletions(-)

diff --git a/eth2/types/src/tree_hash_vector.rs b/eth2/types/src/tree_hash_vector.rs
index a3e63c405..4b6ebad37 100644
--- a/eth2/types/src/tree_hash_vector.rs
+++ b/eth2/types/src/tree_hash_vector.rs
@@ -68,18 +68,8 @@ where
         Ok(cache)
     }
 
-    fn num_tree_hash_cache_chunks(&self) -> usize {
-        cached_tree_hash::BTreeOverlay::new(self, 0, 0)
-            .and_then(|o| Ok(o.num_chunks()))
-            .unwrap_or_else(|_| 1)
-    }
-
-    fn tree_hash_cache_overlay(
-        &self,
-        chunk_offset: usize,
-        depth: usize,
-    ) -> Result<cached_tree_hash::BTreeOverlay, cached_tree_hash::Error> {
-        cached_tree_hash::impls::vec::produce_overlay(self, chunk_offset, depth)
+    fn tree_hash_cache_schema(&self, depth: usize) -> cached_tree_hash::BTreeSchema {
+        cached_tree_hash::impls::vec::produce_schema(self, depth)
     }
 
     fn update_tree_hash_cache(
diff --git a/eth2/utils/cached_tree_hash/src/btree_overlay.rs b/eth2/utils/cached_tree_hash/src/btree_overlay.rs
index 866f5c6b7..84efdb79b 100644
--- a/eth2/utils/cached_tree_hash/src/btree_overlay.rs
+++ b/eth2/utils/cached_tree_hash/src/btree_overlay.rs
@@ -1,10 +1,39 @@
 use super::*;
 
 #[derive(Debug, PartialEq, Clone)]
-pub struct BTreeOverlay {
-    pub offset: usize,
+pub struct BTreeSchema {
     pub depth: usize,
-    pub lengths: Vec<usize>,
+    lengths: Vec<usize>,
+}
+
+impl BTreeSchema {
+    pub fn from_lengths(depth: usize, lengths: Vec<usize>) -> Self {
+        Self { depth, lengths }
+    }
+
+    pub fn into_overlay(self, offset: usize) -> BTreeOverlay {
+        BTreeOverlay {
+            offset,
+            depth: self.depth,
+            lengths: self.lengths,
+        }
+    }
+}
+
+impl Into<BTreeSchema> for BTreeOverlay {
+    fn into(self) -> BTreeSchema {
+        BTreeSchema {
+            depth: self.depth,
+            lengths: self.lengths,
+        }
+    }
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct BTreeOverlay {
+    offset: usize,
+    pub depth: usize,
+    lengths: Vec<usize>,
 }
 
 impl BTreeOverlay {
@@ -12,15 +41,8 @@ impl BTreeOverlay {
     where
         T: CachedTreeHash<T>,
     {
-        item.tree_hash_cache_overlay(initial_offset, depth)
-    }
-
-    pub fn from_lengths(offset: usize, depth: usize, lengths: Vec<usize>) -> Self {
-        Self {
-            offset,
-            depth,
-            lengths,
-        }
+        item.tree_hash_cache_schema(depth)
+            .into_overlay(initial_offset)
     }
 
     pub fn num_leaf_nodes(&self) -> usize {
@@ -166,7 +188,7 @@ mod test {
     use super::*;
 
     fn get_tree_a(n: usize) -> BTreeOverlay {
-        BTreeOverlay::from_lengths(0, 0, vec![1; n])
+        BTreeSchema::from_lengths(0, vec![1; n]).into_overlay(0)
     }
 
     #[test]
@@ -204,7 +226,7 @@ mod test {
         let tree = get_tree_a(2);
         assert_eq!(tree.chunk_range(), 0..3);
 
-        let tree = BTreeOverlay::from_lengths(11, 0, vec![1, 1]);
+        let tree = BTreeSchema::from_lengths(0, vec![1, 1]).into_overlay(11);
         assert_eq!(tree.chunk_range(), 11..14);
     }
 
diff --git a/eth2/utils/cached_tree_hash/src/impls.rs b/eth2/utils/cached_tree_hash/src/impls.rs
index 666461962..687e74918 100644
--- a/eth2/utils/cached_tree_hash/src/impls.rs
+++ b/eth2/utils/cached_tree_hash/src/impls.rs
@@ -12,12 +12,8 @@ impl CachedTreeHash<u64> for u64 {
         )?)
     }
 
-    fn num_tree_hash_cache_chunks(&self) -> usize {
-        1
-    }
-
-    fn tree_hash_cache_overlay(&self, _chunk_offset: usize, _depth: usize) -> BTreeOverlay {
-        unreachable!("Basic should not produce overlay");
+    fn tree_hash_cache_schema(&self, depth: usize) -> BTreeSchema {
+        BTreeSchema::from_lengths(depth, vec![1])
     }
 
     fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
@@ -40,12 +36,8 @@ impl CachedTreeHash<usize> for usize {
         )?)
     }
 
-    fn num_tree_hash_cache_chunks(&self) -> usize {
-        1
-    }
-
-    fn tree_hash_cache_overlay(&self, _chunk_offset: usize, _depth: usize) -> BTreeOverlay {
-        unreachable!("Basic should not produce overlay");
+    fn tree_hash_cache_schema(&self, depth: usize) -> BTreeSchema {
+        BTreeSchema::from_lengths(depth, vec![1])
     }
 
     fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
diff --git a/eth2/utils/cached_tree_hash/src/impls/vec.rs b/eth2/utils/cached_tree_hash/src/impls/vec.rs
index 4d50a7b46..f8ed32399 100644
--- a/eth2/utils/cached_tree_hash/src/impls/vec.rs
+++ b/eth2/utils/cached_tree_hash/src/impls/vec.rs
@@ -6,9 +6,9 @@ where
     T: CachedTreeHash<T> + TreeHash,
 {
     fn new_tree_hash_cache(&self, depth: usize) -> Result<TreeHashCache, Error> {
-        let (mut cache, overlay) = new_tree_hash_cache(self, depth)?;
+        let (mut cache, schema) = new_tree_hash_cache(self, depth)?;
 
-        cache.add_length_nodes(overlay.chunk_range(), self.len())?;
+        cache.add_length_nodes(schema.into_overlay(0).chunk_range(), self.len())?;
 
         Ok(cache)
     }
@@ -18,8 +18,8 @@ where
         BTreeOverlay::new(self, 0, 0).num_chunks() + 2
     }
 
-    fn tree_hash_cache_overlay(&self, chunk_offset: usize, depth: usize) -> BTreeOverlay {
-        produce_overlay(self, chunk_offset, depth)
+    fn tree_hash_cache_schema(&self, depth: usize) -> BTreeSchema {
+        produce_schema(self, depth)
     }
 
     fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
@@ -42,14 +42,14 @@ where
 pub fn new_tree_hash_cache<T: CachedTreeHash<T>>(
     vec: &Vec<T>,
     depth: usize,
-) -> Result<(TreeHashCache, BTreeOverlay), Error> {
-    let overlay = vec.tree_hash_cache_overlay(0, depth);
+) -> Result<(TreeHashCache, BTreeSchema), Error> {
+    let schema = vec.tree_hash_cache_schema(depth);
 
     let cache = match T::tree_hash_type() {
         TreeHashType::Basic => TreeHashCache::from_bytes(
             merkleize(get_packed_leaves(vec)?),
             false,
-            Some(overlay.clone()),
+            Some(schema.clone()),
         ),
         TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
             let subtrees = vec
@@ -61,14 +61,10 @@ pub fn new_tree_hash_cache<T: CachedTreeHash<T>>(
         }
     }?;
 
-    Ok((cache, overlay))
+    Ok((cache, schema))
 }
 
-pub fn produce_overlay<T: CachedTreeHash<T>>(
-    vec: &Vec<T>,
-    chunk_offset: usize,
-    depth: usize,
-) -> BTreeOverlay {
+pub fn produce_schema<T: CachedTreeHash<T>>(vec: &Vec<T>, depth: usize) -> BTreeSchema {
     let lengths = match T::tree_hash_type() {
         TreeHashType::Basic => {
             // Ceil division.
@@ -89,7 +85,7 @@ pub fn produce_overlay<T: CachedTreeHash<T>>(
         }
     };
 
-    BTreeOverlay::from_lengths(chunk_offset, depth, lengths)
+    BTreeSchema::from_lengths(depth, lengths)
 }
 
 pub fn update_tree_hash_cache<T: CachedTreeHash<T>>(
diff --git a/eth2/utils/cached_tree_hash/src/lib.rs b/eth2/utils/cached_tree_hash/src/lib.rs
index 38142aaa8..316d52d07 100644
--- a/eth2/utils/cached_tree_hash/src/lib.rs
+++ b/eth2/utils/cached_tree_hash/src/lib.rs
@@ -9,14 +9,16 @@ pub mod merkleize;
 mod resize;
 mod tree_hash_cache;
 
-pub use btree_overlay::BTreeOverlay;
+pub use btree_overlay::{BTreeOverlay, BTreeSchema};
 pub use errors::Error;
 pub use tree_hash_cache::TreeHashCache;
 
 pub trait CachedTreeHash<Item>: TreeHash {
-    fn tree_hash_cache_overlay(&self, chunk_offset: usize, depth: usize) -> BTreeOverlay;
+    fn tree_hash_cache_schema(&self, depth: usize) -> BTreeSchema;
 
-    fn num_tree_hash_cache_chunks(&self) -> usize;
+    fn num_tree_hash_cache_chunks(&self) -> usize {
+        self.tree_hash_cache_schema(0).into_overlay(0).num_chunks()
+    }
 
     fn new_tree_hash_cache(&self, depth: usize) -> Result<TreeHashCache, Error>;
 
diff --git a/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
index 225286079..1fdd1fa5c 100644
--- a/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
+++ b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
@@ -2,31 +2,6 @@ use super::*;
 use crate::merkleize::{merkleize, pad_for_leaf_count};
 use int_to_bytes::int_to_bytes32;
 
-#[derive(Debug, PartialEq, Clone)]
-pub struct BTreeSchema {
-    pub depth: usize,
-    pub lengths: Vec<usize>,
-}
-
-impl BTreeSchema {
-    pub fn into_overlay(self, offset: usize) -> BTreeOverlay {
-        BTreeOverlay {
-            offset,
-            depth: self.depth,
-            lengths: self.lengths,
-        }
-    }
-}
-
-impl Into<BTreeSchema> for BTreeOverlay {
-    fn into(self) -> BTreeSchema {
-        BTreeSchema {
-            depth: self.depth,
-            lengths: self.lengths,
-        }
-    }
-}
-
 #[derive(Debug, PartialEq, Clone)]
 pub struct TreeHashCache {
     pub cache: Vec<u8>,
@@ -113,14 +88,14 @@ impl TreeHashCache {
     pub fn from_bytes(
         bytes: Vec<u8>,
         initial_modified_state: bool,
-        overlay: Option<BTreeOverlay>,
+        schema: Option<BTreeSchema>,
     ) -> Result<Self, Error> {
         if bytes.len() % BYTES_PER_CHUNK > 0 {
             return Err(Error::BytesAreNotEvenChunks(bytes.len()));
         }
 
-        let schemas = match overlay {
-            Some(overlay) => vec![overlay.into()],
+        let schemas = match schema {
+            Some(schema) => vec![schema],
             None => vec![],
         };
 
diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs
index 104cb8b32..910417b81 100644
--- a/eth2/utils/tree_hash_derive/src/lib.rs
+++ b/eth2/utils/tree_hash_derive/src/lib.rs
@@ -75,14 +75,14 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
                 cached_tree_hash::BTreeOverlay::new(self, 0, 0).num_chunks()
             }
 
-            fn tree_hash_cache_overlay(&self, chunk_offset: usize, depth: usize) ->cached_tree_hash::BTreeOverlay {
+            fn tree_hash_cache_schema(&self, depth: usize) -> cached_tree_hash::BTreeSchema {
                 let mut lengths = vec![];
 
                 #(
                     lengths.push(self.#idents_b.num_tree_hash_cache_chunks());
                 )*
 
-                cached_tree_hash::BTreeOverlay::from_lengths(chunk_offset, depth, lengths)
+                cached_tree_hash::BTreeSchema::from_lengths(depth, lengths)
             }
 
             fn update_tree_hash_cache(&self, cache: &mut cached_tree_hash::TreeHashCache) -> Result<(), cached_tree_hash::Error> {

From f65e981f6f128655071ea1f448970a6ccef15880 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Fri, 26 Apr 2019 15:24:18 +1000
Subject: [PATCH 112/137] Begin implementing cached hashing in types

---
 eth2/types/src/attestation.rs                 |  4 +-
 eth2/types/src/attestation_data.rs            |  4 +-
 .../src/attestation_data_and_custody_bit.rs   |  5 +-
 eth2/types/src/attester_slashing.rs           | 16 +++-
 eth2/types/src/beacon_block.rs                |  4 +-
 eth2/types/src/beacon_block_body.rs           | 16 +++-
 eth2/types/src/beacon_block_header.rs         |  4 +-
 eth2/types/src/beacon_state.rs                | 15 +++-
 eth2/types/src/beacon_state/tests.rs          |  1 +
 eth2/types/src/crosslink.rs                   |  4 +-
 eth2/types/src/crosslink_committee.rs         | 15 +++-
 eth2/types/src/deposit.rs                     | 16 +++-
 eth2/types/src/deposit_data.rs                | 16 +++-
 eth2/types/src/deposit_input.rs               |  4 +-
 eth2/types/src/eth1_data.rs                   | 15 +++-
 eth2/types/src/eth1_data_vote.rs              | 15 +++-
 eth2/types/src/fork.rs                        | 15 +++-
 eth2/types/src/historical_batch.rs            | 16 +++-
 eth2/types/src/pending_attestation.rs         | 16 +++-
 eth2/types/src/proposer_slashing.rs           | 16 +++-
 eth2/types/src/slashable_attestation.rs       |  4 +-
 eth2/types/src/slot_epoch_macros.rs           | 21 +++++
 eth2/types/src/test_utils/macros.rs           | 26 ++++++
 eth2/types/src/transfer.rs                    |  4 +-
 eth2/types/src/tree_hash_vector.rs            | 22 +++++
 eth2/types/src/validator.rs                   | 16 +++-
 eth2/types/src/voluntary_exit.rs              |  4 +-
 eth2/utils/bls/Cargo.toml                     |  1 +
 eth2/utils/bls/src/aggregate_signature.rs     |  2 +
 .../utils/bls/src/fake_aggregate_signature.rs |  2 +
 eth2/utils/bls/src/fake_signature.rs          |  2 +
 eth2/utils/bls/src/public_key.rs              | 20 +++++
 eth2/utils/bls/src/signature.rs               | 19 ++++
 eth2/utils/boolean-bitfield/Cargo.toml        |  1 +
 eth2/utils/boolean-bitfield/src/lib.rs        | 21 +++++
 eth2/utils/cached_tree_hash/src/impls.rs      | 71 +++++++++++++--
 eth2/utils/cached_tree_hash/src/impls/vec.rs  |  5 ++
 eth2/utils/cached_tree_hash/src/lib.rs        | 90 +++++++++++++++++++
 eth2/utils/cached_tree_hash/src/merkleize.rs  |  2 +-
 eth2/utils/cached_tree_hash/tests/tests.rs    | 85 ++++++++++++++++++
 eth2/utils/tree_hash/src/impls.rs             |  2 +-
 41 files changed, 590 insertions(+), 47 deletions(-)

diff --git a/eth2/types/src/attestation.rs b/eth2/types/src/attestation.rs
index f7bfdaab9..d1511763d 100644
--- a/eth2/types/src/attestation.rs
+++ b/eth2/types/src/attestation.rs
@@ -5,7 +5,7 @@ use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
 use tree_hash::TreeHash;
-use tree_hash_derive::{SignedRoot, TreeHash};
+use tree_hash_derive::{CachedTreeHash, SignedRoot, TreeHash};
 
 /// Details an attestation that can be slashable.
 ///
@@ -19,6 +19,7 @@ use tree_hash_derive::{SignedRoot, TreeHash};
     Encode,
     Decode,
     TreeHash,
+    CachedTreeHash,
     TestRandom,
     SignedRoot,
 )]
@@ -58,4 +59,5 @@ mod tests {
     use super::*;
 
     ssz_tests!(Attestation);
+    cached_tree_hash_tests!(Attestation);
 }
diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data.rs
index f8a0ecd15..c963d6001 100644
--- a/eth2/types/src/attestation_data.rs
+++ b/eth2/types/src/attestation_data.rs
@@ -5,7 +5,7 @@ use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
 use tree_hash::TreeHash;
-use tree_hash_derive::{SignedRoot, TreeHash};
+use tree_hash_derive::{CachedTreeHash, SignedRoot, TreeHash};
 
 /// The data upon which an attestation is based.
 ///
@@ -21,6 +21,7 @@ use tree_hash_derive::{SignedRoot, TreeHash};
     Encode,
     Decode,
     TreeHash,
+    CachedTreeHash,
     TestRandom,
     SignedRoot,
 )]
@@ -47,4 +48,5 @@ mod tests {
     use super::*;
 
     ssz_tests!(AttestationData);
+    cached_tree_hash_tests!(AttestationData);
 }
diff --git a/eth2/types/src/attestation_data_and_custody_bit.rs b/eth2/types/src/attestation_data_and_custody_bit.rs
index e5dc920dc..85a5875ab 100644
--- a/eth2/types/src/attestation_data_and_custody_bit.rs
+++ b/eth2/types/src/attestation_data_and_custody_bit.rs
@@ -3,12 +3,12 @@ use crate::test_utils::TestRandom;
 use rand::RngCore;
 use serde_derive::Serialize;
 use ssz_derive::{Decode, Encode};
-use tree_hash_derive::TreeHash;
+use tree_hash_derive::{CachedTreeHash, TreeHash};
 
 /// Used for pairing an attestation with a proof-of-custody.
 ///
 /// Spec v0.5.1
-#[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode, TreeHash)]
+#[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode, TreeHash, CachedTreeHash)]
 pub struct AttestationDataAndCustodyBit {
     pub data: AttestationData,
     pub custody_bit: bool,
@@ -28,4 +28,5 @@ mod test {
     use super::*;
 
     ssz_tests!(AttestationDataAndCustodyBit);
+    cached_tree_hash_tests!(AttestationDataAndCustodyBit);
 }
diff --git a/eth2/types/src/attester_slashing.rs b/eth2/types/src/attester_slashing.rs
index b5e851dbd..d4848b01c 100644
--- a/eth2/types/src/attester_slashing.rs
+++ b/eth2/types/src/attester_slashing.rs
@@ -3,12 +3,23 @@ use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
-use tree_hash_derive::TreeHash;
+use tree_hash_derive::{CachedTreeHash, TreeHash};
 
 /// Two conflicting attestations.
 ///
 /// Spec v0.5.1
-#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(
+    Debug,
+    PartialEq,
+    Clone,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    CachedTreeHash,
+    TestRandom,
+)]
 pub struct AttesterSlashing {
     pub slashable_attestation_1: SlashableAttestation,
     pub slashable_attestation_2: SlashableAttestation,
@@ -19,4 +30,5 @@ mod tests {
     use super::*;
 
     ssz_tests!(AttesterSlashing);
+    cached_tree_hash_tests!(AttesterSlashing);
 }
diff --git a/eth2/types/src/beacon_block.rs b/eth2/types/src/beacon_block.rs
index b4d2752d6..d198d16fc 100644
--- a/eth2/types/src/beacon_block.rs
+++ b/eth2/types/src/beacon_block.rs
@@ -6,7 +6,7 @@ use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
 use tree_hash::TreeHash;
-use tree_hash_derive::{SignedRoot, TreeHash};
+use tree_hash_derive::{CachedTreeHash, SignedRoot, TreeHash};
 
 /// A block of the `BeaconChain`.
 ///
@@ -20,6 +20,7 @@ use tree_hash_derive::{SignedRoot, TreeHash};
     Encode,
     Decode,
     TreeHash,
+    CachedTreeHash,
     TestRandom,
     SignedRoot,
 )]
@@ -100,4 +101,5 @@ mod tests {
     use super::*;
 
     ssz_tests!(BeaconBlock);
+    cached_tree_hash_tests!(BeaconBlock);
 }
diff --git a/eth2/types/src/beacon_block_body.rs b/eth2/types/src/beacon_block_body.rs
index de4951f1f..15ba00d6b 100644
--- a/eth2/types/src/beacon_block_body.rs
+++ b/eth2/types/src/beacon_block_body.rs
@@ -4,12 +4,23 @@ use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
-use tree_hash_derive::TreeHash;
+use tree_hash_derive::{CachedTreeHash, TreeHash};
 
 /// The body of a `BeaconChain` block, containing operations.
 ///
 /// Spec v0.5.1
-#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(
+    Debug,
+    PartialEq,
+    Clone,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    CachedTreeHash,
+    TestRandom,
+)]
 pub struct BeaconBlockBody {
     pub randao_reveal: Signature,
     pub eth1_data: Eth1Data,
@@ -26,4 +37,5 @@ mod tests {
     use super::*;
 
     ssz_tests!(BeaconBlockBody);
+    cached_tree_hash_tests!(BeaconBlockBody);
 }
diff --git a/eth2/types/src/beacon_block_header.rs b/eth2/types/src/beacon_block_header.rs
index e4db3a721..5b35da1b6 100644
--- a/eth2/types/src/beacon_block_header.rs
+++ b/eth2/types/src/beacon_block_header.rs
@@ -6,7 +6,7 @@ use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
 use tree_hash::{SignedRoot, TreeHash};
-use tree_hash_derive::{SignedRoot, TreeHash};
+use tree_hash_derive::{CachedTreeHash, SignedRoot, TreeHash};
 
 /// A header of a `BeaconBlock`.
 ///
@@ -20,6 +20,7 @@ use tree_hash_derive::{SignedRoot, TreeHash};
     Encode,
     Decode,
     TreeHash,
+    CachedTreeHash,
     TestRandom,
     SignedRoot,
 )]
@@ -59,4 +60,5 @@ mod tests {
     use super::*;
 
     ssz_tests!(BeaconBlockHeader);
+    cached_tree_hash_tests!(BeaconBlockHeader);
 }
diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs
index eef408308..227e6a4f9 100644
--- a/eth2/types/src/beacon_state.rs
+++ b/eth2/types/src/beacon_state.rs
@@ -9,7 +9,7 @@ use ssz::{hash, ssz_encode};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
 use tree_hash::TreeHash;
-use tree_hash_derive::TreeHash;
+use tree_hash_derive::{CachedTreeHash, TreeHash};
 
 mod epoch_cache;
 mod pubkey_cache;
@@ -47,7 +47,18 @@ pub enum Error {
 /// The state of the `BeaconChain` at some slot.
 ///
 /// Spec v0.5.1
-#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, TestRandom, Encode, Decode, TreeHash)]
+#[derive(
+    Debug,
+    PartialEq,
+    Clone,
+    Serialize,
+    Deserialize,
+    TestRandom,
+    Encode,
+    Decode,
+    TreeHash,
+    CachedTreeHash,
+)]
 pub struct BeaconState {
     // Misc
     pub slot: Slot,
diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs
index dc16a013b..f417dd555 100644
--- a/eth2/types/src/beacon_state/tests.rs
+++ b/eth2/types/src/beacon_state/tests.rs
@@ -3,6 +3,7 @@ use super::*;
 use crate::test_utils::*;
 
 ssz_tests!(BeaconState);
+cached_tree_hash_tests!(BeaconState);
 
 /// Test that
 ///
diff --git a/eth2/types/src/crosslink.rs b/eth2/types/src/crosslink.rs
index 623226ad6..448f5ea30 100644
--- a/eth2/types/src/crosslink.rs
+++ b/eth2/types/src/crosslink.rs
@@ -4,7 +4,7 @@ use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
-use tree_hash_derive::TreeHash;
+use tree_hash_derive::{CachedTreeHash, TreeHash};
 
 /// Specifies the block hash for a shard at an epoch.
 ///
@@ -20,6 +20,7 @@ use tree_hash_derive::TreeHash;
     Encode,
     Decode,
     TreeHash,
+    CachedTreeHash,
     TestRandom,
 )]
 pub struct Crosslink {
@@ -32,4 +33,5 @@ mod tests {
     use super::*;
 
     ssz_tests!(Crosslink);
+    cached_tree_hash_tests!(Crosslink);
 }
diff --git a/eth2/types/src/crosslink_committee.rs b/eth2/types/src/crosslink_committee.rs
index e8fc1b96d..25c42c07b 100644
--- a/eth2/types/src/crosslink_committee.rs
+++ b/eth2/types/src/crosslink_committee.rs
@@ -1,9 +1,20 @@
 use crate::*;
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
-use tree_hash_derive::TreeHash;
+use tree_hash_derive::{CachedTreeHash, TreeHash};
 
-#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize, Decode, Encode, TreeHash)]
+#[derive(
+    Default,
+    Clone,
+    Debug,
+    PartialEq,
+    Serialize,
+    Deserialize,
+    Decode,
+    Encode,
+    TreeHash,
+    CachedTreeHash,
+)]
 pub struct CrosslinkCommittee {
     pub slot: Slot,
     pub shard: Shard,
diff --git a/eth2/types/src/deposit.rs b/eth2/types/src/deposit.rs
index 291173d34..e8d2f5a4b 100644
--- a/eth2/types/src/deposit.rs
+++ b/eth2/types/src/deposit.rs
@@ -4,12 +4,23 @@ use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
-use tree_hash_derive::TreeHash;
+use tree_hash_derive::{CachedTreeHash, TreeHash};
 
 /// A deposit to potentially become a beacon chain validator.
 ///
 /// Spec v0.5.1
-#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(
+    Debug,
+    PartialEq,
+    Clone,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    CachedTreeHash,
+    TestRandom,
+)]
 pub struct Deposit {
     pub proof: TreeHashVector<Hash256>,
     pub index: u64,
@@ -21,4 +32,5 @@ mod tests {
     use super::*;
 
     ssz_tests!(Deposit);
+    cached_tree_hash_tests!(Deposit);
 }
diff --git a/eth2/types/src/deposit_data.rs b/eth2/types/src/deposit_data.rs
index bc96ac7c4..38c44d1a7 100644
--- a/eth2/types/src/deposit_data.rs
+++ b/eth2/types/src/deposit_data.rs
@@ -4,12 +4,23 @@ use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
-use tree_hash_derive::TreeHash;
+use tree_hash_derive::{CachedTreeHash, TreeHash};
 
 /// Data generated by the deposit contract.
 ///
 /// Spec v0.5.1
-#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(
+    Debug,
+    PartialEq,
+    Clone,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    CachedTreeHash,
+    TestRandom,
+)]
 pub struct DepositData {
     pub amount: u64,
     pub timestamp: u64,
@@ -21,4 +32,5 @@ mod tests {
     use super::*;
 
     ssz_tests!(DepositData);
+    cached_tree_hash_tests!(DepositData);
 }
diff --git a/eth2/types/src/deposit_input.rs b/eth2/types/src/deposit_input.rs
index be2106cb4..af1049a20 100644
--- a/eth2/types/src/deposit_input.rs
+++ b/eth2/types/src/deposit_input.rs
@@ -6,7 +6,7 @@ use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
 use tree_hash::{SignedRoot, TreeHash};
-use tree_hash_derive::{SignedRoot, TreeHash};
+use tree_hash_derive::{CachedTreeHash, SignedRoot, TreeHash};
 
 /// The data supplied by the user to the deposit contract.
 ///
@@ -21,6 +21,7 @@ use tree_hash_derive::{SignedRoot, TreeHash};
     Decode,
     SignedRoot,
     TreeHash,
+    CachedTreeHash,
     TestRandom,
 )]
 pub struct DepositInput {
@@ -68,6 +69,7 @@ mod tests {
     use super::*;
 
     ssz_tests!(DepositInput);
+    cached_tree_hash_tests!(DepositInput);
 
     #[test]
     fn can_create_and_validate() {
diff --git a/eth2/types/src/eth1_data.rs b/eth2/types/src/eth1_data.rs
index 2ad460d13..3c0c3af02 100644
--- a/eth2/types/src/eth1_data.rs
+++ b/eth2/types/src/eth1_data.rs
@@ -4,13 +4,23 @@ use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
-use tree_hash_derive::TreeHash;
+use tree_hash_derive::{CachedTreeHash, TreeHash};
 
 /// Contains data obtained from the Eth1 chain.
 ///
 /// Spec v0.5.1
 #[derive(
-    Debug, PartialEq, Clone, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom,
+    Debug,
+    PartialEq,
+    Clone,
+    Default,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    CachedTreeHash,
+    TestRandom,
 )]
 pub struct Eth1Data {
     pub deposit_root: Hash256,
@@ -22,4 +32,5 @@ mod tests {
     use super::*;
 
     ssz_tests!(Eth1Data);
+    cached_tree_hash_tests!(Eth1Data);
 }
diff --git a/eth2/types/src/eth1_data_vote.rs b/eth2/types/src/eth1_data_vote.rs
index 7a77c8ff0..00818ebf4 100644
--- a/eth2/types/src/eth1_data_vote.rs
+++ b/eth2/types/src/eth1_data_vote.rs
@@ -4,13 +4,23 @@ use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
-use tree_hash_derive::TreeHash;
+use tree_hash_derive::{CachedTreeHash, TreeHash};
 
 /// A summation of votes for some `Eth1Data`.
 ///
 /// Spec v0.5.1
 #[derive(
-    Debug, PartialEq, Clone, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom,
+    Debug,
+    PartialEq,
+    Clone,
+    Default,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    CachedTreeHash,
+    TestRandom,
 )]
 pub struct Eth1DataVote {
     pub eth1_data: Eth1Data,
@@ -22,4 +32,5 @@ mod tests {
     use super::*;
 
     ssz_tests!(Eth1DataVote);
+    cached_tree_hash_tests!(Eth1DataVote);
 }
diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs
index d99842855..83d4f5dc6 100644
--- a/eth2/types/src/fork.rs
+++ b/eth2/types/src/fork.rs
@@ -7,13 +7,23 @@ use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
-use tree_hash_derive::TreeHash;
+use tree_hash_derive::{CachedTreeHash, TreeHash};
 
 /// Specifies a fork of the `BeaconChain`, to prevent replay attacks.
 ///
 /// Spec v0.5.1
 #[derive(
-    Debug, Clone, PartialEq, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom,
+    Debug,
+    Clone,
+    PartialEq,
+    Default,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    CachedTreeHash,
+    TestRandom,
 )]
 pub struct Fork {
     #[serde(deserialize_with = "fork_from_hex_str")]
@@ -54,6 +64,7 @@ mod tests {
     use super::*;
 
     ssz_tests!(Fork);
+    cached_tree_hash_tests!(Fork);
 
     fn test_genesis(version: u32, epoch: Epoch) {
         let mut spec = ChainSpec::foundation();
diff --git a/eth2/types/src/historical_batch.rs b/eth2/types/src/historical_batch.rs
index c4f62fcfc..13e57131a 100644
--- a/eth2/types/src/historical_batch.rs
+++ b/eth2/types/src/historical_batch.rs
@@ -4,12 +4,23 @@ use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
-use tree_hash_derive::TreeHash;
+use tree_hash_derive::{CachedTreeHash, TreeHash};
 
 /// Historical block and state roots.
 ///
 /// Spec v0.5.1
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(
+    Debug,
+    Clone,
+    PartialEq,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    CachedTreeHash,
+    TestRandom,
+)]
 pub struct HistoricalBatch {
     pub block_roots: TreeHashVector<Hash256>,
     pub state_roots: TreeHashVector<Hash256>,
@@ -20,4 +31,5 @@ mod tests {
     use super::*;
 
     ssz_tests!(HistoricalBatch);
+    cached_tree_hash_tests!(HistoricalBatch);
 }
diff --git a/eth2/types/src/pending_attestation.rs b/eth2/types/src/pending_attestation.rs
index ce9ce3d77..b71351f9a 100644
--- a/eth2/types/src/pending_attestation.rs
+++ b/eth2/types/src/pending_attestation.rs
@@ -4,12 +4,23 @@ use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
-use tree_hash_derive::TreeHash;
+use tree_hash_derive::{CachedTreeHash, TreeHash};
 
 /// An attestation that has been included in the state but not yet fully processed.
 ///
 /// Spec v0.5.1
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(
+    Debug,
+    Clone,
+    PartialEq,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    CachedTreeHash,
+    TestRandom,
+)]
 pub struct PendingAttestation {
     pub aggregation_bitfield: Bitfield,
     pub data: AttestationData,
@@ -34,4 +45,5 @@ mod tests {
     use super::*;
 
     ssz_tests!(PendingAttestation);
+    cached_tree_hash_tests!(PendingAttestation);
 }
diff --git a/eth2/types/src/proposer_slashing.rs b/eth2/types/src/proposer_slashing.rs
index a3501a5bd..bf26ae508 100644
--- a/eth2/types/src/proposer_slashing.rs
+++ b/eth2/types/src/proposer_slashing.rs
@@ -4,12 +4,23 @@ use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
-use tree_hash_derive::TreeHash;
+use tree_hash_derive::{CachedTreeHash, TreeHash};
 
 /// Two conflicting proposals from the same proposer (validator).
 ///
 /// Spec v0.5.1
-#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(
+    Debug,
+    PartialEq,
+    Clone,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    CachedTreeHash,
+    TestRandom,
+)]
 pub struct ProposerSlashing {
     pub proposer_index: u64,
     pub header_1: BeaconBlockHeader,
@@ -21,4 +32,5 @@ mod tests {
     use super::*;
 
     ssz_tests!(ProposerSlashing);
+    cached_tree_hash_tests!(ProposerSlashing);
 }
diff --git a/eth2/types/src/slashable_attestation.rs b/eth2/types/src/slashable_attestation.rs
index 9c460e482..fb838e0c4 100644
--- a/eth2/types/src/slashable_attestation.rs
+++ b/eth2/types/src/slashable_attestation.rs
@@ -4,7 +4,7 @@ use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
 use tree_hash::TreeHash;
-use tree_hash_derive::{SignedRoot, TreeHash};
+use tree_hash_derive::{CachedTreeHash, SignedRoot, TreeHash};
 
 /// Details an attestation that can be slashable.
 ///
@@ -20,6 +20,7 @@ use tree_hash_derive::{SignedRoot, TreeHash};
     Encode,
     Decode,
     TreeHash,
+    CachedTreeHash,
     TestRandom,
     SignedRoot,
 )]
@@ -133,6 +134,7 @@ mod tests {
     }
 
     ssz_tests!(SlashableAttestation);
+    cached_tree_hash_tests!(SlashableAttestation);
 
     fn create_slashable_attestation(
         slot_factor: u64,
diff --git a/eth2/types/src/slot_epoch_macros.rs b/eth2/types/src/slot_epoch_macros.rs
index b3ca5c4bc..446838d0a 100644
--- a/eth2/types/src/slot_epoch_macros.rs
+++ b/eth2/types/src/slot_epoch_macros.rs
@@ -224,6 +224,26 @@ macro_rules! impl_ssz {
             }
         }
 
+        impl cached_tree_hash::CachedTreeHash<$type> for $type {
+            fn new_tree_hash_cache(
+                &self,
+                depth: usize,
+            ) -> Result<cached_tree_hash::TreeHashCache, cached_tree_hash::Error> {
+                self.0.new_tree_hash_cache(depth)
+            }
+
+            fn tree_hash_cache_schema(&self, depth: usize) -> cached_tree_hash::BTreeSchema {
+                self.0.tree_hash_cache_schema(depth)
+            }
+
+            fn update_tree_hash_cache(
+                &self,
+                cache: &mut cached_tree_hash::TreeHashCache,
+            ) -> Result<(), cached_tree_hash::Error> {
+                self.0.update_tree_hash_cache(cache)
+            }
+        }
+
         impl<T: RngCore> TestRandom<T> for $type {
             fn random_for_test(rng: &mut T) -> Self {
                 $type::from(u64::random_for_test(rng))
@@ -545,6 +565,7 @@ macro_rules! all_tests {
         math_between_tests!($type, $type);
         math_tests!($type);
         ssz_tests!($type);
+        cached_tree_hash_tests!($type);
 
         mod u64_tests {
             use super::*;
diff --git a/eth2/types/src/test_utils/macros.rs b/eth2/types/src/test_utils/macros.rs
index d5711e96e..5f355bfe9 100644
--- a/eth2/types/src/test_utils/macros.rs
+++ b/eth2/types/src/test_utils/macros.rs
@@ -32,3 +32,29 @@ macro_rules! ssz_tests {
         }
     };
 }
+
+#[cfg(test)]
+#[macro_export]
+macro_rules! cached_tree_hash_tests {
+    ($type: ident) => {
+        #[test]
+        pub fn test_cached_tree_hash() {
+            use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+            use tree_hash::TreeHash;
+
+            let mut rng = XorShiftRng::from_seed([42; 16]);
+
+            let original = $type::random_for_test(&mut rng);
+
+            let mut hasher = cached_tree_hash::CachedTreeHasher::new(&original).unwrap();
+
+            assert_eq!(hasher.tree_hash_root().unwrap(), original.tree_hash_root());
+
+            let modified = $type::random_for_test(&mut rng);
+
+            hasher.update(&modified).unwrap();
+
+            assert_eq!(hasher.tree_hash_root().unwrap(), modified.tree_hash_root());
+        }
+    };
+}
diff --git a/eth2/types/src/transfer.rs b/eth2/types/src/transfer.rs
index 82ead03d5..aea13fdd7 100644
--- a/eth2/types/src/transfer.rs
+++ b/eth2/types/src/transfer.rs
@@ -7,7 +7,7 @@ use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
 use tree_hash::TreeHash;
-use tree_hash_derive::{SignedRoot, TreeHash};
+use tree_hash_derive::{CachedTreeHash, SignedRoot, TreeHash};
 
 /// The data submitted to the deposit contract.
 ///
@@ -20,6 +20,7 @@ use tree_hash_derive::{SignedRoot, TreeHash};
     Encode,
     Decode,
     TreeHash,
+    CachedTreeHash,
     TestRandom,
     SignedRoot,
     Derivative,
@@ -42,4 +43,5 @@ mod tests {
     use super::*;
 
     ssz_tests!(Transfer);
+    cached_tree_hash_tests!(Transfer);
 }
diff --git a/eth2/types/src/tree_hash_vector.rs b/eth2/types/src/tree_hash_vector.rs
index 4b6ebad37..6279c4512 100644
--- a/eth2/types/src/tree_hash_vector.rs
+++ b/eth2/types/src/tree_hash_vector.rs
@@ -108,3 +108,25 @@ where
         Vec::random_for_test(rng).into()
     }
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use tree_hash::TreeHash;
+
+    #[test]
+    pub fn test_cached_tree_hash() {
+        let original = TreeHashVector::from(vec![1_u64, 2, 3, 4]);
+
+        let mut hasher = cached_tree_hash::CachedTreeHasher::new(&original).unwrap();
+
+        assert_eq!(hasher.tree_hash_root().unwrap(), original.tree_hash_root());
+
+        let modified = TreeHashVector::from(vec![1_u64, 1, 1, 1]);
+
+        hasher.update(&modified).unwrap();
+
+        assert_eq!(hasher.tree_hash_root().unwrap(), modified.tree_hash_root());
+    }
+
+}
diff --git a/eth2/types/src/validator.rs b/eth2/types/src/validator.rs
index bbd68ed2b..a20eb6426 100644
--- a/eth2/types/src/validator.rs
+++ b/eth2/types/src/validator.rs
@@ -3,12 +3,23 @@ use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
-use tree_hash_derive::TreeHash;
+use tree_hash_derive::{CachedTreeHash, TreeHash};
 
 /// Information about a `BeaconChain` validator.
 ///
 /// Spec v0.5.1
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TestRandom, TreeHash)]
+#[derive(
+    Debug,
+    Clone,
+    PartialEq,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TestRandom,
+    TreeHash,
+    CachedTreeHash,
+)]
 pub struct Validator {
     pub pubkey: PublicKey,
     pub withdrawal_credentials: Hash256,
@@ -111,4 +122,5 @@ mod tests {
     }
 
     ssz_tests!(Validator);
+    cached_tree_hash_tests!(Validator);
 }
diff --git a/eth2/types/src/voluntary_exit.rs b/eth2/types/src/voluntary_exit.rs
index cb872cb98..8a780db75 100644
--- a/eth2/types/src/voluntary_exit.rs
+++ b/eth2/types/src/voluntary_exit.rs
@@ -5,7 +5,7 @@ use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
 use tree_hash::TreeHash;
-use tree_hash_derive::{SignedRoot, TreeHash};
+use tree_hash_derive::{CachedTreeHash, SignedRoot, TreeHash};
 
 /// An exit voluntarily submitted a validator who wishes to withdraw.
 ///
@@ -19,6 +19,7 @@ use tree_hash_derive::{SignedRoot, TreeHash};
     Encode,
     Decode,
     TreeHash,
+    CachedTreeHash,
     TestRandom,
     SignedRoot,
 )]
@@ -34,4 +35,5 @@ mod tests {
     use super::*;
 
     ssz_tests!(VoluntaryExit);
+    cached_tree_hash_tests!(VoluntaryExit);
 }
diff --git a/eth2/utils/bls/Cargo.toml b/eth2/utils/bls/Cargo.toml
index 4ce499580..dcace15c8 100644
--- a/eth2/utils/bls/Cargo.toml
+++ b/eth2/utils/bls/Cargo.toml
@@ -6,6 +6,7 @@ edition = "2018"
 
 [dependencies]
 bls-aggregates = { git = "https://github.com/sigp/signature-schemes", tag = "0.6.1" }
+cached_tree_hash = { path = "../cached_tree_hash" }
 hashing = { path = "../hashing" }
 hex = "0.3"
 serde = "1.0"
diff --git a/eth2/utils/bls/src/aggregate_signature.rs b/eth2/utils/bls/src/aggregate_signature.rs
index 0fbcc3493..e6c6cff9a 100644
--- a/eth2/utils/bls/src/aggregate_signature.rs
+++ b/eth2/utils/bls/src/aggregate_signature.rs
@@ -2,6 +2,7 @@ use super::{AggregatePublicKey, Signature, BLS_AGG_SIG_BYTE_SIZE};
 use bls_aggregates::{
     AggregatePublicKey as RawAggregatePublicKey, AggregateSignature as RawAggregateSignature,
 };
+use cached_tree_hash::cached_tree_hash_ssz_encoding_as_vector;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::{encode as hex_encode, HexVisitor};
@@ -167,6 +168,7 @@ impl<'de> Deserialize<'de> for AggregateSignature {
 }
 
 tree_hash_ssz_encoding_as_vector!(AggregateSignature);
+cached_tree_hash_ssz_encoding_as_vector!(AggregateSignature, 96);
 
 #[cfg(test)]
 mod tests {
diff --git a/eth2/utils/bls/src/fake_aggregate_signature.rs b/eth2/utils/bls/src/fake_aggregate_signature.rs
index f201eba3e..aeb89507d 100644
--- a/eth2/utils/bls/src/fake_aggregate_signature.rs
+++ b/eth2/utils/bls/src/fake_aggregate_signature.rs
@@ -1,4 +1,5 @@
 use super::{fake_signature::FakeSignature, AggregatePublicKey, BLS_AGG_SIG_BYTE_SIZE};
+use cached_tree_hash::cached_tree_hash_ssz_encoding_as_vector;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::{encode as hex_encode, PrefixedHexVisitor};
@@ -100,6 +101,7 @@ impl<'de> Deserialize<'de> for FakeAggregateSignature {
 }
 
 tree_hash_ssz_encoding_as_vector!(FakeAggregateSignature);
+cached_tree_hash_ssz_encoding_as_vector!(FakeAggregateSignature, 96);
 
 #[cfg(test)]
 mod tests {
diff --git a/eth2/utils/bls/src/fake_signature.rs b/eth2/utils/bls/src/fake_signature.rs
index 3208ed992..8a333b9c0 100644
--- a/eth2/utils/bls/src/fake_signature.rs
+++ b/eth2/utils/bls/src/fake_signature.rs
@@ -1,4 +1,5 @@
 use super::{PublicKey, SecretKey, BLS_SIG_BYTE_SIZE};
+use cached_tree_hash::cached_tree_hash_ssz_encoding_as_vector;
 use hex::encode as hex_encode;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
@@ -75,6 +76,7 @@ impl Decodable for FakeSignature {
 }
 
 tree_hash_ssz_encoding_as_vector!(FakeSignature);
+cached_tree_hash_ssz_encoding_as_vector!(FakeSignature, 96);
 
 impl Serialize for FakeSignature {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
diff --git a/eth2/utils/bls/src/public_key.rs b/eth2/utils/bls/src/public_key.rs
index dcbbc622a..e6e4d3508 100644
--- a/eth2/utils/bls/src/public_key.rs
+++ b/eth2/utils/bls/src/public_key.rs
@@ -1,5 +1,6 @@
 use super::{SecretKey, BLS_PUBLIC_KEY_BYTE_SIZE};
 use bls_aggregates::PublicKey as RawPublicKey;
+use cached_tree_hash::cached_tree_hash_ssz_encoding_as_vector;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::{encode as hex_encode, HexVisitor};
@@ -106,6 +107,7 @@ impl<'de> Deserialize<'de> for PublicKey {
 }
 
 tree_hash_ssz_encoding_as_vector!(PublicKey);
+cached_tree_hash_ssz_encoding_as_vector!(PublicKey, 48);
 
 impl PartialEq for PublicKey {
     fn eq(&self, other: &PublicKey) -> bool {
@@ -129,6 +131,7 @@ impl Hash for PublicKey {
 mod tests {
     use super::*;
     use ssz::ssz_encode;
+    use tree_hash::TreeHash;
 
     #[test]
     pub fn test_ssz_round_trip() {
@@ -140,4 +143,21 @@ mod tests {
 
         assert_eq!(original, decoded);
     }
+
+    #[test]
+    pub fn test_cached_tree_hash() {
+        let sk = SecretKey::random();
+        let original = PublicKey::from_secret_key(&sk);
+
+        let mut hasher = cached_tree_hash::CachedTreeHasher::new(&original).unwrap();
+
+        assert_eq!(hasher.tree_hash_root().unwrap(), original.tree_hash_root());
+
+        let sk = SecretKey::random();
+        let modified = PublicKey::from_secret_key(&sk);
+
+        hasher.update(&modified).unwrap();
+
+        assert_eq!(hasher.tree_hash_root().unwrap(), modified.tree_hash_root());
+    }
 }
diff --git a/eth2/utils/bls/src/signature.rs b/eth2/utils/bls/src/signature.rs
index 3fb68dc53..75f43aaf4 100644
--- a/eth2/utils/bls/src/signature.rs
+++ b/eth2/utils/bls/src/signature.rs
@@ -1,5 +1,6 @@
 use super::{PublicKey, SecretKey, BLS_SIG_BYTE_SIZE};
 use bls_aggregates::Signature as RawSignature;
+use cached_tree_hash::cached_tree_hash_ssz_encoding_as_vector;
 use hex::encode as hex_encode;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
@@ -116,6 +117,7 @@ impl Decodable for Signature {
 }
 
 tree_hash_ssz_encoding_as_vector!(Signature);
+cached_tree_hash_ssz_encoding_as_vector!(Signature, 96);
 
 impl Serialize for Signature {
     /// Serde serialization is compliant the Ethereum YAML test format.
@@ -145,6 +147,7 @@ mod tests {
     use super::super::Keypair;
     use super::*;
     use ssz::ssz_encode;
+    use tree_hash::TreeHash;
 
     #[test]
     pub fn test_ssz_round_trip() {
@@ -158,6 +161,22 @@ mod tests {
         assert_eq!(original, decoded);
     }
 
+    #[test]
+    pub fn test_cached_tree_hash() {
+        let keypair = Keypair::random();
+        let original = Signature::new(&[42, 42], 0, &keypair.sk);
+
+        let mut hasher = cached_tree_hash::CachedTreeHasher::new(&original).unwrap();
+
+        assert_eq!(hasher.tree_hash_root().unwrap(), original.tree_hash_root());
+
+        let modified = Signature::new(&[99, 99], 0, &keypair.sk);
+
+        hasher.update(&modified).unwrap();
+
+        assert_eq!(hasher.tree_hash_root().unwrap(), modified.tree_hash_root());
+    }
+
     #[test]
     pub fn test_empty_signature() {
         let sig = Signature::empty_signature();
diff --git a/eth2/utils/boolean-bitfield/Cargo.toml b/eth2/utils/boolean-bitfield/Cargo.toml
index 61bbc60a8..dfc97ce77 100644
--- a/eth2/utils/boolean-bitfield/Cargo.toml
+++ b/eth2/utils/boolean-bitfield/Cargo.toml
@@ -5,6 +5,7 @@ authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 
 [dependencies]
+cached_tree_hash = { path = "../cached_tree_hash" }
 serde_hex = { path = "../serde_hex" }
 ssz = { path = "../ssz" }
 bit-vec = "0.5.0"
diff --git a/eth2/utils/boolean-bitfield/src/lib.rs b/eth2/utils/boolean-bitfield/src/lib.rs
index 1d0f1c02e..cdc58173f 100644
--- a/eth2/utils/boolean-bitfield/src/lib.rs
+++ b/eth2/utils/boolean-bitfield/src/lib.rs
@@ -3,6 +3,7 @@ extern crate ssz;
 
 use bit_reverse::LookupReverse;
 use bit_vec::BitVec;
+use cached_tree_hash::cached_tree_hash_bytes_as_list;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::{encode, PrefixedHexVisitor};
@@ -270,11 +271,31 @@ impl tree_hash::TreeHash for BooleanBitfield {
     }
 }
 
+cached_tree_hash_bytes_as_list!(BooleanBitfield);
+
 #[cfg(test)]
 mod tests {
     use super::*;
     use serde_yaml;
     use ssz::{decode, ssz_encode, SszStream};
+    use tree_hash::TreeHash;
+
+    #[test]
+    pub fn test_cached_tree_hash() {
+        let original = BooleanBitfield::from_bytes(&vec![18; 12][..]);
+
+        let mut hasher = cached_tree_hash::CachedTreeHasher::new(&original).unwrap();
+
+        assert_eq!(hasher.tree_hash_root().unwrap(), original.tree_hash_root());
+        /*
+
+        let modified = BooleanBitfield::from_bytes(&vec![2; 1][..]);
+
+        hasher.update(&modified).unwrap();
+
+        assert_eq!(hasher.tree_hash_root().unwrap(), modified.tree_hash_root());
+        */
+    }
 
     #[test]
     fn test_new_bitfield() {
diff --git a/eth2/utils/cached_tree_hash/src/impls.rs b/eth2/utils/cached_tree_hash/src/impls.rs
index 687e74918..a859a8847 100644
--- a/eth2/utils/cached_tree_hash/src/impls.rs
+++ b/eth2/utils/cached_tree_hash/src/impls.rs
@@ -1,12 +1,46 @@
 use super::*;
 use crate::merkleize::merkleize;
+use ethereum_types::H256;
 
 pub mod vec;
 
-impl CachedTreeHash<u64> for u64 {
+macro_rules! impl_for_single_leaf_int {
+    ($type: ident) => {
+        impl CachedTreeHash<$type> for $type {
+            fn new_tree_hash_cache(&self, _depth: usize) -> Result<TreeHashCache, Error> {
+                Ok(TreeHashCache::from_bytes(
+                    merkleize(self.to_le_bytes().to_vec()),
+                    false,
+                    None,
+                )?)
+            }
+
+            fn tree_hash_cache_schema(&self, depth: usize) -> BTreeSchema {
+                BTreeSchema::from_lengths(depth, vec![1])
+            }
+
+            fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
+                let leaf = merkleize(self.to_le_bytes().to_vec());
+                cache.maybe_update_chunk(cache.chunk_index, &leaf)?;
+
+                cache.chunk_index += 1;
+
+                Ok(())
+            }
+        }
+    };
+}
+
+impl_for_single_leaf_int!(u8);
+impl_for_single_leaf_int!(u16);
+impl_for_single_leaf_int!(u32);
+impl_for_single_leaf_int!(u64);
+impl_for_single_leaf_int!(usize);
+
+impl CachedTreeHash<bool> for bool {
     fn new_tree_hash_cache(&self, _depth: usize) -> Result<TreeHashCache, Error> {
         Ok(TreeHashCache::from_bytes(
-            merkleize(self.to_le_bytes().to_vec()),
+            merkleize((*self as u8).to_le_bytes().to_vec()),
             false,
             None,
         )?)
@@ -17,20 +51,19 @@ impl CachedTreeHash<u64> for u64 {
     }
 
     fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
-        let leaf = merkleize(self.to_le_bytes().to_vec());
+        let leaf = merkleize((*self as u8).to_le_bytes().to_vec());
         cache.maybe_update_chunk(cache.chunk_index, &leaf)?;
 
         cache.chunk_index += 1;
-        // cache.overlay_index += 1;
 
         Ok(())
     }
 }
 
-impl CachedTreeHash<usize> for usize {
+impl CachedTreeHash<[u8; 4]> for [u8; 4] {
     fn new_tree_hash_cache(&self, _depth: usize) -> Result<TreeHashCache, Error> {
         Ok(TreeHashCache::from_bytes(
-            merkleize(self.to_le_bytes().to_vec()),
+            merkleize(self.to_vec()),
             false,
             None,
         )?)
@@ -41,11 +74,33 @@ impl CachedTreeHash<usize> for usize {
     }
 
     fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
-        let leaf = merkleize(self.to_le_bytes().to_vec());
+        let leaf = merkleize(self.to_vec());
+        cache.maybe_update_chunk(cache.chunk_index, &leaf)?;
+
+        cache.chunk_index += 1;
+
+        Ok(())
+    }
+}
+
+impl CachedTreeHash<H256> for H256 {
+    fn new_tree_hash_cache(&self, _depth: usize) -> Result<TreeHashCache, Error> {
+        Ok(TreeHashCache::from_bytes(
+            merkleize(self.as_bytes().to_vec()),
+            false,
+            None,
+        )?)
+    }
+
+    fn tree_hash_cache_schema(&self, depth: usize) -> BTreeSchema {
+        BTreeSchema::from_lengths(depth, vec![1])
+    }
+
+    fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
+        let leaf = merkleize(self.as_bytes().to_vec());
         cache.maybe_update_chunk(cache.chunk_index, &leaf)?;
 
         cache.chunk_index += 1;
-        // cache.overlay_index += 1;
 
         Ok(())
     }
diff --git a/eth2/utils/cached_tree_hash/src/impls/vec.rs b/eth2/utils/cached_tree_hash/src/impls/vec.rs
index f8ed32399..87ea3bcc3 100644
--- a/eth2/utils/cached_tree_hash/src/impls/vec.rs
+++ b/eth2/utils/cached_tree_hash/src/impls/vec.rs
@@ -95,6 +95,11 @@ pub fn update_tree_hash_cache<T: CachedTreeHash<T>>(
     let old_overlay = cache.get_overlay(cache.schema_index, cache.chunk_index)?;
     let new_overlay = BTreeOverlay::new(vec, cache.chunk_index, old_overlay.depth);
 
+    dbg!(cache.schema_index);
+    dbg!(cache.schemas.len());
+    dbg!(&old_overlay);
+    dbg!(&new_overlay);
+
     cache.replace_overlay(cache.schema_index, cache.chunk_index, new_overlay.clone())?;
 
     cache.schema_index += 1;
diff --git a/eth2/utils/cached_tree_hash/src/lib.rs b/eth2/utils/cached_tree_hash/src/lib.rs
index 316d52d07..24bbe0b07 100644
--- a/eth2/utils/cached_tree_hash/src/lib.rs
+++ b/eth2/utils/cached_tree_hash/src/lib.rs
@@ -1,4 +1,5 @@
 use hashing::hash;
+use merkleize::num_unsanitized_leaves;
 use std::ops::Range;
 use tree_hash::{TreeHash, TreeHashType, BYTES_PER_CHUNK, HASHSIZE};
 
@@ -62,3 +63,92 @@ impl CachedTreeHasher {
         Ok(self.cache.root()?.to_vec())
     }
 }
+
+#[macro_export]
+macro_rules! cached_tree_hash_ssz_encoding_as_vector {
+    ($type: ident, $num_bytes: expr) => {
+        impl cached_tree_hash::CachedTreeHash<$type> for $type {
+            fn new_tree_hash_cache(
+                &self,
+                depth: usize,
+            ) -> Result<cached_tree_hash::TreeHashCache, cached_tree_hash::Error> {
+                let (cache, _schema) = cached_tree_hash::impls::vec::new_tree_hash_cache(
+                    &ssz::ssz_encode(self),
+                    depth,
+                )?;
+
+                Ok(cache)
+            }
+
+            fn tree_hash_cache_schema(&self, depth: usize) -> cached_tree_hash::BTreeSchema {
+                let lengths =
+                    vec![1; cached_tree_hash::merkleize::num_unsanitized_leaves($num_bytes)];
+                cached_tree_hash::BTreeSchema::from_lengths(depth, lengths)
+            }
+
+            fn update_tree_hash_cache(
+                &self,
+                cache: &mut cached_tree_hash::TreeHashCache,
+            ) -> Result<(), cached_tree_hash::Error> {
+                cached_tree_hash::impls::vec::update_tree_hash_cache(
+                    &ssz::ssz_encode(self),
+                    cache,
+                )?;
+
+                Ok(())
+            }
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! cached_tree_hash_bytes_as_list {
+    ($type: ident) => {
+        impl cached_tree_hash::CachedTreeHash<$type> for $type {
+            fn new_tree_hash_cache(
+                &self,
+                depth: usize,
+            ) -> Result<cached_tree_hash::TreeHashCache, cached_tree_hash::Error> {
+                let bytes = self.to_bytes();
+
+                let (mut cache, schema) =
+                    cached_tree_hash::impls::vec::new_tree_hash_cache(&bytes, depth)?;
+
+                cache.add_length_nodes(schema.into_overlay(0).chunk_range(), bytes.len())?;
+
+                Ok(cache)
+            }
+
+            fn num_tree_hash_cache_chunks(&self) -> usize {
+                // Add two extra nodes to cater for the node before and after to allow mixing-in length.
+                cached_tree_hash::BTreeOverlay::new(self, 0, 0).num_chunks() + 2
+            }
+
+            fn tree_hash_cache_schema(&self, depth: usize) -> cached_tree_hash::BTreeSchema {
+                cached_tree_hash::impls::vec::produce_schema(&ssz::ssz_encode(self), depth)
+            }
+
+            fn update_tree_hash_cache(
+                &self,
+                cache: &mut cached_tree_hash::TreeHashCache,
+            ) -> Result<(), cached_tree_hash::Error> {
+                let bytes = self.to_bytes();
+
+                // Skip the length-mixed-in root node.
+                cache.chunk_index += 1;
+
+                // Update the cache, returning the new overlay.
+                let new_overlay =
+                    cached_tree_hash::impls::vec::update_tree_hash_cache(&bytes, cache)?;
+
+                // Mix in length
+                cache.mix_in_length(new_overlay.chunk_range(), bytes.len())?;
+
+                // Skip an extra node to clear the length node.
+                cache.chunk_index = new_overlay.next_node() + 1;
+
+                Ok(())
+            }
+        }
+    };
+}
diff --git a/eth2/utils/cached_tree_hash/src/merkleize.rs b/eth2/utils/cached_tree_hash/src/merkleize.rs
index 6bfa73888..e744961b0 100644
--- a/eth2/utils/cached_tree_hash/src/merkleize.rs
+++ b/eth2/utils/cached_tree_hash/src/merkleize.rs
@@ -60,7 +60,7 @@ fn last_leaf_needs_padding(num_bytes: usize) -> bool {
 }
 
 /// Rounds up
-fn num_unsanitized_leaves(num_bytes: usize) -> usize {
+pub fn num_unsanitized_leaves(num_bytes: usize) -> usize {
     (num_bytes + HASHSIZE - 1) / HASHSIZE
 }
 
diff --git a/eth2/utils/cached_tree_hash/tests/tests.rs b/eth2/utils/cached_tree_hash/tests/tests.rs
index 8837fa1da..bc93e0176 100644
--- a/eth2/utils/cached_tree_hash/tests/tests.rs
+++ b/eth2/utils/cached_tree_hash/tests/tests.rs
@@ -231,6 +231,41 @@ pub struct StructWithVecOfStructs {
     pub c: Vec<Inner>,
 }
 
+fn get_inners() -> Vec<Inner> {
+    vec![
+        Inner {
+            a: 12,
+            b: 13,
+            c: 14,
+            d: 15,
+        },
+        Inner {
+            a: 99,
+            b: 100,
+            c: 101,
+            d: 102,
+        },
+        Inner {
+            a: 255,
+            b: 256,
+            c: 257,
+            d: 0,
+        },
+        Inner {
+            a: 1000,
+            b: 2000,
+            c: 3000,
+            d: 0,
+        },
+        Inner {
+            a: 0,
+            b: 0,
+            c: 0,
+            d: 0,
+        },
+    ]
+}
+
 fn get_struct_with_vec_of_structs() -> Vec<StructWithVecOfStructs> {
     let inner_a = Inner {
         a: 12,
@@ -342,6 +377,56 @@ fn test_struct_with_vec_of_struct_with_vec_of_structs() {
     }
 }
 
+#[derive(Clone, Debug, TreeHash, CachedTreeHash)]
+pub struct StructWithTwoVecs {
+    pub a: Vec<Inner>,
+    pub b: Vec<Inner>,
+}
+
+#[test]
+fn test_struct_with_two_vecs() {
+    let inners = get_inners();
+
+    let variants = vec![
+        StructWithTwoVecs {
+            a: inners[..].to_vec(),
+            b: inners[..].to_vec(),
+        },
+        StructWithTwoVecs {
+            a: inners[0..1].to_vec(),
+            b: inners[..].to_vec(),
+        },
+        StructWithTwoVecs {
+            a: inners[0..1].to_vec(),
+            b: inners[0..2].to_vec(),
+        },
+        StructWithTwoVecs {
+            a: inners[0..4].to_vec(),
+            b: inners[0..2].to_vec(),
+        },
+        StructWithTwoVecs {
+            a: vec![],
+            b: inners[..].to_vec(),
+        },
+        StructWithTwoVecs {
+            a: inners[..].to_vec(),
+            b: vec![],
+        },
+        StructWithTwoVecs {
+            a: inners[0..3].to_vec(),
+            b: inners[0..1].to_vec(),
+        },
+    ];
+
+    test_routine(variants[0].clone(), variants[6..7].to_vec());
+
+    /*
+    for v in &variants {
+        test_routine(v.clone(), variants.clone());
+    }
+    */
+}
+
 #[derive(Clone, Debug, TreeHash, CachedTreeHash)]
 pub struct Inner {
     pub a: u64,
diff --git a/eth2/utils/tree_hash/src/impls.rs b/eth2/utils/tree_hash/src/impls.rs
index f2ff8c2bc..e8485dc2f 100644
--- a/eth2/utils/tree_hash/src/impls.rs
+++ b/eth2/utils/tree_hash/src/impls.rs
@@ -52,7 +52,7 @@ impl TreeHash for bool {
 
 impl TreeHash for [u8; 4] {
     fn tree_hash_type() -> TreeHashType {
-        TreeHashType::List
+        TreeHashType::Vector
     }
 
     fn tree_hash_packed_encoding(&self) -> Vec<u8> {

From dddcc91ef31d0793119fee7104ec8fbe683c9277 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Fri, 26 Apr 2019 16:55:19 +1000
Subject: [PATCH 113/137] Fix bug with shrinking list.

---
 .../cached_tree_hash/src/btree_overlay.rs     |  3 ++
 eth2/utils/cached_tree_hash/src/impls/vec.rs  | 43 ++++++++++++++-----
 .../cached_tree_hash/src/tree_hash_cache.rs   |  4 +-
 eth2/utils/cached_tree_hash/tests/tests.rs    |  4 --
 4 files changed, 38 insertions(+), 16 deletions(-)

diff --git a/eth2/utils/cached_tree_hash/src/btree_overlay.rs b/eth2/utils/cached_tree_hash/src/btree_overlay.rs
index 84efdb79b..f01027e2f 100644
--- a/eth2/utils/cached_tree_hash/src/btree_overlay.rs
+++ b/eth2/utils/cached_tree_hash/src/btree_overlay.rs
@@ -228,6 +228,9 @@ mod test {
 
         let tree = BTreeSchema::from_lengths(0, vec![1, 1]).into_overlay(11);
         assert_eq!(tree.chunk_range(), 11..14);
+
+        let tree = BTreeSchema::from_lengths(0, vec![7, 7, 7]).into_overlay(0);
+        assert_eq!(tree.chunk_range(), 0..25);
     }
 
     #[test]
diff --git a/eth2/utils/cached_tree_hash/src/impls/vec.rs b/eth2/utils/cached_tree_hash/src/impls/vec.rs
index 87ea3bcc3..782a5f285 100644
--- a/eth2/utils/cached_tree_hash/src/impls/vec.rs
+++ b/eth2/utils/cached_tree_hash/src/impls/vec.rs
@@ -33,7 +33,7 @@ where
         cache.mix_in_length(new_overlay.chunk_range(), self.len())?;
 
         // Skip an extra node to clear the length node.
-        cache.chunk_index = new_overlay.next_node() + 1;
+        cache.chunk_index += 1;
 
         Ok(())
     }
@@ -95,11 +95,6 @@ pub fn update_tree_hash_cache<T: CachedTreeHash<T>>(
     let old_overlay = cache.get_overlay(cache.schema_index, cache.chunk_index)?;
     let new_overlay = BTreeOverlay::new(vec, cache.chunk_index, old_overlay.depth);
 
-    dbg!(cache.schema_index);
-    dbg!(cache.schemas.len());
-    dbg!(&old_overlay);
-    dbg!(&new_overlay);
-
     cache.replace_overlay(cache.schema_index, cache.chunk_index, new_overlay.clone())?;
 
     cache.schema_index += 1;
@@ -178,10 +173,36 @@ pub fn update_tree_hash_cache<T: CachedTreeHash<T>>(
                             // this node padding.
                             cache.maybe_update_chunk(new_overlay.root(), &[0; HASHSIZE])?;
                         } else {
-                            // In this case, there are some items in the new list and we should
-                            // splice out the entire tree of the removed node, replacing it
-                            // with a single padding node.
-                            cache.splice(old, vec![0; HASHSIZE], vec![true]);
+                            let old_internal_nodes = old_overlay.num_internal_nodes();
+                            let new_internal_nodes = new_overlay.num_internal_nodes();
+
+                            // If the number of internal nodes have shifted between the two
+                            // overlays, the range for this node needs to be shifted to suit the
+                            // new overlay.
+                            let old = if old_internal_nodes > new_internal_nodes {
+                                let offset = old_internal_nodes - new_internal_nodes;
+
+                                old.start - offset..old.end - offset
+                            } else if old_internal_nodes < new_internal_nodes {
+                                let offset = new_internal_nodes - old_internal_nodes;
+
+                                old.start + offset..old.end + offset
+                            } else {
+                                old.start..old.end
+                            };
+
+                            // If there are still some old bytes left-over from this item, replace
+                            // them with a padding chunk.
+                            if old.start < new_overlay.chunk_range().end {
+                                let start_chunk = old.start;
+                                let end_chunk =
+                                    std::cmp::min(old.end, new_overlay.chunk_range().end);
+
+                                // In this case, there are some items in the new list and we should
+                                // splice out the entire tree of the removed node, replacing it
+                                // with a single padding node.
+                                cache.splice(start_chunk..end_chunk, vec![0; HASHSIZE], vec![true]);
+                            }
                         }
                     }
                     // The item didn't exist in the old list and doesn't exist in the new list,
@@ -198,6 +219,8 @@ pub fn update_tree_hash_cache<T: CachedTreeHash<T>>(
 
     cache.update_internal_nodes(&new_overlay)?;
 
+    cache.chunk_index = new_overlay.next_node();
+
     Ok(new_overlay)
 }
 
diff --git a/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
index 1fdd1fa5c..2697821d5 100644
--- a/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
+++ b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
@@ -77,7 +77,7 @@ impl TreeHashCache {
         cache.splice(0..internal_node_bytes, merkleized);
 
         Ok(Self {
-            chunk_modified: vec![false; cache.len() / BYTES_PER_CHUNK],
+            chunk_modified: vec![true; cache.len() / BYTES_PER_CHUNK],
             cache,
             schemas,
             chunk_index: 0,
@@ -141,7 +141,7 @@ impl TreeHashCache {
         // This grows/shrinks the bytes to accomodate the new tree, preserving as much of the tree
         // as possible.
         if new_overlay.num_leaf_nodes() != old_overlay.num_leaf_nodes() {
-            // Get slices of the exsiting tree from the cache.
+            // Get slices of the existing tree from the cache.
             let (old_bytes, old_flags) = self
                 .slices(old_overlay.chunk_range())
                 .ok_or_else(|| Error::UnableToObtainSlices)?;
diff --git a/eth2/utils/cached_tree_hash/tests/tests.rs b/eth2/utils/cached_tree_hash/tests/tests.rs
index bc93e0176..c3392ba27 100644
--- a/eth2/utils/cached_tree_hash/tests/tests.rs
+++ b/eth2/utils/cached_tree_hash/tests/tests.rs
@@ -418,13 +418,9 @@ fn test_struct_with_two_vecs() {
         },
     ];
 
-    test_routine(variants[0].clone(), variants[6..7].to_vec());
-
-    /*
     for v in &variants {
         test_routine(v.clone(), variants.clone());
     }
-    */
 }
 
 #[derive(Clone, Debug, TreeHash, CachedTreeHash)]

From d3309b9f7ebd2f803c018842573398167ea0f054 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Fri, 26 Apr 2019 17:27:39 +1000
Subject: [PATCH 114/137] Fix bug with bitfield tree hash caching

---
 eth2/utils/boolean-bitfield/src/lib.rs | 2 --
 eth2/utils/cached_tree_hash/src/lib.rs | 5 +++--
 eth2/utils/tree_hash_derive/src/lib.rs | 1 +
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/eth2/utils/boolean-bitfield/src/lib.rs b/eth2/utils/boolean-bitfield/src/lib.rs
index cdc58173f..c5fa590d6 100644
--- a/eth2/utils/boolean-bitfield/src/lib.rs
+++ b/eth2/utils/boolean-bitfield/src/lib.rs
@@ -287,14 +287,12 @@ mod tests {
         let mut hasher = cached_tree_hash::CachedTreeHasher::new(&original).unwrap();
 
         assert_eq!(hasher.tree_hash_root().unwrap(), original.tree_hash_root());
-        /*
 
         let modified = BooleanBitfield::from_bytes(&vec![2; 1][..]);
 
         hasher.update(&modified).unwrap();
 
         assert_eq!(hasher.tree_hash_root().unwrap(), modified.tree_hash_root());
-        */
     }
 
     #[test]
diff --git a/eth2/utils/cached_tree_hash/src/lib.rs b/eth2/utils/cached_tree_hash/src/lib.rs
index 24bbe0b07..0faf56ea2 100644
--- a/eth2/utils/cached_tree_hash/src/lib.rs
+++ b/eth2/utils/cached_tree_hash/src/lib.rs
@@ -125,7 +125,8 @@ macro_rules! cached_tree_hash_bytes_as_list {
             }
 
             fn tree_hash_cache_schema(&self, depth: usize) -> cached_tree_hash::BTreeSchema {
-                cached_tree_hash::impls::vec::produce_schema(&ssz::ssz_encode(self), depth)
+                let bytes = self.to_bytes();
+                cached_tree_hash::impls::vec::produce_schema(&bytes, depth)
             }
 
             fn update_tree_hash_cache(
@@ -145,7 +146,7 @@ macro_rules! cached_tree_hash_bytes_as_list {
                 cache.mix_in_length(new_overlay.chunk_range(), bytes.len())?;
 
                 // Skip an extra node to clear the length node.
-                cache.chunk_index = new_overlay.next_node() + 1;
+                cache.chunk_index += 1;
 
                 Ok(())
             }
diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs
index 910417b81..fdfe1e5c0 100644
--- a/eth2/utils/tree_hash_derive/src/lib.rs
+++ b/eth2/utils/tree_hash_derive/src/lib.rs
@@ -88,6 +88,7 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
             fn update_tree_hash_cache(&self, cache: &mut cached_tree_hash::TreeHashCache) -> Result<(), cached_tree_hash::Error> {
                 let overlay = cached_tree_hash::BTreeOverlay::new(self, cache.chunk_index, 0);
 
+
                 // Skip the chunk index to the first leaf node of this struct.
                 cache.chunk_index = overlay.first_leaf_node();
                 // Skip the overlay index to the first leaf node of this struct.

From a425beb42ac26c53e01a4718df2905ba638ee0c4 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Fri, 26 Apr 2019 17:39:38 +1000
Subject: [PATCH 115/137] Fix chunk index bug with structs

---
 eth2/types/src/test_utils/macros.rs        |  6 ++--
 eth2/utils/cached_tree_hash/src/lib.rs     |  2 +-
 eth2/utils/cached_tree_hash/tests/tests.rs | 39 ++++++++++++++++++++++
 eth2/utils/tree_hash_derive/src/lib.rs     |  2 ++
 4 files changed, 44 insertions(+), 5 deletions(-)

diff --git a/eth2/types/src/test_utils/macros.rs b/eth2/types/src/test_utils/macros.rs
index 5f355bfe9..984f2962f 100644
--- a/eth2/types/src/test_utils/macros.rs
+++ b/eth2/types/src/test_utils/macros.rs
@@ -44,16 +44,14 @@ macro_rules! cached_tree_hash_tests {
 
             let mut rng = XorShiftRng::from_seed([42; 16]);
 
+            // Test the original hash
             let original = $type::random_for_test(&mut rng);
-
             let mut hasher = cached_tree_hash::CachedTreeHasher::new(&original).unwrap();
-
             assert_eq!(hasher.tree_hash_root().unwrap(), original.tree_hash_root());
 
+            // Test the updated hash
             let modified = $type::random_for_test(&mut rng);
-
             hasher.update(&modified).unwrap();
-
             assert_eq!(hasher.tree_hash_root().unwrap(), modified.tree_hash_root());
         }
     };
diff --git a/eth2/utils/cached_tree_hash/src/lib.rs b/eth2/utils/cached_tree_hash/src/lib.rs
index 0faf56ea2..b1397b0f4 100644
--- a/eth2/utils/cached_tree_hash/src/lib.rs
+++ b/eth2/utils/cached_tree_hash/src/lib.rs
@@ -28,7 +28,7 @@ pub trait CachedTreeHash<Item>: TreeHash {
 
 #[derive(Debug, PartialEq)]
 pub struct CachedTreeHasher {
-    cache: TreeHashCache,
+    pub cache: TreeHashCache,
 }
 
 impl CachedTreeHasher {
diff --git a/eth2/utils/cached_tree_hash/tests/tests.rs b/eth2/utils/cached_tree_hash/tests/tests.rs
index c3392ba27..4ac6a4607 100644
--- a/eth2/utils/cached_tree_hash/tests/tests.rs
+++ b/eth2/utils/cached_tree_hash/tests/tests.rs
@@ -423,6 +423,45 @@ fn test_struct_with_two_vecs() {
     }
 }
 
+#[derive(Clone, Debug, TreeHash, CachedTreeHash)]
+pub struct U64AndTwoStructs {
+    pub a: u64,
+    pub b: Inner,
+    pub c: Inner,
+}
+
+#[test]
+fn test_u64_and_two_structs() {
+    let inners = get_inners();
+
+    let variants = vec![
+        U64AndTwoStructs {
+            a: 99,
+            b: inners[0].clone(),
+            c: inners[1].clone(),
+        },
+        U64AndTwoStructs {
+            a: 10,
+            b: inners[2].clone(),
+            c: inners[3].clone(),
+        },
+        U64AndTwoStructs {
+            a: 0,
+            b: inners[1].clone(),
+            c: inners[1].clone(),
+        },
+        U64AndTwoStructs {
+            a: 0,
+            b: inners[1].clone(),
+            c: inners[1].clone(),
+        },
+    ];
+
+    for v in &variants {
+        test_routine(v.clone(), variants.clone());
+    }
+}
+
 #[derive(Clone, Debug, TreeHash, CachedTreeHash)]
 pub struct Inner {
     pub a: u64,
diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs
index fdfe1e5c0..3ba846307 100644
--- a/eth2/utils/tree_hash_derive/src/lib.rs
+++ b/eth2/utils/tree_hash_derive/src/lib.rs
@@ -102,6 +102,8 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
                 // Iterate through the internal nodes, updating them if their children have changed.
                 cache.update_internal_nodes(&overlay)?;
 
+                cache.chunk_index = overlay.next_node();
+
                 Ok(())
             }
         }

From 0f3b74b20eee105bb0666edcd1e64dd5bf5a0f25 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Fri, 26 Apr 2019 18:10:06 +1000
Subject: [PATCH 116/137] Update `TestRandom` to vary list length

---
 eth2/types/src/test_utils/macros.rs      | 17 +++++++++++++++--
 eth2/types/src/test_utils/test_random.rs | 12 +++++++-----
 eth2/types/src/tree_hash_vector.rs       |  6 +++++-
 3 files changed, 27 insertions(+), 8 deletions(-)

diff --git a/eth2/types/src/test_utils/macros.rs b/eth2/types/src/test_utils/macros.rs
index 984f2962f..193d01b4f 100644
--- a/eth2/types/src/test_utils/macros.rs
+++ b/eth2/types/src/test_utils/macros.rs
@@ -47,12 +47,25 @@ macro_rules! cached_tree_hash_tests {
             // Test the original hash
             let original = $type::random_for_test(&mut rng);
             let mut hasher = cached_tree_hash::CachedTreeHasher::new(&original).unwrap();
-            assert_eq!(hasher.tree_hash_root().unwrap(), original.tree_hash_root());
+            assert_eq!(
+                hasher.tree_hash_root().unwrap(),
+                original.tree_hash_root(),
+                "Original hash failed."
+            );
 
             // Test the updated hash
             let modified = $type::random_for_test(&mut rng);
             hasher.update(&modified).unwrap();
-            assert_eq!(hasher.tree_hash_root().unwrap(), modified.tree_hash_root());
+            dbg!(&hasher.cache.chunk_modified);
+            dbg!(hasher.cache.chunk_modified.len());
+            dbg!(hasher.cache.chunk_index);
+            dbg!(hasher.cache.schemas.len());
+            dbg!(hasher.cache.schema_index);
+            assert_eq!(
+                hasher.tree_hash_root().unwrap(),
+                modified.tree_hash_root(),
+                "Modification hash failed"
+            );
         }
     };
 }
diff --git a/eth2/types/src/test_utils/test_random.rs b/eth2/types/src/test_utils/test_random.rs
index cb7abe3a4..2d4269b08 100644
--- a/eth2/types/src/test_utils/test_random.rs
+++ b/eth2/types/src/test_utils/test_random.rs
@@ -44,11 +44,13 @@ where
     U: TestRandom<T>,
 {
     fn random_for_test(rng: &mut T) -> Self {
-        vec![
-            <U>::random_for_test(rng),
-            <U>::random_for_test(rng),
-            <U>::random_for_test(rng),
-        ]
+        let mut output = vec![];
+
+        for _ in 0..(usize::random_for_test(rng) % 4) {
+            output.push(<U>::random_for_test(rng));
+        }
+
+        output
     }
 }
 
diff --git a/eth2/types/src/tree_hash_vector.rs b/eth2/types/src/tree_hash_vector.rs
index 6279c4512..bfbb42e57 100644
--- a/eth2/types/src/tree_hash_vector.rs
+++ b/eth2/types/src/tree_hash_vector.rs
@@ -105,7 +105,11 @@ where
     U: TestRandom<T>,
 {
     fn random_for_test(rng: &mut T) -> Self {
-        Vec::random_for_test(rng).into()
+        TreeHashVector::from(vec![
+            U::random_for_test(rng),
+            U::random_for_test(rng),
+            U::random_for_test(rng),
+        ])
     }
 }
 

From b86e11806257816466ca0c2bedc214cd9457ff9c Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Fri, 26 Apr 2019 18:19:13 +1000
Subject: [PATCH 117/137] Add specific failing cache test

---
 eth2/utils/cached_tree_hash/tests/tests.rs | 34 ++++++++++++++++++----
 1 file changed, 29 insertions(+), 5 deletions(-)

diff --git a/eth2/utils/cached_tree_hash/tests/tests.rs b/eth2/utils/cached_tree_hash/tests/tests.rs
index 4ac6a4607..cdbeeb4cf 100644
--- a/eth2/utils/cached_tree_hash/tests/tests.rs
+++ b/eth2/utils/cached_tree_hash/tests/tests.rs
@@ -209,15 +209,16 @@ fn test_vec_of_struct_with_vec() {
     };
     let d = StructWithVec { a: 0, ..a.clone() };
 
-    // let original: Vec<StructWithVec> = vec![a.clone(), c.clone()];
-    let original: Vec<StructWithVec> = vec![a.clone()];
+    let original: Vec<StructWithVec> = vec![a.clone(), c.clone()];
 
     let modified = vec![
         vec![a.clone(), c.clone()],
+        vec![],
         vec![a.clone(), b.clone(), c.clone(), d.clone()],
         vec![b.clone(), a.clone(), c.clone(), d.clone()],
         vec![],
         vec![a.clone()],
+        vec![],
         vec![a.clone(), b.clone(), c.clone(), d.clone()],
     ];
 
@@ -383,11 +384,10 @@ pub struct StructWithTwoVecs {
     pub b: Vec<Inner>,
 }
 
-#[test]
-fn test_struct_with_two_vecs() {
+fn get_struct_with_two_vecs() -> Vec<StructWithTwoVecs> {
     let inners = get_inners();
 
-    let variants = vec![
+    vec![
         StructWithTwoVecs {
             a: inners[..].to_vec(),
             b: inners[..].to_vec(),
@@ -416,8 +416,32 @@ fn test_struct_with_two_vecs() {
             a: inners[0..3].to_vec(),
             b: inners[0..1].to_vec(),
         },
+    ]
+}
+
+#[test]
+fn test_struct_with_two_vecs() {
+    let variants = get_struct_with_two_vecs();
+
+    for v in &variants {
+        test_routine(v.clone(), variants.clone());
+    }
+}
+
+#[test]
+fn test_vec_of_struct_with_two_vecs() {
+    let structs = get_struct_with_two_vecs();
+
+    let variants = vec![
+        structs[0..].to_vec(),
+        structs[0..2].to_vec(),
+        structs[2..3].to_vec(),
+        vec![],
+        structs[2..4].to_vec(),
     ];
 
+    test_routine(variants[0].clone(), vec![variants[2].clone()]);
+
     for v in &variants {
         test_routine(v.clone(), variants.clone());
     }

From 80fa5d08c509fc088abe2643ebe18822e1799290 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Sat, 27 Apr 2019 16:22:42 +1000
Subject: [PATCH 118/137] Fix bug with cached tree hash, passes tests

---
 eth2/types/src/test_utils/macros.rs           |   5 -
 .../cached_tree_hash/src/btree_overlay.rs     |  49 ++++-
 eth2/utils/cached_tree_hash/src/impls/vec.rs  | 187 +++++++++++-------
 eth2/utils/cached_tree_hash/src/resize.rs     | 109 +++-------
 .../cached_tree_hash/src/tree_hash_cache.rs   |  34 ++--
 eth2/utils/cached_tree_hash/tests/tests.rs    |  42 +++-
 6 files changed, 243 insertions(+), 183 deletions(-)

diff --git a/eth2/types/src/test_utils/macros.rs b/eth2/types/src/test_utils/macros.rs
index 193d01b4f..1f1853438 100644
--- a/eth2/types/src/test_utils/macros.rs
+++ b/eth2/types/src/test_utils/macros.rs
@@ -56,11 +56,6 @@ macro_rules! cached_tree_hash_tests {
             // Test the updated hash
             let modified = $type::random_for_test(&mut rng);
             hasher.update(&modified).unwrap();
-            dbg!(&hasher.cache.chunk_modified);
-            dbg!(hasher.cache.chunk_modified.len());
-            dbg!(hasher.cache.chunk_index);
-            dbg!(hasher.cache.schemas.len());
-            dbg!(hasher.cache.schema_index);
             assert_eq!(
                 hasher.tree_hash_root().unwrap(),
                 modified.tree_hash_root(),
diff --git a/eth2/utils/cached_tree_hash/src/btree_overlay.rs b/eth2/utils/cached_tree_hash/src/btree_overlay.rs
index f01027e2f..a585c399f 100644
--- a/eth2/utils/cached_tree_hash/src/btree_overlay.rs
+++ b/eth2/utils/cached_tree_hash/src/btree_overlay.rs
@@ -29,6 +29,13 @@ impl Into<BTreeSchema> for BTreeOverlay {
     }
 }
 
+#[derive(Debug, PartialEq, Clone)]
+pub enum LeafNode {
+    DoesNotExist,
+    Exists(Range<usize>),
+    Padding,
+}
+
 #[derive(Debug, PartialEq, Clone)]
 pub struct BTreeOverlay {
     offset: usize,
@@ -82,6 +89,10 @@ impl BTreeOverlay {
         self.num_leaf_nodes().trailing_zeros() as usize
     }
 
+    pub fn internal_chunk_range(&self) -> Range<usize> {
+        self.offset..self.offset + self.num_internal_nodes()
+    }
+
     pub fn chunk_range(&self) -> Range<usize> {
         self.first_node()..self.next_node()
     }
@@ -104,13 +115,15 @@ impl BTreeOverlay {
     ///     - The specified node is internal.
     ///     - The specified node is padding.
     ///     - The specified node is OOB of the tree.
-    pub fn get_leaf_node(&self, i: usize) -> Result<Option<Range<usize>>, Error> {
-        if i >= self.num_nodes() - self.num_padding_leaves() {
-            Ok(None)
+    pub fn get_leaf_node(&self, i: usize) -> Result<LeafNode, Error> {
+        if i >= self.num_nodes() {
+            Ok(LeafNode::DoesNotExist)
+        } else if i >= self.num_nodes() - self.num_padding_leaves() {
+            Ok(LeafNode::Padding)
         } else if (i == self.num_internal_nodes()) && (self.lengths.len() == 0) {
             // If this is the first leaf node and the overlay contains zero items, return `None` as
             // this node must be padding.
-            Ok(None)
+            Ok(LeafNode::Padding)
         } else {
             let i = i - self.num_internal_nodes();
 
@@ -119,7 +132,7 @@ impl BTreeOverlay {
                 + self.lengths.iter().take(i).sum::<usize>();
             let last_node = first_node + self.lengths[i];
 
-            Ok(Some(first_node..last_node))
+            Ok(LeafNode::Exists(first_node..last_node))
         }
     }
 
@@ -237,10 +250,28 @@ mod test {
     fn get_leaf_node() {
         let tree = get_tree_a(4);
 
-        assert_eq!(tree.get_leaf_node(3), Ok(Some(3..4)));
-        assert_eq!(tree.get_leaf_node(4), Ok(Some(4..5)));
-        assert_eq!(tree.get_leaf_node(5), Ok(Some(5..6)));
-        assert_eq!(tree.get_leaf_node(6), Ok(Some(6..7)));
+        assert_eq!(tree.get_leaf_node(3), Ok(LeafNode::Exists(3..4)));
+        assert_eq!(tree.get_leaf_node(4), Ok(LeafNode::Exists(4..5)));
+        assert_eq!(tree.get_leaf_node(5), Ok(LeafNode::Exists(5..6)));
+        assert_eq!(tree.get_leaf_node(6), Ok(LeafNode::Exists(6..7)));
+        assert_eq!(tree.get_leaf_node(7), Ok(LeafNode::DoesNotExist));
+
+        let tree = get_tree_a(3);
+
+        assert_eq!(tree.get_leaf_node(3), Ok(LeafNode::Exists(3..4)));
+        assert_eq!(tree.get_leaf_node(4), Ok(LeafNode::Exists(4..5)));
+        assert_eq!(tree.get_leaf_node(5), Ok(LeafNode::Exists(5..6)));
+        assert_eq!(tree.get_leaf_node(6), Ok(LeafNode::Padding));
+        assert_eq!(tree.get_leaf_node(7), Ok(LeafNode::DoesNotExist));
+
+        let tree = get_tree_a(0);
+
+        assert_eq!(tree.get_leaf_node(0), Ok(LeafNode::Padding));
+        assert_eq!(tree.get_leaf_node(1), Ok(LeafNode::DoesNotExist));
+
+        let tree = BTreeSchema::from_lengths(0, vec![3]).into_overlay(0);
+        assert_eq!(tree.get_leaf_node(0), Ok(LeafNode::Exists(0..3)));
+        assert_eq!(tree.get_leaf_node(1), Ok(LeafNode::DoesNotExist));
     }
 
     #[test]
diff --git a/eth2/utils/cached_tree_hash/src/impls/vec.rs b/eth2/utils/cached_tree_hash/src/impls/vec.rs
index 782a5f285..b3c5dc412 100644
--- a/eth2/utils/cached_tree_hash/src/impls/vec.rs
+++ b/eth2/utils/cached_tree_hash/src/impls/vec.rs
@@ -1,4 +1,5 @@
 use super::*;
+use crate::btree_overlay::LeafNode;
 use crate::merkleize::{merkleize, num_sanitized_leaves, sanitise_bytes};
 
 impl<T> CachedTreeHash<Vec<T>> for Vec<T>
@@ -104,7 +105,24 @@ pub fn update_tree_hash_cache<T: CachedTreeHash<T>>(
             let mut buf = vec![0; HASHSIZE];
             let item_bytes = HASHSIZE / T::tree_hash_packing_factor();
 
-            // Iterate through each of the leaf nodes.
+            // If the number of leaf nodes has changed, resize the cache.
+            if new_overlay.num_leaf_nodes() < old_overlay.num_leaf_nodes() {
+                let start = new_overlay.next_node();
+                let end = start + (old_overlay.num_leaf_nodes() - new_overlay.num_leaf_nodes());
+
+                cache.splice(start..end, vec![], vec![]);
+            } else if new_overlay.num_leaf_nodes() > old_overlay.num_leaf_nodes() {
+                let start = old_overlay.next_node();
+                let new_nodes = new_overlay.num_leaf_nodes() - old_overlay.num_leaf_nodes();
+
+                cache.splice(
+                    start..start,
+                    vec![0; new_nodes * HASHSIZE],
+                    vec![true; new_nodes],
+                );
+            }
+
+            // Iterate through each of the leaf nodes in the new list.
             for i in 0..new_overlay.num_leaf_nodes() {
                 // Iterate through the number of items that may be packing into the leaf node.
                 for j in 0..T::tree_hash_packing_factor() {
@@ -129,85 +147,92 @@ pub fn update_tree_hash_cache<T: CachedTreeHash<T>>(
             }
         }
         TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
-            for i in 0..new_overlay.num_leaf_nodes() {
-                // Adjust `i` so it is a leaf node for each of the overlays.
-                let old_i = i + old_overlay.num_internal_nodes();
-                let new_i = i + new_overlay.num_internal_nodes();
-
+            for i in 0..std::cmp::max(new_overlay.num_leaf_nodes(), old_overlay.num_leaf_nodes()) {
                 match (
-                    old_overlay.get_leaf_node(old_i)?,
-                    new_overlay.get_leaf_node(new_i)?,
+                    old_overlay.get_leaf_node(i + old_overlay.num_internal_nodes())?,
+                    new_overlay.get_leaf_node(i + new_overlay.num_internal_nodes())?,
                 ) {
                     // The item existed in the previous list and exists in the current list.
-                    (Some(_old), Some(new)) => {
+                    //
+                    // Update the item.
+                    (LeafNode::Exists(_old), LeafNode::Exists(new)) => {
                         cache.chunk_index = new.start;
 
                         vec[i].update_tree_hash_cache(cache)?;
                     }
-                    // The item did not exist in the previous list but does exist in this list.
+                    // The list has been lengthened and this is a new item that did not exist in
+                    // the previous list.
                     //
-                    // Viz., the list has been lengthened.
-                    (None, Some(new)) => {
-                        let (bytes, mut bools, schemas) =
-                            TreeHashCache::new(&vec[i], new_overlay.depth + 1)?.into_components();
+                    // Splice the tree for the new item into the current chunk_index.
+                    (LeafNode::DoesNotExist, LeafNode::Exists(new)) => {
+                        splice_in_new_tree(&vec[i], new.start..new.start, new_overlay.depth + 1, cache)?;
 
-                        // Record the number of schemas, this will be used later in the fn.
-                        let num_schemas = schemas.len();
-
-                        // Flag the root node of the new tree as dirty.
-                        bools[0] = true;
-
-                        cache.splice(new.start..new.start + 1, bytes, bools);
-                        cache
-                            .schemas
-                            .splice(cache.schema_index..cache.schema_index, schemas);
-
-                        cache.schema_index += num_schemas;
+                        cache.chunk_index = new.end;
                     }
-                    // The item existed in the previous list but does not exist in this list.
+                    // The list has been lengthened and this is a new item that was prevously a
+                    // padding item.
                     //
-                    // Viz., the list has been shortened.
-                    (Some(old), None) => {
-                        if vec.len() == 0 {
-                            // In this case, the list has been made empty and we should make
-                            // this node padding.
-                            cache.maybe_update_chunk(new_overlay.root(), &[0; HASHSIZE])?;
-                        } else {
-                            let old_internal_nodes = old_overlay.num_internal_nodes();
-                            let new_internal_nodes = new_overlay.num_internal_nodes();
+                    // Splice the tree for the new item over the padding chunk.
+                    (LeafNode::Padding, LeafNode::Exists(new)) => {
+                        splice_in_new_tree(&vec[i], new.start..new.start + 1, new_overlay.depth + 1, cache)?;
 
-                            // If the number of internal nodes have shifted between the two
-                            // overlays, the range for this node needs to be shifted to suit the
-                            // new overlay.
-                            let old = if old_internal_nodes > new_internal_nodes {
-                                let offset = old_internal_nodes - new_internal_nodes;
-
-                                old.start - offset..old.end - offset
-                            } else if old_internal_nodes < new_internal_nodes {
-                                let offset = new_internal_nodes - old_internal_nodes;
-
-                                old.start + offset..old.end + offset
-                            } else {
-                                old.start..old.end
-                            };
-
-                            // If there are still some old bytes left-over from this item, replace
-                            // them with a padding chunk.
-                            if old.start < new_overlay.chunk_range().end {
-                                let start_chunk = old.start;
-                                let end_chunk =
-                                    std::cmp::min(old.end, new_overlay.chunk_range().end);
-
-                                // In this case, there are some items in the new list and we should
-                                // splice out the entire tree of the removed node, replacing it
-                                // with a single padding node.
-                                cache.splice(start_chunk..end_chunk, vec![0; HASHSIZE], vec![true]);
-                            }
-                        }
+                        cache.chunk_index = new.end;
                     }
-                    // The item didn't exist in the old list and doesn't exist in the new list,
-                    // nothing to do.
-                    (None, None) => {}
+                    // The list has been shortened and this item was removed from the list and made
+                    // into padding.
+                    //
+                    // Splice a padding node over the number of nodes the previous item occupied,
+                    // starting at the current chunk_index.
+                    (LeafNode::Exists(old), LeafNode::Padding) => {
+                        let num_chunks = old.end - old.start;
+
+                        cache.splice(
+                            cache.chunk_index..cache.chunk_index + num_chunks,
+                            vec![0; HASHSIZE],
+                            vec![true],
+                        );
+
+                        cache.chunk_index += 1;
+                    }
+                    // The list has been shortened and the item for this leaf existed in the
+                    // previous list, but does not exist in this list.
+                    //
+                    // Remove the number of nodes the previous item occupied, starting at the
+                    // current chunk_index.
+                    (LeafNode::Exists(old), LeafNode::DoesNotExist) => {
+                        let num_chunks = old.end - old.start;
+
+                        cache.splice(
+                            cache.chunk_index..cache.chunk_index + num_chunks,
+                            vec![],
+                            vec![],
+                        );
+                    }
+                    // The list has been shortened and this leaf was padding in the previous list,
+                    // however it should not exist in this list.
+                    //
+                    // Remove one node, starting at the current `chunk_index`.
+                    (LeafNode::Padding, LeafNode::DoesNotExist) => {
+                        cache.splice(cache.chunk_index..cache.chunk_index + 1, vec![], vec![]);
+                    }
+                    // The list has been lengthened and this leaf did not exist in the previous
+                    // list, but should be padding for this list.
+                    //
+                    // Splice in a new padding node at the current chunk_index.
+                    (LeafNode::DoesNotExist, LeafNode::Padding) => {
+                        cache.splice(
+                            cache.chunk_index..cache.chunk_index,
+                            vec![0; HASHSIZE],
+                            vec![true],
+                        );
+
+                        cache.chunk_index += 1;
+                    }
+                    // This leaf was padding in both lists, there's nothing to do.
+                    (LeafNode::Padding, LeafNode::Padding) => (),
+                    // As we are looping through the larger of the lists of leaf nodes, it should
+                    // be impossible for either leaf to be non-existant.
+                    (LeafNode::DoesNotExist, LeafNode::DoesNotExist) => unreachable!(),
                 }
             }
 
@@ -224,6 +249,34 @@ pub fn update_tree_hash_cache<T: CachedTreeHash<T>>(
     Ok(new_overlay)
 }
 
+fn splice_in_new_tree<T>(
+    item: &T,
+    chunks_to_replace: Range<usize>,
+    depth: usize,
+    cache: &mut TreeHashCache,
+) -> Result<(), Error>
+where T: CachedTreeHash<T>
+{
+    let (bytes, mut bools, schemas) =
+        TreeHashCache::new(item, depth)?.into_components();
+
+    // Record the number of schemas, this will be used later in the fn.
+    let num_schemas = schemas.len();
+
+    // Flag the root node of the new tree as dirty.
+    bools[0] = true;
+
+    cache.splice(chunks_to_replace, bytes, bools);
+    cache
+        .schemas
+        .splice(cache.schema_index..cache.schema_index, schemas);
+
+    cache.schema_index += num_schemas;
+
+    Ok(())
+    //
+}
+
 fn get_packed_leaves<T>(vec: &Vec<T>) -> Result<Vec<u8>, Error>
 where
     T: CachedTreeHash<T>,
diff --git a/eth2/utils/cached_tree_hash/src/resize.rs b/eth2/utils/cached_tree_hash/src/resize.rs
index c87746654..770522b11 100644
--- a/eth2/utils/cached_tree_hash/src/resize.rs
+++ b/eth2/utils/cached_tree_hash/src/resize.rs
@@ -1,56 +1,24 @@
 use super::*;
-use std::cmp::min;
 
 /// New vec is bigger than old vec.
-pub fn grow_merkle_cache(
+pub fn grow_merkle_tree(
     old_bytes: &[u8],
     old_flags: &[bool],
     from_height: usize,
     to_height: usize,
 ) -> Option<(Vec<u8>, Vec<bool>)> {
-    // Determine the size of our new tree. It is not just a simple `1 << to_height` as there can be
-    // an arbitrary number of nodes in `old_bytes` leaves if those leaves are subtrees.
-    let to_nodes = {
-        let old_nodes = old_bytes.len() / HASHSIZE;
-        let additional_nodes = old_nodes - nodes_in_tree_of_height(from_height);
-        nodes_in_tree_of_height(to_height) + additional_nodes
-    };
+    let to_nodes = nodes_in_tree_of_height(to_height);
 
     let mut bytes = vec![0; to_nodes * HASHSIZE];
     let mut flags = vec![true; to_nodes];
 
-    let leaf_level = from_height;
+    for i in 0..=from_height {
+        let old_byte_slice = old_bytes.get(byte_range_at_height(i))?;
+        let old_flag_slice = old_flags.get(node_range_at_height(i))?;
 
-    for i in 0..=from_height as usize {
-        // If we're on the leaf slice, grab the first byte and all the of the bytes after that.
-        // This is required because we can have an arbitrary number of bytes at the leaf level
-        // (e.g., the case where there are subtrees as leaves).
-        //
-        // If we're not on a leaf level, the number of nodes is fixed and known.
-        let (old_byte_slice, old_flag_slice) = if i == leaf_level {
-            (
-                old_bytes.get(first_byte_at_height(i)..)?,
-                old_flags.get(first_node_at_height(i)..)?,
-            )
-        } else {
-            (
-                old_bytes.get(byte_range_at_height(i))?,
-                old_flags.get(node_range_at_height(i))?,
-            )
-        };
-
-        let new_i = i + to_height - from_height;
-        let (new_byte_slice, new_flag_slice) = if i == leaf_level {
-            (
-                bytes.get_mut(first_byte_at_height(new_i)..)?,
-                flags.get_mut(first_node_at_height(new_i)..)?,
-            )
-        } else {
-            (
-                bytes.get_mut(byte_range_at_height(new_i))?,
-                flags.get_mut(node_range_at_height(new_i))?,
-            )
-        };
+        let offset = i + to_height - from_height;
+        let new_byte_slice = bytes.get_mut(byte_range_at_height(offset))?;
+        let new_flag_slice = flags.get_mut(node_range_at_height(offset))?;
 
         new_byte_slice
             .get_mut(0..old_byte_slice.len())?
@@ -64,58 +32,33 @@ pub fn grow_merkle_cache(
 }
 
 /// New vec is smaller than old vec.
-pub fn shrink_merkle_cache(
+pub fn shrink_merkle_tree(
     from_bytes: &[u8],
     from_flags: &[bool],
     from_height: usize,
     to_height: usize,
-    to_nodes: usize,
 ) -> Option<(Vec<u8>, Vec<bool>)> {
+    let to_nodes = nodes_in_tree_of_height(to_height);
+
     let mut bytes = vec![0; to_nodes * HASHSIZE];
     let mut flags = vec![true; to_nodes];
 
     for i in 0..=to_height as usize {
-        let from_i = i + from_height - to_height;
+        let offset = i + from_height - to_height;
+        let from_byte_slice = from_bytes.get(byte_range_at_height(offset))?;
+        let from_flag_slice = from_flags.get(node_range_at_height(offset))?;
 
-        let (from_byte_slice, from_flag_slice) = if from_i == from_height {
-            (
-                from_bytes.get(first_byte_at_height(from_i)..)?,
-                from_flags.get(first_node_at_height(from_i)..)?,
-            )
-        } else {
-            (
-                from_bytes.get(byte_range_at_height(from_i))?,
-                from_flags.get(node_range_at_height(from_i))?,
-            )
-        };
+        let to_byte_slice = bytes.get_mut(byte_range_at_height(i))?;
+        let to_flag_slice = flags.get_mut(node_range_at_height(i))?;
 
-        let (to_byte_slice, to_flag_slice) = if i == to_height {
-            (
-                bytes.get_mut(first_byte_at_height(i)..)?,
-                flags.get_mut(first_node_at_height(i)..)?,
-            )
-        } else {
-            (
-                bytes.get_mut(byte_range_at_height(i))?,
-                flags.get_mut(node_range_at_height(i))?,
-            )
-        };
-
-        let num_bytes = min(from_byte_slice.len(), to_byte_slice.len());
-        let num_flags = min(from_flag_slice.len(), to_flag_slice.len());
-
-        to_byte_slice
-            .get_mut(0..num_bytes)?
-            .copy_from_slice(from_byte_slice.get(0..num_bytes)?);
-        to_flag_slice
-            .get_mut(0..num_flags)?
-            .copy_from_slice(from_flag_slice.get(0..num_flags)?);
+        to_byte_slice.copy_from_slice(from_byte_slice.get(0..to_byte_slice.len())?);
+        to_flag_slice.copy_from_slice(from_flag_slice.get(0..to_flag_slice.len())?);
     }
 
     Some((bytes, flags))
 }
 
-fn nodes_in_tree_of_height(h: usize) -> usize {
+pub fn nodes_in_tree_of_height(h: usize) -> usize {
     2 * (1 << h) - 1
 }
 
@@ -128,10 +71,6 @@ fn node_range_at_height(h: usize) -> Range<usize> {
     first_node_at_height(h)..last_node_at_height(h) + 1
 }
 
-fn first_byte_at_height(h: usize) -> usize {
-    first_node_at_height(h) * HASHSIZE
-}
-
 fn first_node_at_height(h: usize) -> usize {
     (1 << h) - 1
 }
@@ -152,7 +91,7 @@ mod test {
         let original_bytes = vec![42; small * HASHSIZE];
         let original_flags = vec![false; small];
 
-        let (grown_bytes, grown_flags) = grow_merkle_cache(
+        let (grown_bytes, grown_flags) = grow_merkle_tree(
             &original_bytes,
             &original_flags,
             (small + 1).trailing_zeros() as usize - 1,
@@ -200,12 +139,11 @@ mod test {
         assert_eq!(expected_bytes, grown_bytes);
         assert_eq!(expected_flags, grown_flags);
 
-        let (shrunk_bytes, shrunk_flags) = shrink_merkle_cache(
+        let (shrunk_bytes, shrunk_flags) = shrink_merkle_tree(
             &grown_bytes,
             &grown_flags,
             (big + 1).trailing_zeros() as usize - 1,
             (small + 1).trailing_zeros() as usize - 1,
-            small,
         )
         .unwrap();
 
@@ -221,7 +159,7 @@ mod test {
         let original_bytes = vec![42; small * HASHSIZE];
         let original_flags = vec![false; small];
 
-        let (grown_bytes, grown_flags) = grow_merkle_cache(
+        let (grown_bytes, grown_flags) = grow_merkle_tree(
             &original_bytes,
             &original_flags,
             (small + 1).trailing_zeros() as usize - 1,
@@ -269,12 +207,11 @@ mod test {
         assert_eq!(expected_bytes, grown_bytes);
         assert_eq!(expected_flags, grown_flags);
 
-        let (shrunk_bytes, shrunk_flags) = shrink_merkle_cache(
+        let (shrunk_bytes, shrunk_flags) = shrink_merkle_tree(
             &grown_bytes,
             &grown_flags,
             (big + 1).trailing_zeros() as usize - 1,
             (small + 1).trailing_zeros() as usize - 1,
-            small,
         )
         .unwrap();
 
diff --git a/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
index 2697821d5..efdd9b65b 100644
--- a/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
+++ b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
@@ -134,7 +134,6 @@ impl TreeHashCache {
         new_overlay: BTreeOverlay,
     ) -> Result<BTreeOverlay, Error> {
         let old_overlay = self.get_overlay(schema_index, chunk_index)?;
-
         // If the merkle tree required to represent the new list is of a different size to the one
         // required for the previous list, then update our cache.
         //
@@ -143,31 +142,40 @@ impl TreeHashCache {
         if new_overlay.num_leaf_nodes() != old_overlay.num_leaf_nodes() {
             // Get slices of the existing tree from the cache.
             let (old_bytes, old_flags) = self
-                .slices(old_overlay.chunk_range())
+                .slices(old_overlay.internal_chunk_range())
                 .ok_or_else(|| Error::UnableToObtainSlices)?;
 
-            let (new_bytes, new_bools) =
+            let (new_bytes, new_flags) = if new_overlay.num_internal_nodes() == 0 {
+                (vec![], vec![])
+            } else if old_overlay.num_internal_nodes() == 0 {
+                let nodes = resize::nodes_in_tree_of_height(new_overlay.height() - 1);
+
+                (vec![0; nodes * HASHSIZE], vec![true; nodes])
+            } else {
                 if new_overlay.num_leaf_nodes() > old_overlay.num_leaf_nodes() {
-                    resize::grow_merkle_cache(
+                    resize::grow_merkle_tree(
                         old_bytes,
                         old_flags,
-                        old_overlay.height(),
-                        new_overlay.height(),
+                        old_overlay.height() - 1,
+                        new_overlay.height() - 1,
                     )
                     .ok_or_else(|| Error::UnableToGrowMerkleTree)?
                 } else {
-                    resize::shrink_merkle_cache(
+                    resize::shrink_merkle_tree(
                         old_bytes,
                         old_flags,
-                        old_overlay.height(),
-                        new_overlay.height(),
-                        new_overlay.num_chunks(),
+                        old_overlay.height() - 1,
+                        new_overlay.height() - 1,
                     )
                     .ok_or_else(|| Error::UnableToShrinkMerkleTree)?
-                };
+                }
+            };
 
-            // Splice the newly created `TreeHashCache` over the existing elements.
-            self.splice(old_overlay.chunk_range(), new_bytes, new_bools);
+            assert_eq!(old_overlay.num_internal_nodes(), old_flags.len());
+            assert_eq!(new_overlay.num_internal_nodes(), new_flags.len());
+
+            // Splice the resized created elements over the existing elements.
+            self.splice(old_overlay.internal_chunk_range(), new_bytes, new_flags);
         }
 
         let old_schema = std::mem::replace(&mut self.schemas[schema_index], new_overlay.into());
diff --git a/eth2/utils/cached_tree_hash/tests/tests.rs b/eth2/utils/cached_tree_hash/tests/tests.rs
index cdbeeb4cf..279547665 100644
--- a/eth2/utils/cached_tree_hash/tests/tests.rs
+++ b/eth2/utils/cached_tree_hash/tests/tests.rs
@@ -10,7 +10,7 @@ pub struct NestedStruct {
 
 fn test_routine<T>(original: T, modified: Vec<T>)
 where
-    T: CachedTreeHash<T>,
+    T: CachedTreeHash<T> + std::fmt::Debug,
 {
     let mut hasher = CachedTreeHasher::new(&original).unwrap();
 
@@ -20,10 +20,23 @@ where
 
     for (i, modified) in modified.iter().enumerate() {
         println!("-- Start of modification {} --", i);
-        // Test after a modification
+
+        // Update the existing hasher.
         hasher
             .update(modified)
             .expect(&format!("Modification {}", i));
+
+        // Create a new hasher from the "modified" struct.
+        let modified_hasher = CachedTreeHasher::new(modified).unwrap();
+
+        // Test that the modified hasher has the same number of chunks as a newly built hasher.
+        assert_eq!(
+            hasher.cache.chunk_modified.len(),
+            modified_hasher.cache.chunk_modified.len(),
+            "Number of chunks is different"
+        );
+
+        // Test the root generated by the updated hasher matches a non-cached tree hash root.
         let standard_root = modified.tree_hash_root();
         let cached_root = hasher
             .tree_hash_root()
@@ -73,7 +86,7 @@ fn test_inner() {
 }
 
 #[test]
-fn test_vec() {
+fn test_vec_of_u64() {
     let original: Vec<u64> = vec![1, 2, 3, 4, 5];
 
     let modified: Vec<Vec<u64>> = vec![
@@ -113,6 +126,29 @@ fn test_nested_list_of_u64() {
     test_routine(original, modified);
 }
 
+#[test]
+fn test_shrinking_vec_of_vec() {
+    let original: Vec<Vec<u64>> = vec![vec![1], vec![2], vec![3], vec![4], vec![5]];
+    let modified: Vec<Vec<u64>> = original[0..3].to_vec();
+
+    let new_hasher = CachedTreeHasher::new(&modified).unwrap();
+
+    let mut modified_hasher = CachedTreeHasher::new(&original).unwrap();
+    modified_hasher.update(&modified).unwrap();
+
+    assert_eq!(
+        new_hasher.cache.schemas.len(),
+        modified_hasher.cache.schemas.len(),
+        "Schema count is different"
+    );
+
+    assert_eq!(
+        new_hasher.cache.chunk_modified.len(),
+        modified_hasher.cache.chunk_modified.len(),
+        "Chunk count is different"
+    );
+}
+
 #[derive(Clone, Debug, TreeHash, CachedTreeHash)]
 pub struct StructWithVec {
     pub a: u64,

From b70ebd09ea46a641335f0ff7c38166ee4c7c3101 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Sat, 27 Apr 2019 16:33:31 +1000
Subject: [PATCH 119/137] Increase detail of cached hashing testing

---
 eth2/utils/cached_tree_hash/tests/tests.rs | 23 +++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/eth2/utils/cached_tree_hash/tests/tests.rs b/eth2/utils/cached_tree_hash/tests/tests.rs
index 279547665..11889fc4b 100644
--- a/eth2/utils/cached_tree_hash/tests/tests.rs
+++ b/eth2/utils/cached_tree_hash/tests/tests.rs
@@ -29,13 +29,34 @@ where
         // Create a new hasher from the "modified" struct.
         let modified_hasher = CachedTreeHasher::new(modified).unwrap();
 
-        // Test that the modified hasher has the same number of chunks as a newly built hasher.
         assert_eq!(
             hasher.cache.chunk_modified.len(),
             modified_hasher.cache.chunk_modified.len(),
             "Number of chunks is different"
         );
 
+        assert_eq!(
+            hasher.cache.cache.len(),
+            modified_hasher.cache.cache.len(),
+            "Number of bytes is different"
+        );
+
+        assert_eq!(
+            hasher.cache.cache, modified_hasher.cache.cache,
+            "Bytes are different"
+        );
+
+        assert_eq!(
+            hasher.cache.schemas.len(),
+            modified_hasher.cache.schemas.len(),
+            "Number of schemas is different"
+        );
+
+        assert_eq!(
+            hasher.cache.schemas, modified_hasher.cache.schemas,
+            "Schemas are different"
+        );
+
         // Test the root generated by the updated hasher matches a non-cached tree hash root.
         let standard_root = modified.tree_hash_root();
         let cached_root = hasher

From 6c9be1a73c41527ec1e13bc411e5ed870b66cb15 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Sat, 27 Apr 2019 19:02:52 +1000
Subject: [PATCH 120/137] Add tree hash cache as field to `BeaconState`.

---
 eth2/types/src/beacon_state.rs                | 48 +++++++++++++++++++
 eth2/types/src/beacon_state/tests.rs          | 19 ++++++++
 eth2/utils/cached_tree_hash/src/errors.rs     |  1 +
 eth2/utils/cached_tree_hash/src/lib.rs        | 12 +----
 .../cached_tree_hash/src/tree_hash_cache.rs   | 44 +++++++++++++++--
 5 files changed, 110 insertions(+), 14 deletions(-)

diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs
index 227e6a4f9..015816403 100644
--- a/eth2/types/src/beacon_state.rs
+++ b/eth2/types/src/beacon_state.rs
@@ -1,6 +1,7 @@
 use self::epoch_cache::{get_active_validator_indices, EpochCache, Error as EpochCacheError};
 use crate::test_utils::TestRandom;
 use crate::*;
+use cached_tree_hash::{TreeHashCache, Error as TreeHashCacheError};
 use int_to_bytes::int_to_bytes32;
 use pubkey_cache::PubkeyCache;
 use rand::RngCore;
@@ -42,6 +43,7 @@ pub enum Error {
     EpochCacheUninitialized(RelativeEpoch),
     RelativeEpochError(RelativeEpochError),
     EpochCacheError(EpochCacheError),
+    TreeHashCacheError(TreeHashCacheError),
 }
 
 /// The state of the `BeaconChain` at some slot.
@@ -123,6 +125,12 @@ pub struct BeaconState {
     #[tree_hash(skip_hashing)]
     #[test_random(default)]
     pub pubkey_cache: PubkeyCache,
+    #[serde(skip_serializing, skip_deserializing)]
+    #[ssz(skip_serializing)]
+    #[ssz(skip_deserializing)]
+    #[tree_hash(skip_hashing)]
+    #[test_random(default)]
+    pub tree_hash_cache: TreeHashCache,
 }
 
 impl BeaconState {
@@ -198,6 +206,7 @@ impl BeaconState {
                 EpochCache::default(),
             ],
             pubkey_cache: PubkeyCache::default(),
+            tree_hash_cache: TreeHashCache::default(),
         }
     }
 
@@ -683,6 +692,7 @@ impl BeaconState {
         self.build_epoch_cache(RelativeEpoch::NextWithoutRegistryChange, spec)?;
         self.build_epoch_cache(RelativeEpoch::NextWithRegistryChange, spec)?;
         self.update_pubkey_cache()?;
+        self.update_tree_hash_cache()?;
 
         Ok(())
     }
@@ -789,6 +799,38 @@ impl BeaconState {
     pub fn drop_pubkey_cache(&mut self) {
         self.pubkey_cache = PubkeyCache::default()
     }
+
+    /// Update the tree hash cache, building it for the first time if it is empty.
+    ///
+    /// Returns the `tree_hash_root` resulting from the update. This root can be considered the
+    /// canonical root of `self`.
+    pub fn update_tree_hash_cache(&mut self) -> Result<Hash256, Error> {
+        if self.tree_hash_cache.is_empty() {
+            self.tree_hash_cache = TreeHashCache::new(self, 0)?;
+        } else {
+            // Move the cache outside of `self` to satisfy the borrow checker.
+            let mut cache = std::mem::replace(&mut self.tree_hash_cache, TreeHashCache::default());
+
+            cache.update(self)?;
+
+            // Move the updated cache back into `self`.
+            self.tree_hash_cache = cache
+        }
+
+        self.cached_tree_hash_root()
+    }
+
+    /// Returns the tree hash root determined by the last execution of `self.update_tree_hash_cache(..)`.
+    ///
+    /// Note: does _not_ update the cache and may return an outdated root.
+    ///
+    /// Returns an error if the cache is not initialized or if an error is encountered during the
+    /// cache update.
+    pub fn cached_tree_hash_root(&self) -> Result<Hash256, Error> {
+        self.tree_hash_cache.root()
+            .and_then(|b| Ok(Hash256::from_slice(b)))
+            .map_err(|e| e.into())
+    }
 }
 
 impl From<RelativeEpochError> for Error {
@@ -802,3 +844,9 @@ impl From<EpochCacheError> for Error {
         Error::EpochCacheError(e)
     }
 }
+
+impl From<TreeHashCacheError> for Error {
+    fn from(e: TreeHashCacheError) -> Error {
+        Error::TreeHashCacheError(e)
+    }
+}
diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs
index f417dd555..d5862559a 100644
--- a/eth2/types/src/beacon_state/tests.rs
+++ b/eth2/types/src/beacon_state/tests.rs
@@ -56,3 +56,22 @@ fn cache_initialization() {
     test_cache_initialization(&mut state, RelativeEpoch::NextWithRegistryChange, &spec);
     test_cache_initialization(&mut state, RelativeEpoch::NextWithoutRegistryChange, &spec);
 }
+
+#[test]
+fn tree_hash_cache() {
+    use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use tree_hash::TreeHash;
+
+    let mut rng = XorShiftRng::from_seed([42; 16]);
+
+    let mut state = BeaconState::random_for_test(&mut rng);
+
+    let root = state.update_tree_hash_cache().unwrap();
+
+    assert_eq!(root.as_bytes(), &state.tree_hash_root()[..]);
+
+    state.slot = state.slot + 1;
+
+    let root = state.update_tree_hash_cache().unwrap();
+    assert_eq!(root.as_bytes(), &state.tree_hash_root()[..]);
+}
diff --git a/eth2/utils/cached_tree_hash/src/errors.rs b/eth2/utils/cached_tree_hash/src/errors.rs
index cd387fa47..d9ac02913 100644
--- a/eth2/utils/cached_tree_hash/src/errors.rs
+++ b/eth2/utils/cached_tree_hash/src/errors.rs
@@ -9,6 +9,7 @@ pub enum Error {
     UnableToGrowMerkleTree,
     UnableToShrinkMerkleTree,
     TreeCannotHaveZeroNodes,
+    CacheNotInitialized,
     ShouldNeverBePacked(TreeHashType),
     BytesAreNotEvenChunks(usize),
     NoModifiedFieldForChunk(usize),
diff --git a/eth2/utils/cached_tree_hash/src/lib.rs b/eth2/utils/cached_tree_hash/src/lib.rs
index b1397b0f4..ee5f98275 100644
--- a/eth2/utils/cached_tree_hash/src/lib.rs
+++ b/eth2/utils/cached_tree_hash/src/lib.rs
@@ -45,17 +45,7 @@ impl CachedTreeHasher {
     where
         T: CachedTreeHash<T>,
     {
-        // Reset the per-hash counters.
-        self.cache.chunk_index = 0;
-        self.cache.schema_index = 0;
-
-        // Reset the "modified" flags for the cache.
-        self.cache.reset_modifications();
-
-        // Update the cache with the (maybe) changed object.
-        item.update_tree_hash_cache(&mut self.cache)?;
-
-        Ok(())
+        self.cache.update(item)
     }
 
     pub fn tree_hash_root(&self) -> Result<Vec<u8>, Error> {
diff --git a/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
index efdd9b65b..089e38469 100644
--- a/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
+++ b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
@@ -12,6 +12,18 @@ pub struct TreeHashCache {
     pub schema_index: usize,
 }
 
+impl Default for TreeHashCache {
+    fn default() -> TreeHashCache {
+        TreeHashCache {
+            cache: vec![],
+            chunk_modified: vec![],
+            schemas: vec![],
+            chunk_index: 0,
+            schema_index: 0,
+        }
+    }
+}
+
 impl Into<Vec<u8>> for TreeHashCache {
     fn into(self) -> Vec<u8> {
         self.cache
@@ -26,6 +38,24 @@ impl TreeHashCache {
         item.new_tree_hash_cache(depth)
     }
 
+    pub fn update<T>(&mut self, item: &T) -> Result<(), Error>
+    where
+        T: CachedTreeHash<T>,
+    {
+        if self.is_empty() {
+            Err(Error::CacheNotInitialized)
+        } else {
+            // Reset the per-hash counters.
+            self.chunk_index = 0;
+            self.schema_index = 0;
+
+            // Reset the "modified" flags for the cache.
+            self.reset_modifications();
+
+            item.update_tree_hash_cache(self)
+        }
+    }
+
     pub fn from_leaves_and_subtrees<T>(
         item: &T,
         leaves_and_subtrees: Vec<Self>,
@@ -108,6 +138,10 @@ impl TreeHashCache {
         })
     }
 
+    pub fn is_empty(&self) -> bool {
+        self.chunk_modified.is_empty()
+    }
+
     pub fn get_overlay(
         &self,
         schema_index: usize,
@@ -210,9 +244,13 @@ impl TreeHashCache {
     }
 
     pub fn root(&self) -> Result<&[u8], Error> {
-        self.cache
-            .get(0..HASHSIZE)
-            .ok_or_else(|| Error::NoBytesForRoot)
+        if self.is_empty() {
+            Err(Error::CacheNotInitialized)
+        } else {
+            self.cache
+                .get(0..HASHSIZE)
+                .ok_or_else(|| Error::NoBytesForRoot)
+        }
     }
 
     pub fn splice(&mut self, chunk_range: Range<usize>, bytes: Vec<u8>, bools: Vec<bool>) {

From 89d64b007f1389432b0492b70eb0b83c71c873bb Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Sat, 27 Apr 2019 19:04:26 +1000
Subject: [PATCH 121/137] Run cargofmt --all

---
 eth2/types/src/beacon_state.rs               |  5 +++--
 eth2/utils/cached_tree_hash/src/impls/vec.rs | 20 +++++++++++++++-----
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs
index 015816403..6948997c5 100644
--- a/eth2/types/src/beacon_state.rs
+++ b/eth2/types/src/beacon_state.rs
@@ -1,7 +1,7 @@
 use self::epoch_cache::{get_active_validator_indices, EpochCache, Error as EpochCacheError};
 use crate::test_utils::TestRandom;
 use crate::*;
-use cached_tree_hash::{TreeHashCache, Error as TreeHashCacheError};
+use cached_tree_hash::{Error as TreeHashCacheError, TreeHashCache};
 use int_to_bytes::int_to_bytes32;
 use pubkey_cache::PubkeyCache;
 use rand::RngCore;
@@ -827,7 +827,8 @@ impl BeaconState {
     /// Returns an error if the cache is not initialized or if an error is encountered during the
     /// cache update.
     pub fn cached_tree_hash_root(&self) -> Result<Hash256, Error> {
-        self.tree_hash_cache.root()
+        self.tree_hash_cache
+            .root()
             .and_then(|b| Ok(Hash256::from_slice(b)))
             .map_err(|e| e.into())
     }
diff --git a/eth2/utils/cached_tree_hash/src/impls/vec.rs b/eth2/utils/cached_tree_hash/src/impls/vec.rs
index b3c5dc412..046a1f97e 100644
--- a/eth2/utils/cached_tree_hash/src/impls/vec.rs
+++ b/eth2/utils/cached_tree_hash/src/impls/vec.rs
@@ -165,7 +165,12 @@ pub fn update_tree_hash_cache<T: CachedTreeHash<T>>(
                     //
                     // Splice the tree for the new item into the current chunk_index.
                     (LeafNode::DoesNotExist, LeafNode::Exists(new)) => {
-                        splice_in_new_tree(&vec[i], new.start..new.start, new_overlay.depth + 1, cache)?;
+                        splice_in_new_tree(
+                            &vec[i],
+                            new.start..new.start,
+                            new_overlay.depth + 1,
+                            cache,
+                        )?;
 
                         cache.chunk_index = new.end;
                     }
@@ -174,7 +179,12 @@ pub fn update_tree_hash_cache<T: CachedTreeHash<T>>(
                     //
                     // Splice the tree for the new item over the padding chunk.
                     (LeafNode::Padding, LeafNode::Exists(new)) => {
-                        splice_in_new_tree(&vec[i], new.start..new.start + 1, new_overlay.depth + 1, cache)?;
+                        splice_in_new_tree(
+                            &vec[i],
+                            new.start..new.start + 1,
+                            new_overlay.depth + 1,
+                            cache,
+                        )?;
 
                         cache.chunk_index = new.end;
                     }
@@ -255,10 +265,10 @@ fn splice_in_new_tree<T>(
     depth: usize,
     cache: &mut TreeHashCache,
 ) -> Result<(), Error>
-where T: CachedTreeHash<T>
+where
+    T: CachedTreeHash<T>,
 {
-    let (bytes, mut bools, schemas) =
-        TreeHashCache::new(item, depth)?.into_components();
+    let (bytes, mut bools, schemas) = TreeHashCache::new(item, depth)?.into_components();
 
     // Record the number of schemas, this will be used later in the fn.
     let num_schemas = schemas.len();

From c58723350ce3d86d81cb2fdcd0cc4ca7eca59348 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Sun, 28 Apr 2019 11:33:29 +1000
Subject: [PATCH 122/137] Improve cached hashing performance

---
 .../cached_tree_hash/src/btree_overlay.rs     | 44 +++++++++++++------
 eth2/utils/cached_tree_hash/src/impls.rs      |  9 ++--
 eth2/utils/cached_tree_hash/tests/tests.rs    | 42 ++++++++++++++++++
 3 files changed, 79 insertions(+), 16 deletions(-)

diff --git a/eth2/utils/cached_tree_hash/src/btree_overlay.rs b/eth2/utils/cached_tree_hash/src/btree_overlay.rs
index a585c399f..2ad2383c8 100644
--- a/eth2/utils/cached_tree_hash/src/btree_overlay.rs
+++ b/eth2/utils/cached_tree_hash/src/btree_overlay.rs
@@ -12,11 +12,7 @@ impl BTreeSchema {
     }
 
     pub fn into_overlay(self, offset: usize) -> BTreeOverlay {
-        BTreeOverlay {
-            offset,
-            depth: self.depth,
-            lengths: self.lengths,
-        }
+        BTreeOverlay::from_schema(self, offset)
     }
 }
 
@@ -41,6 +37,7 @@ pub struct BTreeOverlay {
     offset: usize,
     pub depth: usize,
     lengths: Vec<usize>,
+    leaf_nodes: Vec<LeafNode>,
 }
 
 impl BTreeOverlay {
@@ -48,8 +45,30 @@ impl BTreeOverlay {
     where
         T: CachedTreeHash<T>,
     {
-        item.tree_hash_cache_schema(depth)
-            .into_overlay(initial_offset)
+        Self::from_schema(item.tree_hash_cache_schema(depth), initial_offset)
+    }
+
+    pub fn from_schema(schema: BTreeSchema, offset: usize) -> Self {
+        let num_leaf_nodes = schema.lengths.len().next_power_of_two();
+        let num_internal_nodes = num_leaf_nodes - 1;
+
+        let mut running_offset = offset + num_internal_nodes;
+        let leaf_nodes: Vec<LeafNode> = schema
+            .lengths
+            .iter()
+            .map(|length| {
+                let range = running_offset..running_offset + length;
+                running_offset += length;
+                LeafNode::Exists(range)
+            })
+            .collect();
+
+        Self {
+            offset,
+            depth: schema.depth,
+            lengths: schema.lengths,
+            leaf_nodes,
+        }
     }
 
     pub fn num_leaf_nodes(&self) -> usize {
@@ -127,12 +146,7 @@ impl BTreeOverlay {
         } else {
             let i = i - self.num_internal_nodes();
 
-            let first_node = self.offset
-                + self.num_internal_nodes()
-                + self.lengths.iter().take(i).sum::<usize>();
-            let last_node = first_node + self.lengths[i];
-
-            Ok(LeafNode::Exists(first_node..last_node))
+            Ok(self.leaf_nodes[i].clone())
         }
     }
 
@@ -272,6 +286,10 @@ mod test {
         let tree = BTreeSchema::from_lengths(0, vec![3]).into_overlay(0);
         assert_eq!(tree.get_leaf_node(0), Ok(LeafNode::Exists(0..3)));
         assert_eq!(tree.get_leaf_node(1), Ok(LeafNode::DoesNotExist));
+
+        let tree = BTreeSchema::from_lengths(0, vec![3]).into_overlay(10);
+        assert_eq!(tree.get_leaf_node(0), Ok(LeafNode::Exists(10..13)));
+        assert_eq!(tree.get_leaf_node(1), Ok(LeafNode::DoesNotExist));
     }
 
     #[test]
diff --git a/eth2/utils/cached_tree_hash/src/impls.rs b/eth2/utils/cached_tree_hash/src/impls.rs
index a859a8847..357f94d32 100644
--- a/eth2/utils/cached_tree_hash/src/impls.rs
+++ b/eth2/utils/cached_tree_hash/src/impls.rs
@@ -86,19 +86,22 @@ impl CachedTreeHash<[u8; 4]> for [u8; 4] {
 impl CachedTreeHash<H256> for H256 {
     fn new_tree_hash_cache(&self, _depth: usize) -> Result<TreeHashCache, Error> {
         Ok(TreeHashCache::from_bytes(
-            merkleize(self.as_bytes().to_vec()),
+            self.as_bytes().to_vec(),
             false,
             None,
         )?)
     }
 
+    fn num_tree_hash_cache_chunks(&self) -> usize {
+        1
+    }
+
     fn tree_hash_cache_schema(&self, depth: usize) -> BTreeSchema {
         BTreeSchema::from_lengths(depth, vec![1])
     }
 
     fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
-        let leaf = merkleize(self.as_bytes().to_vec());
-        cache.maybe_update_chunk(cache.chunk_index, &leaf)?;
+        cache.maybe_update_chunk(cache.chunk_index, self.as_bytes())?;
 
         cache.chunk_index += 1;
 
diff --git a/eth2/utils/cached_tree_hash/tests/tests.rs b/eth2/utils/cached_tree_hash/tests/tests.rs
index 11889fc4b..024277e16 100644
--- a/eth2/utils/cached_tree_hash/tests/tests.rs
+++ b/eth2/utils/cached_tree_hash/tests/tests.rs
@@ -1,7 +1,31 @@
 use cached_tree_hash::{merkleize::merkleize, *};
+use ethereum_types::H256 as Hash256;
 use int_to_bytes::int_to_bytes32;
 use tree_hash_derive::{CachedTreeHash, TreeHash};
 
+#[test]
+fn modifications() {
+    let n = 2048;
+
+    let vec: Vec<Hash256> = (0..n).map(|_| Hash256::random()).collect();
+
+    let mut cache = TreeHashCache::new(&vec, 0).unwrap();
+    cache.update(&vec).unwrap();
+
+    let modifications = cache.chunk_modified.iter().filter(|b| **b).count();
+
+    assert_eq!(modifications, 0);
+
+    let mut modified_vec = vec.clone();
+    modified_vec[n - 1] = Hash256::random();
+
+    cache.update(&modified_vec).unwrap();
+
+    let modifications = cache.chunk_modified.iter().filter(|b| **b).count();
+
+    assert_eq!(modifications, n.trailing_zeros() as usize + 2);
+}
+
 #[derive(Clone, Debug, TreeHash, CachedTreeHash)]
 pub struct NestedStruct {
     pub a: u64,
@@ -106,6 +130,24 @@ fn test_inner() {
     test_routine(original, modified);
 }
 
+#[test]
+fn test_vec_of_hash256() {
+    let n = 16;
+
+    let original: Vec<Hash256> = (0..n).map(|_| Hash256::random()).collect();
+
+    let modified: Vec<Vec<Hash256>> = vec![
+        original[..].to_vec(),
+        original[0..n / 2].to_vec(),
+        vec![],
+        original[0..1].to_vec(),
+        original[0..3].to_vec(),
+        original[0..n - 12].to_vec(),
+    ];
+
+    test_routine(original, modified);
+}
+
 #[test]
 fn test_vec_of_u64() {
     let original: Vec<u64> = vec![1, 2, 3, 4, 5];

From 4dd1239b243b467cc61bfec282ecae696f082270 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Sun, 28 Apr 2019 11:33:48 +1000
Subject: [PATCH 123/137] Add caching to state transition

---
 eth2/state_processing/src/per_slot_processing.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/eth2/state_processing/src/per_slot_processing.rs b/eth2/state_processing/src/per_slot_processing.rs
index a68f98c6d..8dbf65fa7 100644
--- a/eth2/state_processing/src/per_slot_processing.rs
+++ b/eth2/state_processing/src/per_slot_processing.rs
@@ -24,7 +24,8 @@ pub fn per_slot_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result<
 }
 
 fn cache_state(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> {
-    let previous_slot_state_root = Hash256::from_slice(&state.tree_hash_root()[..]);
+    // let previous_slot_state_root = Hash256::from_slice(&state.tree_hash_root()[..]);
+    let previous_slot_state_root = state.update_tree_hash_cache()?;
 
     // Note: increment the state slot here to allow use of our `state_root` and `block_root`
     // getter/setter functions.

From a4559e798ddaf45926bccfaf036e318706beec4c Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Sun, 28 Apr 2019 11:38:32 +1000
Subject: [PATCH 124/137] Add benches, examples for cached hashing.

Ignore flamegraph files
---
 .gitignore                                    |  2 +
 eth2/utils/cached_tree_hash/Cargo.toml        |  8 ++
 .../utils/cached_tree_hash/benches/benches.rs | 73 +++++++++++++++++++
 .../examples/8k_hashes_cached.rs              | 21 ++++++
 .../examples/8k_hashes_standard.rs            | 11 +++
 5 files changed, 115 insertions(+)
 create mode 100644 eth2/utils/cached_tree_hash/benches/benches.rs
 create mode 100644 eth2/utils/cached_tree_hash/examples/8k_hashes_cached.rs
 create mode 100644 eth2/utils/cached_tree_hash/examples/8k_hashes_standard.rs

diff --git a/.gitignore b/.gitignore
index 346ef9afa..6b8d4ab21 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,3 +4,5 @@ Cargo.lock
 *.pk
 *.sk
 *.raw_keypairs
+flamegraph.svg
+perf.data*
diff --git a/eth2/utils/cached_tree_hash/Cargo.toml b/eth2/utils/cached_tree_hash/Cargo.toml
index c8881eb0f..c33f20add 100644
--- a/eth2/utils/cached_tree_hash/Cargo.toml
+++ b/eth2/utils/cached_tree_hash/Cargo.toml
@@ -4,7 +4,15 @@ version = "0.1.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 
+[profile.release]
+debug = true
+
+[[bench]]
+name = "benches"
+harness = false
+
 [dev-dependencies]
+criterion = "0.2"
 tree_hash_derive = { path = "../tree_hash_derive" }
 
 [dependencies]
diff --git a/eth2/utils/cached_tree_hash/benches/benches.rs b/eth2/utils/cached_tree_hash/benches/benches.rs
new file mode 100644
index 000000000..be7e26bb5
--- /dev/null
+++ b/eth2/utils/cached_tree_hash/benches/benches.rs
@@ -0,0 +1,73 @@
+#[macro_use]
+extern crate criterion;
+
+use cached_tree_hash::TreeHashCache;
+use criterion::black_box;
+use criterion::{Benchmark, Criterion};
+use ethereum_types::H256 as Hash256;
+use hashing::hash;
+use tree_hash::TreeHash;
+
+fn criterion_benchmark(c: &mut Criterion) {
+    let n = 1024;
+
+    let source_vec: Vec<Hash256> = (0..n).map(|_| Hash256::random()).collect();
+
+    let mut source_modified_vec = source_vec.clone();
+    source_modified_vec[n - 1] = Hash256::random();
+
+    let modified_vec = source_modified_vec.clone();
+    c.bench(
+        &format!("vec_of_{}_hashes", n),
+        Benchmark::new("standard", move |b| {
+            b.iter_with_setup(
+                || modified_vec.clone(),
+                |modified_vec| black_box(modified_vec.tree_hash_root()),
+            )
+        })
+        .sample_size(100),
+    );
+
+    let modified_vec = source_modified_vec.clone();
+    c.bench(
+        &format!("vec_of_{}_hashes", n),
+        Benchmark::new("build_cache", move |b| {
+            b.iter_with_setup(
+                || modified_vec.clone(),
+                |vec| black_box(TreeHashCache::new(&vec, 0)),
+            )
+        })
+        .sample_size(100),
+    );
+
+    let vec = source_vec.clone();
+    let modified_vec = source_modified_vec.clone();
+    c.bench(
+        &format!("vec_of_{}_hashes", n),
+        Benchmark::new("cache_update", move |b| {
+            b.iter_with_setup(
+                || {
+                    let cache = TreeHashCache::new(&vec, 0).unwrap();
+                    (cache, modified_vec.clone())
+                },
+                |(mut cache, modified_vec)| black_box(cache.update(&modified_vec)),
+            )
+        })
+        .sample_size(100),
+    );
+
+    c.bench(
+        &format!("{}_hashes", n),
+        Benchmark::new("hash_64_bytes", move |b| {
+            b.iter(|| {
+                for _ in 0..n {
+                    let _digest = hash(&[42; 64]);
+                }
+            })
+        })
+        .sample_size(100),
+    );
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
diff --git a/eth2/utils/cached_tree_hash/examples/8k_hashes_cached.rs b/eth2/utils/cached_tree_hash/examples/8k_hashes_cached.rs
new file mode 100644
index 000000000..9843e1258
--- /dev/null
+++ b/eth2/utils/cached_tree_hash/examples/8k_hashes_cached.rs
@@ -0,0 +1,21 @@
+use cached_tree_hash::TreeHashCache;
+use ethereum_types::H256 as Hash256;
+
+fn run(vec: &Vec<Hash256>, modified_vec: &Vec<Hash256>) {
+    let mut cache = TreeHashCache::new(vec, 0).unwrap();
+
+    cache.update(modified_vec).unwrap();
+}
+
+fn main() {
+    let n = 2048;
+
+    let vec: Vec<Hash256> = (0..n).map(|_| Hash256::random()).collect();
+
+    let mut modified_vec = vec.clone();
+    modified_vec[n - 1] = Hash256::random();
+
+    for _ in 0..100 {
+        run(&vec, &modified_vec);
+    }
+}
diff --git a/eth2/utils/cached_tree_hash/examples/8k_hashes_standard.rs b/eth2/utils/cached_tree_hash/examples/8k_hashes_standard.rs
new file mode 100644
index 000000000..e5df7ea95
--- /dev/null
+++ b/eth2/utils/cached_tree_hash/examples/8k_hashes_standard.rs
@@ -0,0 +1,11 @@
+use cached_tree_hash::TreeHashCache;
+use ethereum_types::H256 as Hash256;
+use tree_hash::TreeHash;
+
+fn main() {
+    let n = 2048;
+
+    let vec: Vec<Hash256> = (0..n).map(|_| Hash256::random()).collect();
+
+    vec.tree_hash_root();
+}

From 58308e3dc50001a86883e6a398ee88a8afc3a7af Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Sun, 28 Apr 2019 19:10:59 +1000
Subject: [PATCH 125/137] Modify example execution counts

---
 eth2/utils/cached_tree_hash/examples/8k_hashes_cached.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eth2/utils/cached_tree_hash/examples/8k_hashes_cached.rs b/eth2/utils/cached_tree_hash/examples/8k_hashes_cached.rs
index 9843e1258..cb8dc42fb 100644
--- a/eth2/utils/cached_tree_hash/examples/8k_hashes_cached.rs
+++ b/eth2/utils/cached_tree_hash/examples/8k_hashes_cached.rs
@@ -15,7 +15,7 @@ fn main() {
     let mut modified_vec = vec.clone();
     modified_vec[n - 1] = Hash256::random();
 
-    for _ in 0..100 {
+    for _ in 0..10_000 {
         run(&vec, &modified_vec);
     }
 }

From 68b36787e2303149ccae7cc239fee074ed442a3d Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Sun, 28 Apr 2019 20:30:48 +1000
Subject: [PATCH 126/137] Move leaf node creation into separate fn

---
 .../cached_tree_hash/src/btree_overlay.rs     | 84 ++++++++-----------
 eth2/utils/cached_tree_hash/src/impls/vec.rs  | 17 ++--
 2 files changed, 47 insertions(+), 54 deletions(-)

diff --git a/eth2/utils/cached_tree_hash/src/btree_overlay.rs b/eth2/utils/cached_tree_hash/src/btree_overlay.rs
index 2ad2383c8..46b1f31b8 100644
--- a/eth2/utils/cached_tree_hash/src/btree_overlay.rs
+++ b/eth2/utils/cached_tree_hash/src/btree_overlay.rs
@@ -37,7 +37,6 @@ pub struct BTreeOverlay {
     offset: usize,
     pub depth: usize,
     lengths: Vec<usize>,
-    leaf_nodes: Vec<LeafNode>,
 }
 
 impl BTreeOverlay {
@@ -49,11 +48,17 @@ impl BTreeOverlay {
     }
 
     pub fn from_schema(schema: BTreeSchema, offset: usize) -> Self {
-        let num_leaf_nodes = schema.lengths.len().next_power_of_two();
-        let num_internal_nodes = num_leaf_nodes - 1;
+        Self {
+            offset,
+            depth: schema.depth,
+            lengths: schema.lengths,
+        }
+    }
 
-        let mut running_offset = offset + num_internal_nodes;
-        let leaf_nodes: Vec<LeafNode> = schema
+    pub fn get_leaf_nodes(&self, n: usize) -> Vec<LeafNode> {
+        let mut running_offset = self.offset + self.num_internal_nodes();
+
+        let mut leaf_nodes: Vec<LeafNode> = self
             .lengths
             .iter()
             .map(|length| {
@@ -63,12 +68,10 @@ impl BTreeOverlay {
             })
             .collect();
 
-        Self {
-            offset,
-            depth: schema.depth,
-            lengths: schema.lengths,
-            leaf_nodes,
-        }
+        leaf_nodes.resize(self.num_leaf_nodes(), LeafNode::Padding);
+        leaf_nodes.resize(n, LeafNode::DoesNotExist);
+
+        leaf_nodes
     }
 
     pub fn num_leaf_nodes(&self) -> usize {
@@ -128,28 +131,6 @@ impl BTreeOverlay {
         self.offset + self.num_internal_nodes()
     }
 
-    /// Returns the chunk-range for a given leaf node.
-    ///
-    /// Returns `None` if:
-    ///     - The specified node is internal.
-    ///     - The specified node is padding.
-    ///     - The specified node is OOB of the tree.
-    pub fn get_leaf_node(&self, i: usize) -> Result<LeafNode, Error> {
-        if i >= self.num_nodes() {
-            Ok(LeafNode::DoesNotExist)
-        } else if i >= self.num_nodes() - self.num_padding_leaves() {
-            Ok(LeafNode::Padding)
-        } else if (i == self.num_internal_nodes()) && (self.lengths.len() == 0) {
-            // If this is the first leaf node and the overlay contains zero items, return `None` as
-            // this node must be padding.
-            Ok(LeafNode::Padding)
-        } else {
-            let i = i - self.num_internal_nodes();
-
-            Ok(self.leaf_nodes[i].clone())
-        }
-    }
-
     pub fn child_chunks(&self, parent: usize) -> (usize, usize) {
         let children = children(parent);
 
@@ -263,33 +244,38 @@ mod test {
     #[test]
     fn get_leaf_node() {
         let tree = get_tree_a(4);
+        let leaves = tree.get_leaf_nodes(5);
 
-        assert_eq!(tree.get_leaf_node(3), Ok(LeafNode::Exists(3..4)));
-        assert_eq!(tree.get_leaf_node(4), Ok(LeafNode::Exists(4..5)));
-        assert_eq!(tree.get_leaf_node(5), Ok(LeafNode::Exists(5..6)));
-        assert_eq!(tree.get_leaf_node(6), Ok(LeafNode::Exists(6..7)));
-        assert_eq!(tree.get_leaf_node(7), Ok(LeafNode::DoesNotExist));
+        assert_eq!(leaves[0], LeafNode::Exists(3..4));
+        assert_eq!(leaves[1], LeafNode::Exists(4..5));
+        assert_eq!(leaves[2], LeafNode::Exists(5..6));
+        assert_eq!(leaves[3], LeafNode::Exists(6..7));
+        assert_eq!(leaves[4], LeafNode::DoesNotExist);
 
         let tree = get_tree_a(3);
+        let leaves = tree.get_leaf_nodes(5);
 
-        assert_eq!(tree.get_leaf_node(3), Ok(LeafNode::Exists(3..4)));
-        assert_eq!(tree.get_leaf_node(4), Ok(LeafNode::Exists(4..5)));
-        assert_eq!(tree.get_leaf_node(5), Ok(LeafNode::Exists(5..6)));
-        assert_eq!(tree.get_leaf_node(6), Ok(LeafNode::Padding));
-        assert_eq!(tree.get_leaf_node(7), Ok(LeafNode::DoesNotExist));
+        assert_eq!(leaves[0], LeafNode::Exists(3..4));
+        assert_eq!(leaves[1], LeafNode::Exists(4..5));
+        assert_eq!(leaves[2], LeafNode::Exists(5..6));
+        assert_eq!(leaves[3], LeafNode::Padding);
+        assert_eq!(leaves[4], LeafNode::DoesNotExist);
 
         let tree = get_tree_a(0);
+        let leaves = tree.get_leaf_nodes(2);
 
-        assert_eq!(tree.get_leaf_node(0), Ok(LeafNode::Padding));
-        assert_eq!(tree.get_leaf_node(1), Ok(LeafNode::DoesNotExist));
+        assert_eq!(leaves[0], LeafNode::Padding);
+        assert_eq!(leaves[1], LeafNode::DoesNotExist);
 
         let tree = BTreeSchema::from_lengths(0, vec![3]).into_overlay(0);
-        assert_eq!(tree.get_leaf_node(0), Ok(LeafNode::Exists(0..3)));
-        assert_eq!(tree.get_leaf_node(1), Ok(LeafNode::DoesNotExist));
+        let leaves = tree.get_leaf_nodes(2);
+        assert_eq!(leaves[0], LeafNode::Exists(0..3));
+        assert_eq!(leaves[1], LeafNode::DoesNotExist);
 
         let tree = BTreeSchema::from_lengths(0, vec![3]).into_overlay(10);
-        assert_eq!(tree.get_leaf_node(0), Ok(LeafNode::Exists(10..13)));
-        assert_eq!(tree.get_leaf_node(1), Ok(LeafNode::DoesNotExist));
+        let leaves = tree.get_leaf_nodes(2);
+        assert_eq!(leaves[0], LeafNode::Exists(10..13));
+        assert_eq!(leaves[1], LeafNode::DoesNotExist);
     }
 
     #[test]
diff --git a/eth2/utils/cached_tree_hash/src/impls/vec.rs b/eth2/utils/cached_tree_hash/src/impls/vec.rs
index 046a1f97e..67a48ac0b 100644
--- a/eth2/utils/cached_tree_hash/src/impls/vec.rs
+++ b/eth2/utils/cached_tree_hash/src/impls/vec.rs
@@ -147,11 +147,18 @@ pub fn update_tree_hash_cache<T: CachedTreeHash<T>>(
             }
         }
         TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
-            for i in 0..std::cmp::max(new_overlay.num_leaf_nodes(), old_overlay.num_leaf_nodes()) {
-                match (
-                    old_overlay.get_leaf_node(i + old_overlay.num_internal_nodes())?,
-                    new_overlay.get_leaf_node(i + new_overlay.num_internal_nodes())?,
-                ) {
+            let longest_len =
+                std::cmp::max(new_overlay.num_leaf_nodes(), old_overlay.num_leaf_nodes());
+
+            let old_leaf_nodes = old_overlay.get_leaf_nodes(longest_len);
+            let new_leaf_nodes = if old_overlay == new_overlay {
+                old_leaf_nodes.clone()
+            } else {
+                new_overlay.get_leaf_nodes(longest_len)
+            };
+
+            for i in 0..longest_len {
+                match (&old_leaf_nodes[i], &new_leaf_nodes[i]) {
                     // The item existed in the previous list and exists in the current list.
                     //
                     // Update the item.

From 6258abfa9f801ed734ea10102af5b0ed52ad9937 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Mon, 29 Apr 2019 07:34:01 +1000
Subject: [PATCH 127/137] Tidy per_slot_processing fn

---
 eth2/state_processing/src/per_slot_processing.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/eth2/state_processing/src/per_slot_processing.rs b/eth2/state_processing/src/per_slot_processing.rs
index 8dbf65fa7..8f9606723 100644
--- a/eth2/state_processing/src/per_slot_processing.rs
+++ b/eth2/state_processing/src/per_slot_processing.rs
@@ -1,5 +1,5 @@
 use crate::*;
-use tree_hash::{SignedRoot, TreeHash};
+use tree_hash::SignedRoot;
 use types::*;
 
 #[derive(Debug, PartialEq)]
@@ -24,7 +24,6 @@ pub fn per_slot_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result<
 }
 
 fn cache_state(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> {
-    // let previous_slot_state_root = Hash256::from_slice(&state.tree_hash_root()[..]);
     let previous_slot_state_root = state.update_tree_hash_cache()?;
 
     // Note: increment the state slot here to allow use of our `state_root` and `block_root`

From 0599d3f1f8dbe271c7812584454631b162370b37 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Mon, 29 Apr 2019 07:48:48 +1000
Subject: [PATCH 128/137] Remove redundant type param fom `CachedTreeHash`

---
 eth2/types/src/slot_epoch_macros.rs                |  2 +-
 eth2/types/src/tree_hash_vector.rs                 |  4 ++--
 eth2/utils/cached_tree_hash/src/btree_overlay.rs   |  2 +-
 eth2/utils/cached_tree_hash/src/impls.rs           |  8 ++++----
 eth2/utils/cached_tree_hash/src/impls/vec.rs       | 14 +++++++-------
 eth2/utils/cached_tree_hash/src/lib.rs             | 11 +++++------
 eth2/utils/cached_tree_hash/src/tree_hash_cache.rs |  6 +++---
 eth2/utils/cached_tree_hash/tests/tests.rs         |  2 +-
 eth2/utils/tree_hash_derive/src/lib.rs             |  2 +-
 eth2/utils/tree_hash_derive/tests/tests.rs         |  5 +----
 10 files changed, 26 insertions(+), 30 deletions(-)

diff --git a/eth2/types/src/slot_epoch_macros.rs b/eth2/types/src/slot_epoch_macros.rs
index 446838d0a..4a48bba9f 100644
--- a/eth2/types/src/slot_epoch_macros.rs
+++ b/eth2/types/src/slot_epoch_macros.rs
@@ -224,7 +224,7 @@ macro_rules! impl_ssz {
             }
         }
 
-        impl cached_tree_hash::CachedTreeHash<$type> for $type {
+        impl cached_tree_hash::CachedTreeHash for $type {
             fn new_tree_hash_cache(
                 &self,
                 depth: usize,
diff --git a/eth2/types/src/tree_hash_vector.rs b/eth2/types/src/tree_hash_vector.rs
index bfbb42e57..8a7d99a6c 100644
--- a/eth2/types/src/tree_hash_vector.rs
+++ b/eth2/types/src/tree_hash_vector.rs
@@ -55,9 +55,9 @@ where
     }
 }
 
-impl<T> CachedTreeHash<TreeHashVector<T>> for TreeHashVector<T>
+impl<T> CachedTreeHash for TreeHashVector<T>
 where
-    T: CachedTreeHash<T> + TreeHash,
+    T: CachedTreeHash + TreeHash,
 {
     fn new_tree_hash_cache(
         &self,
diff --git a/eth2/utils/cached_tree_hash/src/btree_overlay.rs b/eth2/utils/cached_tree_hash/src/btree_overlay.rs
index 46b1f31b8..7d08210fb 100644
--- a/eth2/utils/cached_tree_hash/src/btree_overlay.rs
+++ b/eth2/utils/cached_tree_hash/src/btree_overlay.rs
@@ -42,7 +42,7 @@ pub struct BTreeOverlay {
 impl BTreeOverlay {
     pub fn new<T>(item: &T, initial_offset: usize, depth: usize) -> Self
     where
-        T: CachedTreeHash<T>,
+        T: CachedTreeHash,
     {
         Self::from_schema(item.tree_hash_cache_schema(depth), initial_offset)
     }
diff --git a/eth2/utils/cached_tree_hash/src/impls.rs b/eth2/utils/cached_tree_hash/src/impls.rs
index 357f94d32..5105ad6a7 100644
--- a/eth2/utils/cached_tree_hash/src/impls.rs
+++ b/eth2/utils/cached_tree_hash/src/impls.rs
@@ -6,7 +6,7 @@ pub mod vec;
 
 macro_rules! impl_for_single_leaf_int {
     ($type: ident) => {
-        impl CachedTreeHash<$type> for $type {
+        impl CachedTreeHash for $type {
             fn new_tree_hash_cache(&self, _depth: usize) -> Result<TreeHashCache, Error> {
                 Ok(TreeHashCache::from_bytes(
                     merkleize(self.to_le_bytes().to_vec()),
@@ -37,7 +37,7 @@ impl_for_single_leaf_int!(u32);
 impl_for_single_leaf_int!(u64);
 impl_for_single_leaf_int!(usize);
 
-impl CachedTreeHash<bool> for bool {
+impl CachedTreeHash for bool {
     fn new_tree_hash_cache(&self, _depth: usize) -> Result<TreeHashCache, Error> {
         Ok(TreeHashCache::from_bytes(
             merkleize((*self as u8).to_le_bytes().to_vec()),
@@ -60,7 +60,7 @@ impl CachedTreeHash<bool> for bool {
     }
 }
 
-impl CachedTreeHash<[u8; 4]> for [u8; 4] {
+impl CachedTreeHash for [u8; 4] {
     fn new_tree_hash_cache(&self, _depth: usize) -> Result<TreeHashCache, Error> {
         Ok(TreeHashCache::from_bytes(
             merkleize(self.to_vec()),
@@ -83,7 +83,7 @@ impl CachedTreeHash<[u8; 4]> for [u8; 4] {
     }
 }
 
-impl CachedTreeHash<H256> for H256 {
+impl CachedTreeHash for H256 {
     fn new_tree_hash_cache(&self, _depth: usize) -> Result<TreeHashCache, Error> {
         Ok(TreeHashCache::from_bytes(
             self.as_bytes().to_vec(),
diff --git a/eth2/utils/cached_tree_hash/src/impls/vec.rs b/eth2/utils/cached_tree_hash/src/impls/vec.rs
index 67a48ac0b..00a1ef9d9 100644
--- a/eth2/utils/cached_tree_hash/src/impls/vec.rs
+++ b/eth2/utils/cached_tree_hash/src/impls/vec.rs
@@ -2,9 +2,9 @@ use super::*;
 use crate::btree_overlay::LeafNode;
 use crate::merkleize::{merkleize, num_sanitized_leaves, sanitise_bytes};
 
-impl<T> CachedTreeHash<Vec<T>> for Vec<T>
+impl<T> CachedTreeHash for Vec<T>
 where
-    T: CachedTreeHash<T> + TreeHash,
+    T: CachedTreeHash + TreeHash,
 {
     fn new_tree_hash_cache(&self, depth: usize) -> Result<TreeHashCache, Error> {
         let (mut cache, schema) = new_tree_hash_cache(self, depth)?;
@@ -40,7 +40,7 @@ where
     }
 }
 
-pub fn new_tree_hash_cache<T: CachedTreeHash<T>>(
+pub fn new_tree_hash_cache<T: CachedTreeHash>(
     vec: &Vec<T>,
     depth: usize,
 ) -> Result<(TreeHashCache, BTreeSchema), Error> {
@@ -65,7 +65,7 @@ pub fn new_tree_hash_cache<T: CachedTreeHash<T>>(
     Ok((cache, schema))
 }
 
-pub fn produce_schema<T: CachedTreeHash<T>>(vec: &Vec<T>, depth: usize) -> BTreeSchema {
+pub fn produce_schema<T: CachedTreeHash>(vec: &Vec<T>, depth: usize) -> BTreeSchema {
     let lengths = match T::tree_hash_type() {
         TreeHashType::Basic => {
             // Ceil division.
@@ -89,7 +89,7 @@ pub fn produce_schema<T: CachedTreeHash<T>>(vec: &Vec<T>, depth: usize) -> BTree
     BTreeSchema::from_lengths(depth, lengths)
 }
 
-pub fn update_tree_hash_cache<T: CachedTreeHash<T>>(
+pub fn update_tree_hash_cache<T: CachedTreeHash>(
     vec: &Vec<T>,
     cache: &mut TreeHashCache,
 ) -> Result<BTreeOverlay, Error> {
@@ -273,7 +273,7 @@ fn splice_in_new_tree<T>(
     cache: &mut TreeHashCache,
 ) -> Result<(), Error>
 where
-    T: CachedTreeHash<T>,
+    T: CachedTreeHash,
 {
     let (bytes, mut bools, schemas) = TreeHashCache::new(item, depth)?.into_components();
 
@@ -296,7 +296,7 @@ where
 
 fn get_packed_leaves<T>(vec: &Vec<T>) -> Result<Vec<u8>, Error>
 where
-    T: CachedTreeHash<T>,
+    T: CachedTreeHash,
 {
     let num_packed_bytes = (BYTES_PER_CHUNK / T::tree_hash_packing_factor()) * vec.len();
     let num_leaves = num_sanitized_leaves(num_packed_bytes);
diff --git a/eth2/utils/cached_tree_hash/src/lib.rs b/eth2/utils/cached_tree_hash/src/lib.rs
index ee5f98275..b9bb8457b 100644
--- a/eth2/utils/cached_tree_hash/src/lib.rs
+++ b/eth2/utils/cached_tree_hash/src/lib.rs
@@ -1,5 +1,4 @@
 use hashing::hash;
-use merkleize::num_unsanitized_leaves;
 use std::ops::Range;
 use tree_hash::{TreeHash, TreeHashType, BYTES_PER_CHUNK, HASHSIZE};
 
@@ -14,7 +13,7 @@ pub use btree_overlay::{BTreeOverlay, BTreeSchema};
 pub use errors::Error;
 pub use tree_hash_cache::TreeHashCache;
 
-pub trait CachedTreeHash<Item>: TreeHash {
+pub trait CachedTreeHash: TreeHash {
     fn tree_hash_cache_schema(&self, depth: usize) -> BTreeSchema;
 
     fn num_tree_hash_cache_chunks(&self) -> usize {
@@ -34,7 +33,7 @@ pub struct CachedTreeHasher {
 impl CachedTreeHasher {
     pub fn new<T>(item: &T) -> Result<Self, Error>
     where
-        T: CachedTreeHash<T>,
+        T: CachedTreeHash,
     {
         Ok(Self {
             cache: TreeHashCache::new(item, 0)?,
@@ -43,7 +42,7 @@ impl CachedTreeHasher {
 
     pub fn update<T>(&mut self, item: &T) -> Result<(), Error>
     where
-        T: CachedTreeHash<T>,
+        T: CachedTreeHash,
     {
         self.cache.update(item)
     }
@@ -57,7 +56,7 @@ impl CachedTreeHasher {
 #[macro_export]
 macro_rules! cached_tree_hash_ssz_encoding_as_vector {
     ($type: ident, $num_bytes: expr) => {
-        impl cached_tree_hash::CachedTreeHash<$type> for $type {
+        impl cached_tree_hash::CachedTreeHash for $type {
             fn new_tree_hash_cache(
                 &self,
                 depth: usize,
@@ -94,7 +93,7 @@ macro_rules! cached_tree_hash_ssz_encoding_as_vector {
 #[macro_export]
 macro_rules! cached_tree_hash_bytes_as_list {
     ($type: ident) => {
-        impl cached_tree_hash::CachedTreeHash<$type> for $type {
+        impl cached_tree_hash::CachedTreeHash for $type {
             fn new_tree_hash_cache(
                 &self,
                 depth: usize,
diff --git a/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
index 089e38469..225ae9d5f 100644
--- a/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
+++ b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
@@ -33,14 +33,14 @@ impl Into<Vec<u8>> for TreeHashCache {
 impl TreeHashCache {
     pub fn new<T>(item: &T, depth: usize) -> Result<Self, Error>
     where
-        T: CachedTreeHash<T>,
+        T: CachedTreeHash,
     {
         item.new_tree_hash_cache(depth)
     }
 
     pub fn update<T>(&mut self, item: &T) -> Result<(), Error>
     where
-        T: CachedTreeHash<T>,
+        T: CachedTreeHash,
     {
         if self.is_empty() {
             Err(Error::CacheNotInitialized)
@@ -62,7 +62,7 @@ impl TreeHashCache {
         depth: usize,
     ) -> Result<Self, Error>
     where
-        T: CachedTreeHash<T>,
+        T: CachedTreeHash,
     {
         let overlay = BTreeOverlay::new(item, 0, depth);
 
diff --git a/eth2/utils/cached_tree_hash/tests/tests.rs b/eth2/utils/cached_tree_hash/tests/tests.rs
index 024277e16..4b7a4e830 100644
--- a/eth2/utils/cached_tree_hash/tests/tests.rs
+++ b/eth2/utils/cached_tree_hash/tests/tests.rs
@@ -34,7 +34,7 @@ pub struct NestedStruct {
 
 fn test_routine<T>(original: T, modified: Vec<T>)
 where
-    T: CachedTreeHash<T> + std::fmt::Debug,
+    T: CachedTreeHash + std::fmt::Debug,
 {
     let mut hasher = CachedTreeHasher::new(&original).unwrap();
 
diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs
index 3ba846307..b111ae7c4 100644
--- a/eth2/utils/tree_hash_derive/src/lib.rs
+++ b/eth2/utils/tree_hash_derive/src/lib.rs
@@ -56,7 +56,7 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
     let idents_c = idents_a.clone();
 
     let output = quote! {
-        impl cached_tree_hash::CachedTreeHash<#name> for #name {
+        impl cached_tree_hash::CachedTreeHash for #name {
             fn new_tree_hash_cache(&self, depth: usize) -> Result<cached_tree_hash::TreeHashCache, cached_tree_hash::Error> {
                 let tree = cached_tree_hash::TreeHashCache::from_leaves_and_subtrees(
                     self,
diff --git a/eth2/utils/tree_hash_derive/tests/tests.rs b/eth2/utils/tree_hash_derive/tests/tests.rs
index 2166fd146..11eae4e02 100644
--- a/eth2/utils/tree_hash_derive/tests/tests.rs
+++ b/eth2/utils/tree_hash_derive/tests/tests.rs
@@ -10,10 +10,7 @@ pub struct Inner {
     pub d: u64,
 }
 
-fn test_standard_and_cached<T>(original: &T, modified: &T)
-where
-    T: CachedTreeHash<T>,
-{
+fn test_standard_and_cached<T: CachedTreeHash>(original: &T, modified: &T) {
     // let mut cache = original.new_tree_hash_cache().unwrap();
     let mut hasher = CachedTreeHasher::new(original).unwrap();
 

From fbf8fad4f1f09174ddcc755eaee4b5fb530c1d92 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Mon, 29 Apr 2019 10:57:19 +1000
Subject: [PATCH 129/137] Add counter-resets to `reset_modifications`

---
 eth2/utils/cached_tree_hash/src/tree_hash_cache.rs | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
index 225ae9d5f..c6b3833c8 100644
--- a/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
+++ b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
@@ -45,11 +45,6 @@ impl TreeHashCache {
         if self.is_empty() {
             Err(Error::CacheNotInitialized)
         } else {
-            // Reset the per-hash counters.
-            self.chunk_index = 0;
-            self.schema_index = 0;
-
-            // Reset the "modified" flags for the cache.
             self.reset_modifications();
 
             item.update_tree_hash_cache(self)
@@ -156,6 +151,10 @@ impl TreeHashCache {
     }
 
     pub fn reset_modifications(&mut self) {
+        // Reset the per-hash counters.
+        self.chunk_index = 0;
+        self.schema_index = 0;
+
         for chunk_modified in &mut self.chunk_modified {
             *chunk_modified = false;
         }
@@ -243,6 +242,10 @@ impl TreeHashCache {
         self.cache.len()
     }
 
+    pub fn tree_hash_root(&self) -> Result<&[u8], Error> {
+        self.root()
+    }
+
     pub fn root(&self) -> Result<&[u8], Error> {
         if self.is_empty() {
             Err(Error::CacheNotInitialized)

From 52695c29e89dbd89b0ca11ce085ad9511b56bff2 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Mon, 29 Apr 2019 10:57:39 +1000
Subject: [PATCH 130/137] Improve cached hash testing in `types`

---
 eth2/types/src/test_utils/macros.rs | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/eth2/types/src/test_utils/macros.rs b/eth2/types/src/test_utils/macros.rs
index 1f1853438..d6739ca0b 100644
--- a/eth2/types/src/test_utils/macros.rs
+++ b/eth2/types/src/test_utils/macros.rs
@@ -46,21 +46,37 @@ macro_rules! cached_tree_hash_tests {
 
             // Test the original hash
             let original = $type::random_for_test(&mut rng);
-            let mut hasher = cached_tree_hash::CachedTreeHasher::new(&original).unwrap();
+            let mut cache = cached_tree_hash::TreeHashCache::new(&original, 0).unwrap();
+
             assert_eq!(
-                hasher.tree_hash_root().unwrap(),
+                cache.tree_hash_root().unwrap().to_vec(),
                 original.tree_hash_root(),
                 "Original hash failed."
             );
 
             // Test the updated hash
             let modified = $type::random_for_test(&mut rng);
-            hasher.update(&modified).unwrap();
+            cache.update(&modified).unwrap();
             assert_eq!(
-                hasher.tree_hash_root().unwrap(),
+                cache.tree_hash_root().unwrap().to_vec(),
                 modified.tree_hash_root(),
                 "Modification hash failed"
             );
+
+            // Produce a new cache for the modified object and compare it to the updated cache.
+            let mut modified_cache = cached_tree_hash::TreeHashCache::new(&modified, 0).unwrap();
+
+            // Reset the caches.
+            cache.reset_modifications();
+            modified_cache.reset_modifications();
+
+            // Ensure the modified cache is the same as a newly created cache. This is a sanity
+            // check to make sure there are no artifacts of the original cache remaining after an
+            // update.
+            assert_eq!(
+                modified_cache, cache,
+                "The modified cache does not match a new cache."
+            )
         }
     };
 }

From a90bbbfd826422cd90659820bcb36642c13a160f Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Mon, 29 Apr 2019 14:04:52 +1000
Subject: [PATCH 131/137] Address various clippy lints, improve comments

---
 .../cached_tree_hash/src/btree_overlay.rs     |  1 -
 eth2/utils/cached_tree_hash/src/impls/vec.rs  |  5 +-
 .../cached_tree_hash/src/tree_hash_cache.rs   | 55 +++++++++++--------
 eth2/utils/tree_hash/src/impls.rs             |  1 +
 4 files changed, 35 insertions(+), 27 deletions(-)

diff --git a/eth2/utils/cached_tree_hash/src/btree_overlay.rs b/eth2/utils/cached_tree_hash/src/btree_overlay.rs
index 7d08210fb..4b3c6cc27 100644
--- a/eth2/utils/cached_tree_hash/src/btree_overlay.rs
+++ b/eth2/utils/cached_tree_hash/src/btree_overlay.rs
@@ -149,7 +149,6 @@ impl BTreeOverlay {
         chunks.append(&mut self.leaf_node_chunks());
 
         (0..self.num_internal_nodes())
-            .into_iter()
             .map(|parent| {
                 let children = children(parent);
                 (chunks[parent], (chunks[children.0], chunks[children.1]))
diff --git a/eth2/utils/cached_tree_hash/src/impls/vec.rs b/eth2/utils/cached_tree_hash/src/impls/vec.rs
index 00a1ef9d9..b04884636 100644
--- a/eth2/utils/cached_tree_hash/src/impls/vec.rs
+++ b/eth2/utils/cached_tree_hash/src/impls/vec.rs
@@ -65,7 +65,7 @@ pub fn new_tree_hash_cache<T: CachedTreeHash>(
     Ok((cache, schema))
 }
 
-pub fn produce_schema<T: CachedTreeHash>(vec: &Vec<T>, depth: usize) -> BTreeSchema {
+pub fn produce_schema<T: CachedTreeHash>(vec: &[T], depth: usize) -> BTreeSchema {
     let lengths = match T::tree_hash_type() {
         TreeHashType::Basic => {
             // Ceil division.
@@ -89,6 +89,7 @@ pub fn produce_schema<T: CachedTreeHash>(vec: &Vec<T>, depth: usize) -> BTreeSch
     BTreeSchema::from_lengths(depth, lengths)
 }
 
+#[allow(clippy::range_plus_one)] // Minor readability lint requiring structural changes; not worth it.
 pub fn update_tree_hash_cache<T: CachedTreeHash>(
     vec: &Vec<T>,
     cache: &mut TreeHashCache,
@@ -294,7 +295,7 @@ where
     //
 }
 
-fn get_packed_leaves<T>(vec: &Vec<T>) -> Result<Vec<u8>, Error>
+fn get_packed_leaves<T>(vec: &[T]) -> Result<Vec<u8>, Error>
 where
     T: CachedTreeHash,
 {
diff --git a/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
index c6b3833c8..283a98974 100644
--- a/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
+++ b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
@@ -168,46 +168,53 @@ impl TreeHashCache {
     ) -> Result<BTreeOverlay, Error> {
         let old_overlay = self.get_overlay(schema_index, chunk_index)?;
         // If the merkle tree required to represent the new list is of a different size to the one
-        // required for the previous list, then update our cache.
+        // required for the previous list, then update the internal nodes.
         //
-        // This grows/shrinks the bytes to accomodate the new tree, preserving as much of the tree
+        // Leaf nodes are not touched, they should be updated externally to this function.
+        //
+        // This grows/shrinks the bytes to accommodate the new tree, preserving as much of the tree
         // as possible.
-        if new_overlay.num_leaf_nodes() != old_overlay.num_leaf_nodes() {
+        if new_overlay.num_internal_nodes() != old_overlay.num_internal_nodes() {
             // Get slices of the existing tree from the cache.
             let (old_bytes, old_flags) = self
                 .slices(old_overlay.internal_chunk_range())
                 .ok_or_else(|| Error::UnableToObtainSlices)?;
 
             let (new_bytes, new_flags) = if new_overlay.num_internal_nodes() == 0 {
+                // The new tree has zero internal nodes, simply return empty lists.
                 (vec![], vec![])
             } else if old_overlay.num_internal_nodes() == 0 {
+                // The old tree has zero nodes and the new tree has some nodes. Create new nodes to
+                // suit.
                 let nodes = resize::nodes_in_tree_of_height(new_overlay.height() - 1);
 
                 (vec![0; nodes * HASHSIZE], vec![true; nodes])
+            } else if new_overlay.num_internal_nodes() > old_overlay.num_internal_nodes() {
+                // The new tree is bigger than the old tree.
+                //
+                // Grow the internal nodes, preserving any existing nodes.
+                resize::grow_merkle_tree(
+                    old_bytes,
+                    old_flags,
+                    old_overlay.height() - 1,
+                    new_overlay.height() - 1,
+                )
+                .ok_or_else(|| Error::UnableToGrowMerkleTree)?
             } else {
-                if new_overlay.num_leaf_nodes() > old_overlay.num_leaf_nodes() {
-                    resize::grow_merkle_tree(
-                        old_bytes,
-                        old_flags,
-                        old_overlay.height() - 1,
-                        new_overlay.height() - 1,
-                    )
-                    .ok_or_else(|| Error::UnableToGrowMerkleTree)?
-                } else {
-                    resize::shrink_merkle_tree(
-                        old_bytes,
-                        old_flags,
-                        old_overlay.height() - 1,
-                        new_overlay.height() - 1,
-                    )
-                    .ok_or_else(|| Error::UnableToShrinkMerkleTree)?
-                }
+                // The new tree is smaller than the old tree.
+                //
+                // Shrink the internal nodes, preserving any existing nodes.
+                resize::shrink_merkle_tree(
+                    old_bytes,
+                    old_flags,
+                    old_overlay.height() - 1,
+                    new_overlay.height() - 1,
+                )
+                .ok_or_else(|| Error::UnableToShrinkMerkleTree)?
             };
 
-            assert_eq!(old_overlay.num_internal_nodes(), old_flags.len());
-            assert_eq!(new_overlay.num_internal_nodes(), new_flags.len());
-
-            // Splice the resized created elements over the existing elements.
+            // Splice the resized created elements over the existing elements, effectively updating
+            // the number of stored internal nodes for this tree.
             self.splice(old_overlay.internal_chunk_range(), new_bytes, new_flags);
         }
 
diff --git a/eth2/utils/tree_hash/src/impls.rs b/eth2/utils/tree_hash/src/impls.rs
index e8485dc2f..56eb7dbf4 100644
--- a/eth2/utils/tree_hash/src/impls.rs
+++ b/eth2/utils/tree_hash/src/impls.rs
@@ -19,6 +19,7 @@ macro_rules! impl_for_bitsize {
                 HASHSIZE / ($bit_size / 8)
             }
 
+            #[allow(clippy::cast_lossless)]
             fn tree_hash_root(&self) -> Vec<u8> {
                 int_to_bytes32(*self as u64)
             }

From 240d1e197a435f129b56fff17aa0e01fe4bdd3e7 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Mon, 29 Apr 2019 14:24:57 +1000
Subject: [PATCH 132/137] Ignore clippy `range_plus_one` lints

---
 eth2/utils/cached_tree_hash/src/resize.rs          | 2 ++
 eth2/utils/cached_tree_hash/src/tree_hash_cache.rs | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/eth2/utils/cached_tree_hash/src/resize.rs b/eth2/utils/cached_tree_hash/src/resize.rs
index 770522b11..5428e234b 100644
--- a/eth2/utils/cached_tree_hash/src/resize.rs
+++ b/eth2/utils/cached_tree_hash/src/resize.rs
@@ -1,3 +1,5 @@
+#![allow(clippy::range_plus_one)] // Minor readability lint requiring structural changes; not worth it.
+
 use super::*;
 
 /// New vec is bigger than old vec.
diff --git a/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
index 283a98974..acb96ba24 100644
--- a/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
+++ b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
@@ -1,3 +1,5 @@
+#![allow(clippy::range_plus_one)] // Minor readability lint requiring structural changes; not worth it.
+
 use super::*;
 use crate::merkleize::{merkleize, pad_for_leaf_count};
 use int_to_bytes::int_to_bytes32;

From f20314bd8710a3c2fb90d78b3d0cc2473fd0ea9f Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Mon, 29 Apr 2019 15:32:41 +1000
Subject: [PATCH 133/137] Fix clippy lints, impl treehash for slices

---
 eth2/utils/cached_tree_hash/Cargo.toml        |  3 -
 .../examples/8k_hashes_standard.rs            |  1 -
 eth2/utils/cached_tree_hash/src/impls/vec.rs  | 69 ++++++++++---------
 eth2/utils/tree_hash/src/impls.rs             | 45 +++++++-----
 4 files changed, 64 insertions(+), 54 deletions(-)

diff --git a/eth2/utils/cached_tree_hash/Cargo.toml b/eth2/utils/cached_tree_hash/Cargo.toml
index c33f20add..7b331ad68 100644
--- a/eth2/utils/cached_tree_hash/Cargo.toml
+++ b/eth2/utils/cached_tree_hash/Cargo.toml
@@ -4,9 +4,6 @@ version = "0.1.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 
-[profile.release]
-debug = true
-
 [[bench]]
 name = "benches"
 harness = false
diff --git a/eth2/utils/cached_tree_hash/examples/8k_hashes_standard.rs b/eth2/utils/cached_tree_hash/examples/8k_hashes_standard.rs
index e5df7ea95..bcbb392e2 100644
--- a/eth2/utils/cached_tree_hash/examples/8k_hashes_standard.rs
+++ b/eth2/utils/cached_tree_hash/examples/8k_hashes_standard.rs
@@ -1,4 +1,3 @@
-use cached_tree_hash::TreeHashCache;
 use ethereum_types::H256 as Hash256;
 use tree_hash::TreeHash;
 
diff --git a/eth2/utils/cached_tree_hash/src/impls/vec.rs b/eth2/utils/cached_tree_hash/src/impls/vec.rs
index b04884636..89f8b85db 100644
--- a/eth2/utils/cached_tree_hash/src/impls/vec.rs
+++ b/eth2/utils/cached_tree_hash/src/impls/vec.rs
@@ -2,46 +2,53 @@ use super::*;
 use crate::btree_overlay::LeafNode;
 use crate::merkleize::{merkleize, num_sanitized_leaves, sanitise_bytes};
 
-impl<T> CachedTreeHash for Vec<T>
-where
-    T: CachedTreeHash + TreeHash,
-{
-    fn new_tree_hash_cache(&self, depth: usize) -> Result<TreeHashCache, Error> {
-        let (mut cache, schema) = new_tree_hash_cache(self, depth)?;
+macro_rules! impl_for_list {
+    ($type: ty) => {
+        impl<T> CachedTreeHash for $type
+        where
+            T: CachedTreeHash + TreeHash,
+        {
+            fn new_tree_hash_cache(&self, depth: usize) -> Result<TreeHashCache, Error> {
+                let (mut cache, schema) = new_tree_hash_cache(self, depth)?;
 
-        cache.add_length_nodes(schema.into_overlay(0).chunk_range(), self.len())?;
+                cache.add_length_nodes(schema.into_overlay(0).chunk_range(), self.len())?;
 
-        Ok(cache)
-    }
+                Ok(cache)
+            }
 
-    fn num_tree_hash_cache_chunks(&self) -> usize {
-        // Add two extra nodes to cater for the node before and after to allow mixing-in length.
-        BTreeOverlay::new(self, 0, 0).num_chunks() + 2
-    }
+            fn num_tree_hash_cache_chunks(&self) -> usize {
+                // Add two extra nodes to cater for the node before and after to allow mixing-in length.
+                BTreeOverlay::new(self, 0, 0).num_chunks() + 2
+            }
 
-    fn tree_hash_cache_schema(&self, depth: usize) -> BTreeSchema {
-        produce_schema(self, depth)
-    }
+            fn tree_hash_cache_schema(&self, depth: usize) -> BTreeSchema {
+                produce_schema(self, depth)
+            }
 
-    fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
-        // Skip the length-mixed-in root node.
-        cache.chunk_index += 1;
+            fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
+                // Skip the length-mixed-in root node.
+                cache.chunk_index += 1;
 
-        // Update the cache, returning the new overlay.
-        let new_overlay = update_tree_hash_cache(self, cache)?;
+                // Update the cache, returning the new overlay.
+                let new_overlay = update_tree_hash_cache(&self, cache)?;
 
-        // Mix in length
-        cache.mix_in_length(new_overlay.chunk_range(), self.len())?;
+                // Mix in length
+                cache.mix_in_length(new_overlay.chunk_range(), self.len())?;
 
-        // Skip an extra node to clear the length node.
-        cache.chunk_index += 1;
+                // Skip an extra node to clear the length node.
+                cache.chunk_index += 1;
 
-        Ok(())
-    }
+                Ok(())
+            }
+        }
+    };
 }
 
+impl_for_list!(Vec<T>);
+impl_for_list!(&[T]);
+
 pub fn new_tree_hash_cache<T: CachedTreeHash>(
-    vec: &Vec<T>,
+    vec: &[T],
     depth: usize,
 ) -> Result<(TreeHashCache, BTreeSchema), Error> {
     let schema = vec.tree_hash_cache_schema(depth);
@@ -58,7 +65,7 @@ pub fn new_tree_hash_cache<T: CachedTreeHash>(
                 .map(|item| TreeHashCache::new(item, depth + 1))
                 .collect::<Result<Vec<TreeHashCache>, _>>()?;
 
-            TreeHashCache::from_leaves_and_subtrees(vec, subtrees, depth)
+            TreeHashCache::from_leaves_and_subtrees(&vec, subtrees, depth)
         }
     }?;
 
@@ -91,11 +98,11 @@ pub fn produce_schema<T: CachedTreeHash>(vec: &[T], depth: usize) -> BTreeSchema
 
 #[allow(clippy::range_plus_one)] // Minor readability lint requiring structural changes; not worth it.
 pub fn update_tree_hash_cache<T: CachedTreeHash>(
-    vec: &Vec<T>,
+    vec: &[T],
     cache: &mut TreeHashCache,
 ) -> Result<BTreeOverlay, Error> {
     let old_overlay = cache.get_overlay(cache.schema_index, cache.chunk_index)?;
-    let new_overlay = BTreeOverlay::new(vec, cache.chunk_index, old_overlay.depth);
+    let new_overlay = BTreeOverlay::new(&vec, cache.chunk_index, old_overlay.depth);
 
     cache.replace_overlay(cache.schema_index, cache.chunk_index, new_overlay.clone())?;
 
diff --git a/eth2/utils/tree_hash/src/impls.rs b/eth2/utils/tree_hash/src/impls.rs
index 56eb7dbf4..42ea9add0 100644
--- a/eth2/utils/tree_hash/src/impls.rs
+++ b/eth2/utils/tree_hash/src/impls.rs
@@ -87,31 +87,38 @@ impl TreeHash for H256 {
     }
 }
 
-impl<T> TreeHash for Vec<T>
-where
-    T: TreeHash,
-{
-    fn tree_hash_type() -> TreeHashType {
-        TreeHashType::List
-    }
+macro_rules! impl_for_list {
+    ($type: ty) => {
+        impl<T> TreeHash for $type
+        where
+            T: TreeHash,
+        {
+            fn tree_hash_type() -> TreeHashType {
+                TreeHashType::List
+            }
 
-    fn tree_hash_packed_encoding(&self) -> Vec<u8> {
-        unreachable!("List should never be packed.")
-    }
+            fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+                unreachable!("List should never be packed.")
+            }
 
-    fn tree_hash_packing_factor() -> usize {
-        unreachable!("List should never be packed.")
-    }
+            fn tree_hash_packing_factor() -> usize {
+                unreachable!("List should never be packed.")
+            }
 
-    fn tree_hash_root(&self) -> Vec<u8> {
-        let mut root_and_len = Vec::with_capacity(HASHSIZE * 2);
-        root_and_len.append(&mut vec_tree_hash_root(self));
-        root_and_len.append(&mut int_to_bytes32(self.len() as u64));
+            fn tree_hash_root(&self) -> Vec<u8> {
+                let mut root_and_len = Vec::with_capacity(HASHSIZE * 2);
+                root_and_len.append(&mut vec_tree_hash_root(self));
+                root_and_len.append(&mut int_to_bytes32(self.len() as u64));
 
-        hash(&root_and_len)
-    }
+                hash(&root_and_len)
+            }
+        }
+    };
 }
 
+impl_for_list!(Vec<T>);
+impl_for_list!(&[T]);
+
 pub fn vec_tree_hash_root<T>(vec: &[T]) -> Vec<u8>
 where
     T: TreeHash,

From f622aa0b6547237c80280e40981f8fb5597d7108 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Mon, 29 Apr 2019 15:45:02 +1000
Subject: [PATCH 134/137] Add doc comments to vec impl

---
 eth2/utils/cached_tree_hash/src/impls/vec.rs | 21 +++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/eth2/utils/cached_tree_hash/src/impls/vec.rs b/eth2/utils/cached_tree_hash/src/impls/vec.rs
index 89f8b85db..a4ecee3f3 100644
--- a/eth2/utils/cached_tree_hash/src/impls/vec.rs
+++ b/eth2/utils/cached_tree_hash/src/impls/vec.rs
@@ -47,6 +47,10 @@ macro_rules! impl_for_list {
 impl_for_list!(Vec<T>);
 impl_for_list!(&[T]);
 
+/// Build a new tree hash cache for some slice.
+///
+/// Valid for both variable- and fixed-length slices. Does _not_ mix-in the length of the list,
+/// the caller must do this.
 pub fn new_tree_hash_cache<T: CachedTreeHash>(
     vec: &[T],
     depth: usize,
@@ -72,6 +76,10 @@ pub fn new_tree_hash_cache<T: CachedTreeHash>(
     Ok((cache, schema))
 }
 
+/// Produce a schema for some slice.
+///
+/// Valid for both variable- and fixed-length slices. Does _not_ add the mix-in length nodes, the
+/// caller must do this.
 pub fn produce_schema<T: CachedTreeHash>(vec: &[T], depth: usize) -> BTreeSchema {
     let lengths = match T::tree_hash_type() {
         TreeHashType::Basic => {
@@ -96,6 +104,10 @@ pub fn produce_schema<T: CachedTreeHash>(vec: &[T], depth: usize) -> BTreeSchema
     BTreeSchema::from_lengths(depth, lengths)
 }
 
+/// Updates the cache for some slice.
+///
+/// Valid for both variable- and fixed-length slices. Does _not_ cater for the mix-in length nodes,
+/// the caller must do this.
 #[allow(clippy::range_plus_one)] // Minor readability lint requiring structural changes; not worth it.
 pub fn update_tree_hash_cache<T: CachedTreeHash>(
     vec: &[T],
@@ -274,6 +286,12 @@ pub fn update_tree_hash_cache<T: CachedTreeHash>(
     Ok(new_overlay)
 }
 
+/// Create a new `TreeHashCache` from `item` and splice it over the `chunks_to_replace` chunks of
+/// the given `cache`.
+///
+/// Useful for the case where a new element is added to a list.
+///
+/// The schemas created for `item` will have the given `depth`.
 fn splice_in_new_tree<T>(
     item: &T,
     chunks_to_replace: Range<usize>,
@@ -299,9 +317,10 @@ where
     cache.schema_index += num_schemas;
 
     Ok(())
-    //
 }
 
+/// Packs all of the leaves of `vec` into a single byte-array, appending `0` to ensure the number
+/// of chunks in the byte-array is a power-of-two.
 fn get_packed_leaves<T>(vec: &[T]) -> Result<Vec<u8>, Error>
 where
     T: CachedTreeHash,

From 84d72cfed6bd3e1fe01381894ad217a68734f5f9 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Mon, 29 Apr 2019 17:46:01 +1000
Subject: [PATCH 135/137] Tidy and add docs for cached tree hash

---
 eth2/types/src/beacon_state.rs                |   4 +-
 eth2/types/src/test_utils/macros.rs           |   4 +-
 eth2/types/src/tree_hash_vector.rs            |  20 ++-
 eth2/utils/bls/src/public_key.rs              |  14 +-
 eth2/utils/bls/src/signature.rs               |  14 +-
 eth2/utils/boolean-bitfield/src/lib.rs        |  14 +-
 .../examples/8k_hashes_cached.rs              |   2 +-
 .../cached_tree_hash/src/btree_overlay.rs     |  53 ++++++-
 eth2/utils/cached_tree_hash/src/impls/vec.rs  |   6 +-
 eth2/utils/cached_tree_hash/src/lib.rs        |  88 +++++-----
 eth2/utils/cached_tree_hash/src/merkleize.rs  |  15 +-
 .../cached_tree_hash/src/tree_hash_cache.rs   | 150 ++++++++++++------
 eth2/utils/cached_tree_hash/tests/tests.rs    |  55 +++----
 eth2/utils/tree_hash_derive/src/lib.rs        |   2 +-
 eth2/utils/tree_hash_derive/tests/tests.rs    |  10 +-
 15 files changed, 292 insertions(+), 159 deletions(-)

diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs
index 6948997c5..e9b052f99 100644
--- a/eth2/types/src/beacon_state.rs
+++ b/eth2/types/src/beacon_state.rs
@@ -806,7 +806,7 @@ impl BeaconState {
     /// canonical root of `self`.
     pub fn update_tree_hash_cache(&mut self) -> Result<Hash256, Error> {
         if self.tree_hash_cache.is_empty() {
-            self.tree_hash_cache = TreeHashCache::new(self, 0)?;
+            self.tree_hash_cache = TreeHashCache::new(self)?;
         } else {
             // Move the cache outside of `self` to satisfy the borrow checker.
             let mut cache = std::mem::replace(&mut self.tree_hash_cache, TreeHashCache::default());
@@ -828,7 +828,7 @@ impl BeaconState {
     /// cache update.
     pub fn cached_tree_hash_root(&self) -> Result<Hash256, Error> {
         self.tree_hash_cache
-            .root()
+            .tree_hash_root()
             .and_then(|b| Ok(Hash256::from_slice(b)))
             .map_err(|e| e.into())
     }
diff --git a/eth2/types/src/test_utils/macros.rs b/eth2/types/src/test_utils/macros.rs
index d6739ca0b..71f462c1a 100644
--- a/eth2/types/src/test_utils/macros.rs
+++ b/eth2/types/src/test_utils/macros.rs
@@ -46,7 +46,7 @@ macro_rules! cached_tree_hash_tests {
 
             // Test the original hash
             let original = $type::random_for_test(&mut rng);
-            let mut cache = cached_tree_hash::TreeHashCache::new(&original, 0).unwrap();
+            let mut cache = cached_tree_hash::TreeHashCache::new(&original).unwrap();
 
             assert_eq!(
                 cache.tree_hash_root().unwrap().to_vec(),
@@ -64,7 +64,7 @@ macro_rules! cached_tree_hash_tests {
             );
 
             // Produce a new cache for the modified object and compare it to the updated cache.
-            let mut modified_cache = cached_tree_hash::TreeHashCache::new(&modified, 0).unwrap();
+            let mut modified_cache = cached_tree_hash::TreeHashCache::new(&modified).unwrap();
 
             // Reset the caches.
             cache.reset_modifications();
diff --git a/eth2/types/src/tree_hash_vector.rs b/eth2/types/src/tree_hash_vector.rs
index 8a7d99a6c..42a730f25 100644
--- a/eth2/types/src/tree_hash_vector.rs
+++ b/eth2/types/src/tree_hash_vector.rs
@@ -63,20 +63,20 @@ where
         &self,
         depth: usize,
     ) -> Result<cached_tree_hash::TreeHashCache, cached_tree_hash::Error> {
-        let (cache, _overlay) = cached_tree_hash::impls::vec::new_tree_hash_cache(self, depth)?;
+        let (cache, _overlay) = cached_tree_hash::vec::new_tree_hash_cache(self, depth)?;
 
         Ok(cache)
     }
 
     fn tree_hash_cache_schema(&self, depth: usize) -> cached_tree_hash::BTreeSchema {
-        cached_tree_hash::impls::vec::produce_schema(self, depth)
+        cached_tree_hash::vec::produce_schema(self, depth)
     }
 
     fn update_tree_hash_cache(
         &self,
         cache: &mut cached_tree_hash::TreeHashCache,
     ) -> Result<(), cached_tree_hash::Error> {
-        cached_tree_hash::impls::vec::update_tree_hash_cache(self, cache)?;
+        cached_tree_hash::vec::update_tree_hash_cache(self, cache)?;
 
         Ok(())
     }
@@ -122,15 +122,21 @@ mod test {
     pub fn test_cached_tree_hash() {
         let original = TreeHashVector::from(vec![1_u64, 2, 3, 4]);
 
-        let mut hasher = cached_tree_hash::CachedTreeHasher::new(&original).unwrap();
+        let mut cache = cached_tree_hash::TreeHashCache::new(&original).unwrap();
 
-        assert_eq!(hasher.tree_hash_root().unwrap(), original.tree_hash_root());
+        assert_eq!(
+            cache.tree_hash_root().unwrap().to_vec(),
+            original.tree_hash_root()
+        );
 
         let modified = TreeHashVector::from(vec![1_u64, 1, 1, 1]);
 
-        hasher.update(&modified).unwrap();
+        cache.update(&modified).unwrap();
 
-        assert_eq!(hasher.tree_hash_root().unwrap(), modified.tree_hash_root());
+        assert_eq!(
+            cache.tree_hash_root().unwrap().to_vec(),
+            modified.tree_hash_root()
+        );
     }
 
 }
diff --git a/eth2/utils/bls/src/public_key.rs b/eth2/utils/bls/src/public_key.rs
index e6e4d3508..41b87d383 100644
--- a/eth2/utils/bls/src/public_key.rs
+++ b/eth2/utils/bls/src/public_key.rs
@@ -149,15 +149,21 @@ mod tests {
         let sk = SecretKey::random();
         let original = PublicKey::from_secret_key(&sk);
 
-        let mut hasher = cached_tree_hash::CachedTreeHasher::new(&original).unwrap();
+        let mut cache = cached_tree_hash::TreeHashCache::new(&original).unwrap();
 
-        assert_eq!(hasher.tree_hash_root().unwrap(), original.tree_hash_root());
+        assert_eq!(
+            cache.tree_hash_root().unwrap().to_vec(),
+            original.tree_hash_root()
+        );
 
         let sk = SecretKey::random();
         let modified = PublicKey::from_secret_key(&sk);
 
-        hasher.update(&modified).unwrap();
+        cache.update(&modified).unwrap();
 
-        assert_eq!(hasher.tree_hash_root().unwrap(), modified.tree_hash_root());
+        assert_eq!(
+            cache.tree_hash_root().unwrap().to_vec(),
+            modified.tree_hash_root()
+        );
     }
 }
diff --git a/eth2/utils/bls/src/signature.rs b/eth2/utils/bls/src/signature.rs
index 75f43aaf4..e2dbd9c27 100644
--- a/eth2/utils/bls/src/signature.rs
+++ b/eth2/utils/bls/src/signature.rs
@@ -166,15 +166,21 @@ mod tests {
         let keypair = Keypair::random();
         let original = Signature::new(&[42, 42], 0, &keypair.sk);
 
-        let mut hasher = cached_tree_hash::CachedTreeHasher::new(&original).unwrap();
+        let mut cache = cached_tree_hash::TreeHashCache::new(&original).unwrap();
 
-        assert_eq!(hasher.tree_hash_root().unwrap(), original.tree_hash_root());
+        assert_eq!(
+            cache.tree_hash_root().unwrap().to_vec(),
+            original.tree_hash_root()
+        );
 
         let modified = Signature::new(&[99, 99], 0, &keypair.sk);
 
-        hasher.update(&modified).unwrap();
+        cache.update(&modified).unwrap();
 
-        assert_eq!(hasher.tree_hash_root().unwrap(), modified.tree_hash_root());
+        assert_eq!(
+            cache.tree_hash_root().unwrap().to_vec(),
+            modified.tree_hash_root()
+        );
     }
 
     #[test]
diff --git a/eth2/utils/boolean-bitfield/src/lib.rs b/eth2/utils/boolean-bitfield/src/lib.rs
index c5fa590d6..d49da0d10 100644
--- a/eth2/utils/boolean-bitfield/src/lib.rs
+++ b/eth2/utils/boolean-bitfield/src/lib.rs
@@ -284,15 +284,21 @@ mod tests {
     pub fn test_cached_tree_hash() {
         let original = BooleanBitfield::from_bytes(&vec![18; 12][..]);
 
-        let mut hasher = cached_tree_hash::CachedTreeHasher::new(&original).unwrap();
+        let mut cache = cached_tree_hash::TreeHashCache::new(&original).unwrap();
 
-        assert_eq!(hasher.tree_hash_root().unwrap(), original.tree_hash_root());
+        assert_eq!(
+            cache.tree_hash_root().unwrap().to_vec(),
+            original.tree_hash_root()
+        );
 
         let modified = BooleanBitfield::from_bytes(&vec![2; 1][..]);
 
-        hasher.update(&modified).unwrap();
+        cache.update(&modified).unwrap();
 
-        assert_eq!(hasher.tree_hash_root().unwrap(), modified.tree_hash_root());
+        assert_eq!(
+            cache.tree_hash_root().unwrap().to_vec(),
+            modified.tree_hash_root()
+        );
     }
 
     #[test]
diff --git a/eth2/utils/cached_tree_hash/examples/8k_hashes_cached.rs b/eth2/utils/cached_tree_hash/examples/8k_hashes_cached.rs
index cb8dc42fb..1e67571d5 100644
--- a/eth2/utils/cached_tree_hash/examples/8k_hashes_cached.rs
+++ b/eth2/utils/cached_tree_hash/examples/8k_hashes_cached.rs
@@ -2,7 +2,7 @@ use cached_tree_hash::TreeHashCache;
 use ethereum_types::H256 as Hash256;
 
 fn run(vec: &Vec<Hash256>, modified_vec: &Vec<Hash256>) {
-    let mut cache = TreeHashCache::new(vec, 0).unwrap();
+    let mut cache = TreeHashCache::new(vec).unwrap();
 
     cache.update(modified_vec).unwrap();
 }
diff --git a/eth2/utils/cached_tree_hash/src/btree_overlay.rs b/eth2/utils/cached_tree_hash/src/btree_overlay.rs
index 4b3c6cc27..a96df769c 100644
--- a/eth2/utils/cached_tree_hash/src/btree_overlay.rs
+++ b/eth2/utils/cached_tree_hash/src/btree_overlay.rs
@@ -1,7 +1,19 @@
 use super::*;
 
+/// A schema defining a binary tree over a `TreeHashCache`.
+///
+/// This structure is used for succinct storage, run-time functionality is gained by converting the
+/// schema into a `BTreeOverlay`.
 #[derive(Debug, PartialEq, Clone)]
 pub struct BTreeSchema {
+    /// The depth of a schema defines how far it is nested within other fixed-length items.
+    ///
+    /// Each time a new variable-length object is created all items within it are assigned a depth
+    /// of `depth + 1`.
+    ///
+    /// When storing the schemas in a list, the depth parameter allows for removing all schemas
+    /// belonging to a specific variable-length item without removing schemas related to adjacent
+    /// variable-length items.
     pub depth: usize,
     lengths: Vec<usize>,
 }
@@ -25,21 +37,35 @@ impl Into<BTreeSchema> for BTreeOverlay {
     }
 }
 
+/// Provides a status for some leaf-node in binary tree.
 #[derive(Debug, PartialEq, Clone)]
 pub enum LeafNode {
+    /// The leaf node does not exist in this tree.
     DoesNotExist,
+    /// The leaf node exists in the tree and has a real value within the given `chunk` range.
     Exists(Range<usize>),
+    /// The leaf node exists in the tree only as padding.
     Padding,
 }
 
+/// Instantiated from a `BTreeSchema`, allows for interpreting some chunks of a `TreeHashCache` as
+/// a perfect binary tree.
+///
+/// The primary purpose of this struct is to map from binary tree "nodes" to `TreeHashCache`
+/// "chunks". Each tree has nodes `0..n` where `n` is the number of nodes and `0` is the root node.
+/// Each of these nodes is mapped to a chunk, starting from `self.offset` and increasing in steps
+/// of `1` for internal nodes and arbitrary steps for leaf-nodes.
 #[derive(Debug, PartialEq, Clone)]
 pub struct BTreeOverlay {
     offset: usize,
+    /// See `BTreeSchema.depth` for a description.
     pub depth: usize,
     lengths: Vec<usize>,
 }
 
 impl BTreeOverlay {
+    /// Instantiates a new instance for `item`, where it's first chunk is `inital_offset` and has
+    /// the specified `depth`.
     pub fn new<T>(item: &T, initial_offset: usize, depth: usize) -> Self
     where
         T: CachedTreeHash,
@@ -47,6 +73,7 @@ impl BTreeOverlay {
         Self::from_schema(item.tree_hash_cache_schema(depth), initial_offset)
     }
 
+    /// Instantiates a new instance from a schema, where it's first chunk is `offset`.
     pub fn from_schema(schema: BTreeSchema, offset: usize) -> Self {
         Self {
             offset,
@@ -55,6 +82,10 @@ impl BTreeOverlay {
         }
     }
 
+    /// Returns a `LeafNode` for each of the `n` leaves of the tree.
+    ///
+    /// `LeafNode::DoesNotExist` is returned for each element `i` in `0..n` where `i >=
+    /// self.num_leaf_nodes()`.
     pub fn get_leaf_nodes(&self, n: usize) -> Vec<LeafNode> {
         let mut running_offset = self.offset + self.num_internal_nodes();
 
@@ -74,10 +105,12 @@ impl BTreeOverlay {
         leaf_nodes
     }
 
+    /// Returns the number of leaf nodes in the tree.
     pub fn num_leaf_nodes(&self) -> usize {
         self.lengths.len().next_power_of_two()
     }
 
+    /// Returns the number of leafs in the tree which are padding.
     pub fn num_padding_leaves(&self) -> usize {
         self.num_leaf_nodes() - self.lengths.len()
     }
@@ -90,31 +123,39 @@ impl BTreeOverlay {
         2 * self.num_leaf_nodes() - 1
     }
 
+    /// Returns the number of internal (non-leaf) nodes in the tree.
     pub fn num_internal_nodes(&self) -> usize {
         self.num_leaf_nodes() - 1
     }
 
+    /// Returns the chunk of the first node of the tree.
     fn first_node(&self) -> usize {
         self.offset
     }
 
+    /// Returns the root chunk of the tree (the zero-th node)
     pub fn root(&self) -> usize {
         self.first_node()
     }
 
+    /// Returns the first chunk outside of the boundary of this tree. It is the root node chunk
+    /// plus the total number of chunks in the tree.
     pub fn next_node(&self) -> usize {
         self.first_node() + self.num_internal_nodes() + self.num_leaf_nodes() - self.lengths.len()
             + self.lengths.iter().sum::<usize>()
     }
 
+    /// Returns the height of the tree where a tree with a single node has a height of 1.
     pub fn height(&self) -> usize {
         self.num_leaf_nodes().trailing_zeros() as usize
     }
 
+    /// Returns the range of chunks that belong to the internal nodes of the tree.
     pub fn internal_chunk_range(&self) -> Range<usize> {
         self.offset..self.offset + self.num_internal_nodes()
     }
 
+    /// Returns all of the chunks that are encompassed by the tree.
     pub fn chunk_range(&self) -> Range<usize> {
         self.first_node()..self.next_node()
     }
@@ -127,10 +168,14 @@ impl BTreeOverlay {
         self.next_node() - self.first_node()
     }
 
+    /// Returns the first chunk of the first leaf node in the tree.
     pub fn first_leaf_node(&self) -> usize {
         self.offset + self.num_internal_nodes()
     }
 
+    /// Returns the chunks for some given parent node.
+    ///
+    /// Note: it is a parent _node_ not a parent _chunk_.
     pub fn child_chunks(&self, parent: usize) -> (usize, usize) {
         let children = children(parent);
 
@@ -142,7 +187,7 @@ impl BTreeOverlay {
         }
     }
 
-    /// (parent, (left_child, right_child))
+    /// Returns a vec of (parent_chunk, (left_child_chunk, right_child_chunk)).
     pub fn internal_parents_and_children(&self) -> Vec<(usize, (usize, usize))> {
         let mut chunks = Vec::with_capacity(self.num_nodes());
         chunks.append(&mut self.internal_node_chunks());
@@ -156,17 +201,17 @@ impl BTreeOverlay {
             .collect()
     }
 
-    // Returns a `Vec` of chunk indices for each internal node of the tree.
+    /// Returns a vec of chunk indices for each internal node of the tree.
     pub fn internal_node_chunks(&self) -> Vec<usize> {
         (self.offset..self.offset + self.num_internal_nodes()).collect()
     }
 
-    // Returns a `Vec` of the first chunk index for each leaf node of the tree.
+    /// Returns a vec of the first chunk for each leaf node of the tree.
     pub fn leaf_node_chunks(&self) -> Vec<usize> {
         self.n_leaf_node_chunks(self.num_leaf_nodes())
     }
 
-    // Returns a `Vec` of the first chunk index for the first `n` leaf nodes of the tree.
+    /// Returns a vec of the first chunk index for the first `n` leaf nodes of the tree.
     fn n_leaf_node_chunks(&self, n: usize) -> Vec<usize> {
         let mut chunks = Vec::with_capacity(n);
 
diff --git a/eth2/utils/cached_tree_hash/src/impls/vec.rs b/eth2/utils/cached_tree_hash/src/impls/vec.rs
index a4ecee3f3..bdb7eb134 100644
--- a/eth2/utils/cached_tree_hash/src/impls/vec.rs
+++ b/eth2/utils/cached_tree_hash/src/impls/vec.rs
@@ -66,10 +66,10 @@ pub fn new_tree_hash_cache<T: CachedTreeHash>(
         TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
             let subtrees = vec
                 .iter()
-                .map(|item| TreeHashCache::new(item, depth + 1))
+                .map(|item| TreeHashCache::new_at_depth(item, depth + 1))
                 .collect::<Result<Vec<TreeHashCache>, _>>()?;
 
-            TreeHashCache::from_leaves_and_subtrees(&vec, subtrees, depth)
+            TreeHashCache::from_subtrees(&vec, subtrees, depth)
         }
     }?;
 
@@ -301,7 +301,7 @@ fn splice_in_new_tree<T>(
 where
     T: CachedTreeHash,
 {
-    let (bytes, mut bools, schemas) = TreeHashCache::new(item, depth)?.into_components();
+    let (bytes, mut bools, schemas) = TreeHashCache::new_at_depth(item, depth)?.into_components();
 
     // Record the number of schemas, this will be used later in the fn.
     let num_schemas = schemas.len();
diff --git a/eth2/utils/cached_tree_hash/src/lib.rs b/eth2/utils/cached_tree_hash/src/lib.rs
index b9bb8457b..21fa786e4 100644
--- a/eth2/utils/cached_tree_hash/src/lib.rs
+++ b/eth2/utils/cached_tree_hash/src/lib.rs
@@ -1,16 +1,52 @@
+//! Performs cached merkle-hashing adhering to the Ethereum 2.0 specification defined
+//! [here](https://github.com/ethereum/eth2.0-specs/blob/v0.5.1/specs/simple-serialize.md#merkleization).
+//!
+//! Caching allows for reduced hashing when some object has only been partially modified. This
+//! allows for significant CPU-time savings (at the cost of additional storage). For example,
+//! determining the root of a list of 1024 items with a single modification has been observed to
+//! run in 1/25th of the time of a full merkle hash.
+//!
+//!
+//! # Example:
+//!
+//! ```
+//! use cached_tree_hash::TreeHashCache;
+//! use tree_hash_derive::{TreeHash, CachedTreeHash};
+//!
+//! #[derive(TreeHash, CachedTreeHash)]
+//! struct Foo {
+//!     bar: u64,
+//!     baz: Vec<u64>
+//! }
+//!
+//! let mut foo = Foo {
+//!     bar: 1,
+//!     baz: vec![0, 1, 2]
+//! };
+//!
+//! let mut cache = TreeHashCache::new(&foo).unwrap();
+//!
+//! foo.baz[1] = 0;
+//!
+//! cache.update(&foo).unwrap();
+//!
+//! println!("Root is: {:?}", cache.tree_hash_root().unwrap());
+//! ```
+
 use hashing::hash;
 use std::ops::Range;
 use tree_hash::{TreeHash, TreeHashType, BYTES_PER_CHUNK, HASHSIZE};
 
 mod btree_overlay;
 mod errors;
-pub mod impls;
+mod impls;
 pub mod merkleize;
 mod resize;
 mod tree_hash_cache;
 
 pub use btree_overlay::{BTreeOverlay, BTreeSchema};
 pub use errors::Error;
+pub use impls::vec;
 pub use tree_hash_cache::TreeHashCache;
 
 pub trait CachedTreeHash: TreeHash {
@@ -25,34 +61,8 @@ pub trait CachedTreeHash: TreeHash {
     fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error>;
 }
 
-#[derive(Debug, PartialEq)]
-pub struct CachedTreeHasher {
-    pub cache: TreeHashCache,
-}
-
-impl CachedTreeHasher {
-    pub fn new<T>(item: &T) -> Result<Self, Error>
-    where
-        T: CachedTreeHash,
-    {
-        Ok(Self {
-            cache: TreeHashCache::new(item, 0)?,
-        })
-    }
-
-    pub fn update<T>(&mut self, item: &T) -> Result<(), Error>
-    where
-        T: CachedTreeHash,
-    {
-        self.cache.update(item)
-    }
-
-    pub fn tree_hash_root(&self) -> Result<Vec<u8>, Error> {
-        // Return the root of the cache -- the merkle root.
-        Ok(self.cache.root()?.to_vec())
-    }
-}
-
+/// Implements `CachedTreeHash` on `$type` as a fixed-length tree-hash vector of the ssz encoding
+/// of `$type`.
 #[macro_export]
 macro_rules! cached_tree_hash_ssz_encoding_as_vector {
     ($type: ident, $num_bytes: expr) => {
@@ -61,10 +71,8 @@ macro_rules! cached_tree_hash_ssz_encoding_as_vector {
                 &self,
                 depth: usize,
             ) -> Result<cached_tree_hash::TreeHashCache, cached_tree_hash::Error> {
-                let (cache, _schema) = cached_tree_hash::impls::vec::new_tree_hash_cache(
-                    &ssz::ssz_encode(self),
-                    depth,
-                )?;
+                let (cache, _schema) =
+                    cached_tree_hash::vec::new_tree_hash_cache(&ssz::ssz_encode(self), depth)?;
 
                 Ok(cache)
             }
@@ -79,10 +87,7 @@ macro_rules! cached_tree_hash_ssz_encoding_as_vector {
                 &self,
                 cache: &mut cached_tree_hash::TreeHashCache,
             ) -> Result<(), cached_tree_hash::Error> {
-                cached_tree_hash::impls::vec::update_tree_hash_cache(
-                    &ssz::ssz_encode(self),
-                    cache,
-                )?;
+                cached_tree_hash::vec::update_tree_hash_cache(&ssz::ssz_encode(self), cache)?;
 
                 Ok(())
             }
@@ -90,6 +95,8 @@ macro_rules! cached_tree_hash_ssz_encoding_as_vector {
     };
 }
 
+/// Implements `CachedTreeHash` on `$type` as a variable-length tree-hash list of the result of
+/// calling `.as_bytes()` on `$type`.
 #[macro_export]
 macro_rules! cached_tree_hash_bytes_as_list {
     ($type: ident) => {
@@ -101,7 +108,7 @@ macro_rules! cached_tree_hash_bytes_as_list {
                 let bytes = self.to_bytes();
 
                 let (mut cache, schema) =
-                    cached_tree_hash::impls::vec::new_tree_hash_cache(&bytes, depth)?;
+                    cached_tree_hash::vec::new_tree_hash_cache(&bytes, depth)?;
 
                 cache.add_length_nodes(schema.into_overlay(0).chunk_range(), bytes.len())?;
 
@@ -115,7 +122,7 @@ macro_rules! cached_tree_hash_bytes_as_list {
 
             fn tree_hash_cache_schema(&self, depth: usize) -> cached_tree_hash::BTreeSchema {
                 let bytes = self.to_bytes();
-                cached_tree_hash::impls::vec::produce_schema(&bytes, depth)
+                cached_tree_hash::vec::produce_schema(&bytes, depth)
             }
 
             fn update_tree_hash_cache(
@@ -128,8 +135,7 @@ macro_rules! cached_tree_hash_bytes_as_list {
                 cache.chunk_index += 1;
 
                 // Update the cache, returning the new overlay.
-                let new_overlay =
-                    cached_tree_hash::impls::vec::update_tree_hash_cache(&bytes, cache)?;
+                let new_overlay = cached_tree_hash::vec::update_tree_hash_cache(&bytes, cache)?;
 
                 // Mix in length
                 cache.mix_in_length(new_overlay.chunk_range(), bytes.len())?;
diff --git a/eth2/utils/cached_tree_hash/src/merkleize.rs b/eth2/utils/cached_tree_hash/src/merkleize.rs
index e744961b0..9d8c83200 100644
--- a/eth2/utils/cached_tree_hash/src/merkleize.rs
+++ b/eth2/utils/cached_tree_hash/src/merkleize.rs
@@ -35,6 +35,7 @@ pub fn merkleize(values: Vec<u8>) -> Vec<u8> {
     o
 }
 
+/// Ensures that the given `bytes` are a power-of-two chunks, padding with zero if not.
 pub fn sanitise_bytes(mut bytes: Vec<u8>) -> Vec<u8> {
     let present_leaves = num_unsanitized_leaves(bytes.len());
     let required_leaves = present_leaves.next_power_of_two();
@@ -46,6 +47,7 @@ pub fn sanitise_bytes(mut bytes: Vec<u8>) -> Vec<u8> {
     bytes
 }
 
+/// Pads out `bytes` to ensure it is a clean `num_leaves` chunks.
 pub fn pad_for_leaf_count(num_leaves: usize, bytes: &mut Vec<u8>) {
     let required_leaves = num_leaves.next_power_of_two();
 
@@ -59,9 +61,10 @@ fn last_leaf_needs_padding(num_bytes: usize) -> bool {
     num_bytes % HASHSIZE != 0
 }
 
-/// Rounds up
-pub fn num_unsanitized_leaves(num_bytes: usize) -> usize {
-    (num_bytes + HASHSIZE - 1) / HASHSIZE
+/// Returns the number of leaves for a given `bytes_len` number of bytes, rounding up if
+/// `num_bytes` is not a client multiple of chunk size.
+pub fn num_unsanitized_leaves(bytes_len: usize) -> usize {
+    (bytes_len + HASHSIZE - 1) / HASHSIZE
 }
 
 fn num_bytes(num_leaves: usize) -> usize {
@@ -72,7 +75,9 @@ fn num_nodes(num_leaves: usize) -> usize {
     2 * num_leaves - 1
 }
 
-pub fn num_sanitized_leaves(num_bytes: usize) -> usize {
-    let leaves = (num_bytes + HASHSIZE - 1) / HASHSIZE;
+/// Returns the power-of-two number of leaves that would result from the given `bytes_len` number
+/// of bytes.
+pub fn num_sanitized_leaves(bytes_len: usize) -> usize {
+    let leaves = (bytes_len + HASHSIZE - 1) / HASHSIZE;
     leaves.next_power_of_two()
 }
diff --git a/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
index acb96ba24..8f7b9de86 100644
--- a/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
+++ b/eth2/utils/cached_tree_hash/src/tree_hash_cache.rs
@@ -4,20 +4,35 @@ use super::*;
 use crate::merkleize::{merkleize, pad_for_leaf_count};
 use int_to_bytes::int_to_bytes32;
 
+/// Provides cached tree hashing for some object implementing `CachedTreeHash`.
+///
+/// Caching allows for doing minimal internal-node hashing when an object has only been partially
+/// changed.
+///
+/// See the crate root for an example.
 #[derive(Debug, PartialEq, Clone)]
 pub struct TreeHashCache {
-    pub cache: Vec<u8>,
+    /// Stores the binary-tree in 32-byte chunks.
+    pub bytes: Vec<u8>,
+    /// Maps to each chunk of `self.bytes`, indicating if the chunk is dirty.
     pub chunk_modified: Vec<bool>,
+    /// Contains a schema for each variable-length item stored in the cache.
     pub schemas: Vec<BTreeSchema>,
 
+    /// A counter used during updates.
     pub chunk_index: usize,
+    /// A counter used during updates.
     pub schema_index: usize,
 }
 
 impl Default for TreeHashCache {
+    /// Create an empty cache.
+    ///
+    /// Note: an empty cache is effectively useless, an error will be raised if `self.update` is
+    /// called.
     fn default() -> TreeHashCache {
         TreeHashCache {
-            cache: vec![],
+            bytes: vec![],
             chunk_modified: vec![],
             schemas: vec![],
             chunk_index: 0,
@@ -26,20 +41,34 @@ impl Default for TreeHashCache {
     }
 }
 
-impl Into<Vec<u8>> for TreeHashCache {
-    fn into(self) -> Vec<u8> {
-        self.cache
-    }
-}
-
 impl TreeHashCache {
-    pub fn new<T>(item: &T, depth: usize) -> Result<Self, Error>
+    /// Instantiates a new cache from `item` at a depth of `0`.
+    ///
+    /// The returned cache is fully-built and will return an accurate tree-hash root.
+    pub fn new<T>(item: &T) -> Result<Self, Error>
+    where
+        T: CachedTreeHash,
+    {
+        Self::new_at_depth(item, 0)
+    }
+
+    /// Instantiates a new cache from `item` at the specified `depth`.
+    ///
+    /// The returned cache is fully-built and will return an accurate tree-hash root.
+    pub fn new_at_depth<T>(item: &T, depth: usize) -> Result<Self, Error>
     where
         T: CachedTreeHash,
     {
         item.new_tree_hash_cache(depth)
     }
 
+    /// Updates the cache with `item`.
+    ///
+    /// `item` _must_ be of the same type as the `item` used to build the cache, otherwise an error
+    /// may be returned.
+    ///
+    /// After calling `update`, the cache will return an accurate tree-hash root using
+    /// `self.tree_hash_root()`.
     pub fn update<T>(&mut self, item: &T) -> Result<(), Error>
     where
         T: CachedTreeHash,
@@ -53,11 +82,10 @@ impl TreeHashCache {
         }
     }
 
-    pub fn from_leaves_and_subtrees<T>(
-        item: &T,
-        leaves_and_subtrees: Vec<Self>,
-        depth: usize,
-    ) -> Result<Self, Error>
+    /// Builds a new cache for `item`, given `subtrees` contains a `Self` for field/item of `item`.
+    ///
+    /// Each `subtree` in `subtree` will become a leaf-node of the merkle-tree of `item`.
+    pub fn from_subtrees<T>(item: &T, subtrees: Vec<Self>, depth: usize) -> Result<Self, Error>
     where
         T: CachedTreeHash,
     {
@@ -65,20 +93,18 @@ impl TreeHashCache {
 
         // Note how many leaves were provided. If is not a power-of-two, we'll need to pad it out
         // later.
-        let num_provided_leaf_nodes = leaves_and_subtrees.len();
+        let num_provided_leaf_nodes = subtrees.len();
 
         // Allocate enough bytes to store the internal nodes and the leaves and subtrees, then fill
         // all the to-be-built internal nodes with zeros and append the leaves and subtrees.
         let internal_node_bytes = overlay.num_internal_nodes() * BYTES_PER_CHUNK;
-        let leaves_and_subtrees_bytes = leaves_and_subtrees
-            .iter()
-            .fold(0, |acc, t| acc + t.bytes_len());
-        let mut cache = Vec::with_capacity(leaves_and_subtrees_bytes + internal_node_bytes);
-        cache.resize(internal_node_bytes, 0);
+        let subtrees_bytes = subtrees.iter().fold(0, |acc, t| acc + t.bytes.len());
+        let mut bytes = Vec::with_capacity(subtrees_bytes + internal_node_bytes);
+        bytes.resize(internal_node_bytes, 0);
 
         // Allocate enough bytes to store all the leaves.
         let mut leaves = Vec::with_capacity(overlay.num_leaf_nodes() * HASHSIZE);
-        let mut schemas = Vec::with_capacity(leaves_and_subtrees.len());
+        let mut schemas = Vec::with_capacity(subtrees.len());
 
         if T::tree_hash_type() == TreeHashType::List {
             schemas.push(overlay.into());
@@ -86,32 +112,36 @@ impl TreeHashCache {
 
         // Iterate through all of the leaves/subtrees, adding their root as a leaf node and then
         // concatenating their merkle trees.
-        for t in leaves_and_subtrees {
-            leaves.append(&mut t.root()?.to_vec());
+        for t in subtrees {
+            leaves.append(&mut t.tree_hash_root()?.to_vec());
 
-            let (mut bytes, _bools, mut t_schemas) = t.into_components();
-            cache.append(&mut bytes);
+            let (mut t_bytes, _bools, mut t_schemas) = t.into_components();
+            bytes.append(&mut t_bytes);
             schemas.append(&mut t_schemas);
         }
 
         // Pad the leaves to an even power-of-two, using zeros.
-        pad_for_leaf_count(num_provided_leaf_nodes, &mut cache);
+        pad_for_leaf_count(num_provided_leaf_nodes, &mut bytes);
 
         // Merkleize the leaves, then split the leaf nodes off them. Then, replace all-zeros
         // internal nodes created earlier with the internal nodes generated by `merkleize`.
         let mut merkleized = merkleize(leaves);
         merkleized.split_off(internal_node_bytes);
-        cache.splice(0..internal_node_bytes, merkleized);
+        bytes.splice(0..internal_node_bytes, merkleized);
 
         Ok(Self {
-            chunk_modified: vec![true; cache.len() / BYTES_PER_CHUNK],
-            cache,
+            chunk_modified: vec![true; bytes.len() / BYTES_PER_CHUNK],
+            bytes,
             schemas,
             chunk_index: 0,
             schema_index: 0,
         })
     }
 
+    /// Instantiate a new cache from the pre-built `bytes` where each `self.chunk_modified` will be
+    /// set to `intitial_modified_state`.
+    ///
+    /// Note: `bytes.len()` must be a multiple of 32
     pub fn from_bytes(
         bytes: Vec<u8>,
         initial_modified_state: bool,
@@ -128,17 +158,22 @@ impl TreeHashCache {
 
         Ok(Self {
             chunk_modified: vec![initial_modified_state; bytes.len() / BYTES_PER_CHUNK],
-            cache: bytes,
+            bytes,
             schemas,
             chunk_index: 0,
             schema_index: 0,
         })
     }
 
+    /// Returns `true` if this cache is empty (i.e., it has never been built for some item).
+    ///
+    /// Note: an empty cache is effectively useless, an error will be raised if `self.update` is
+    /// called.
     pub fn is_empty(&self) -> bool {
         self.chunk_modified.is_empty()
     }
 
+    /// Return an overlay, built from the schema at `schema_index` with an offset of `chunk_index`.
     pub fn get_overlay(
         &self,
         schema_index: usize,
@@ -152,6 +187,9 @@ impl TreeHashCache {
             .into_overlay(chunk_index))
     }
 
+    /// Resets the per-update counters, allowing a new update to start.
+    ///
+    /// Note: this does _not_ delete the contents of the cache.
     pub fn reset_modifications(&mut self) {
         // Reset the per-hash counters.
         self.chunk_index = 0;
@@ -162,9 +200,14 @@ impl TreeHashCache {
         }
     }
 
+    /// Replace the schema at `schema_index` with the schema derived from `new_overlay`.
+    ///
+    /// If the `new_overlay` schema has a different number of internal nodes to the schema at
+    /// `schema_index`, the cache will be updated to add/remove these new internal nodes.
     pub fn replace_overlay(
         &mut self,
         schema_index: usize,
+        // TODO: remove chunk index (if possible)
         chunk_index: usize,
         new_overlay: BTreeOverlay,
     ) -> Result<BTreeOverlay, Error> {
@@ -225,6 +268,9 @@ impl TreeHashCache {
         Ok(old_schema.into_overlay(chunk_index))
     }
 
+    /// Remove all of the child schemas following `schema_index`.
+    ///
+    /// Schema `a` is a child of schema `b` if `a.depth < b.depth`.
     pub fn remove_proceeding_child_schemas(&mut self, schema_index: usize, depth: usize) {
         let end = self
             .schemas
@@ -237,6 +283,8 @@ impl TreeHashCache {
         self.schemas.splice(schema_index..end, vec![]);
     }
 
+    /// Iterate through the internal nodes chunks of `overlay`, updating the chunk with the
+    /// merkle-root of it's children if either of those children are dirty.
     pub fn update_internal_nodes(&mut self, overlay: &BTreeOverlay) -> Result<(), Error> {
         for (parent, children) in overlay.internal_parents_and_children().into_iter().rev() {
             if self.either_modified(children)? {
@@ -247,37 +295,34 @@ impl TreeHashCache {
         Ok(())
     }
 
-    fn bytes_len(&self) -> usize {
-        self.cache.len()
-    }
-
+    /// Returns to the tree-hash root of the cache.
     pub fn tree_hash_root(&self) -> Result<&[u8], Error> {
-        self.root()
-    }
-
-    pub fn root(&self) -> Result<&[u8], Error> {
         if self.is_empty() {
             Err(Error::CacheNotInitialized)
         } else {
-            self.cache
+            self.bytes
                 .get(0..HASHSIZE)
                 .ok_or_else(|| Error::NoBytesForRoot)
         }
     }
 
+    /// Splices the given `bytes` over `self.bytes` and `bools` over `self.chunk_modified` at the
+    /// specified `chunk_range`.
     pub fn splice(&mut self, chunk_range: Range<usize>, bytes: Vec<u8>, bools: Vec<bool>) {
         // Update the `chunk_modified` vec, marking all spliced-in nodes as changed.
         self.chunk_modified.splice(chunk_range.clone(), bools);
-        self.cache
+        self.bytes
             .splice(node_range_to_byte_range(&chunk_range), bytes);
     }
 
+    /// If the bytes at `chunk` are not the same as `to`, `self.bytes` is updated and
+    /// `self.chunk_modified` is set to `true`.
     pub fn maybe_update_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> {
         let start = chunk * BYTES_PER_CHUNK;
         let end = start + BYTES_PER_CHUNK;
 
         if !self.chunk_equals(chunk, to)? {
-            self.cache
+            self.bytes
                 .get_mut(start..end)
                 .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))?
                 .copy_from_slice(to);
@@ -287,18 +332,20 @@ impl TreeHashCache {
         Ok(())
     }
 
+    /// Returns the slices of `self.bytes` and `self.chunk_modified` at the given `chunk_range`.
     fn slices(&self, chunk_range: Range<usize>) -> Option<(&[u8], &[bool])> {
         Some((
-            self.cache.get(node_range_to_byte_range(&chunk_range))?,
+            self.bytes.get(node_range_to_byte_range(&chunk_range))?,
             self.chunk_modified.get(chunk_range)?,
         ))
     }
 
+    /// Updates `self.bytes` at `chunk` and sets `self.chunk_modified` for the `chunk` to `true`.
     pub fn modify_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> {
         let start = chunk * BYTES_PER_CHUNK;
         let end = start + BYTES_PER_CHUNK;
 
-        self.cache
+        self.bytes
             .get_mut(start..end)
             .ok_or_else(|| Error::NoBytesForChunk(chunk))?
             .copy_from_slice(to);
@@ -308,20 +355,23 @@ impl TreeHashCache {
         Ok(())
     }
 
+    /// Returns the bytes at `chunk`.
     fn get_chunk(&self, chunk: usize) -> Result<&[u8], Error> {
         let start = chunk * BYTES_PER_CHUNK;
         let end = start + BYTES_PER_CHUNK;
 
         Ok(self
-            .cache
+            .bytes
             .get(start..end)
             .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))?)
     }
 
+    /// Returns `true` if the bytes at `chunk` are equal to `other`.
     fn chunk_equals(&mut self, chunk: usize, other: &[u8]) -> Result<bool, Error> {
         Ok(self.get_chunk(chunk)? == other)
     }
 
+    /// Returns `true` if `chunk` is dirty.
     pub fn changed(&self, chunk: usize) -> Result<bool, Error> {
         self.chunk_modified
             .get(chunk)
@@ -329,10 +379,12 @@ impl TreeHashCache {
             .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))
     }
 
+    /// Returns `true` if either of the `children` chunks is dirty.
     fn either_modified(&self, children: (usize, usize)) -> Result<bool, Error> {
         Ok(self.changed(children.0)? | self.changed(children.1)?)
     }
 
+    /// Returns the hash of the concatenation of the given `children`.
     pub fn hash_children(&self, children: (usize, usize)) -> Result<Vec<u8>, Error> {
         let mut child_bytes = Vec::with_capacity(BYTES_PER_CHUNK * 2);
         child_bytes.append(&mut self.get_chunk(children.0)?.to_vec());
@@ -341,6 +393,7 @@ impl TreeHashCache {
         Ok(hash(&child_bytes))
     }
 
+    /// Adds a chunk before and after the given `chunk` range and calls `self.mix_in_length()`.
     pub fn add_length_nodes(
         &mut self,
         chunk_range: Range<usize>,
@@ -351,13 +404,13 @@ impl TreeHashCache {
         let byte_range = node_range_to_byte_range(&chunk_range);
 
         // Add the last node.
-        self.cache
+        self.bytes
             .splice(byte_range.end..byte_range.end, vec![0; HASHSIZE]);
         self.chunk_modified
             .splice(chunk_range.end..chunk_range.end, vec![false]);
 
         // Add the first node.
-        self.cache
+        self.bytes
             .splice(byte_range.start..byte_range.start, vec![0; HASHSIZE]);
         self.chunk_modified
             .splice(chunk_range.start..chunk_range.start, vec![false]);
@@ -367,6 +420,8 @@ impl TreeHashCache {
         Ok(())
     }
 
+    /// Sets `chunk_range.end + 1` equal to the little-endian serialization of `length`. Sets
+    /// `chunk_range.start - 1` equal to `self.hash_children(chunk_range.start, chunk_range.end + 1)`.
     pub fn mix_in_length(&mut self, chunk_range: Range<usize>, length: usize) -> Result<(), Error> {
         // Update the length chunk.
         self.maybe_update_chunk(chunk_range.end, &int_to_bytes32(length as u64))?;
@@ -380,8 +435,9 @@ impl TreeHashCache {
         Ok(())
     }
 
+    /// Returns `(self.bytes, self.chunk_modified, self.schemas)`.
     pub fn into_components(self) -> (Vec<u8>, Vec<bool>, Vec<BTreeSchema>) {
-        (self.cache, self.chunk_modified, self.schemas)
+        (self.bytes, self.chunk_modified, self.schemas)
     }
 }
 
diff --git a/eth2/utils/cached_tree_hash/tests/tests.rs b/eth2/utils/cached_tree_hash/tests/tests.rs
index 4b7a4e830..3e2598e2b 100644
--- a/eth2/utils/cached_tree_hash/tests/tests.rs
+++ b/eth2/utils/cached_tree_hash/tests/tests.rs
@@ -9,7 +9,7 @@ fn modifications() {
 
     let vec: Vec<Hash256> = (0..n).map(|_| Hash256::random()).collect();
 
-    let mut cache = TreeHashCache::new(&vec, 0).unwrap();
+    let mut cache = TreeHashCache::new(&vec).unwrap();
     cache.update(&vec).unwrap();
 
     let modifications = cache.chunk_modified.iter().filter(|b| **b).count();
@@ -36,60 +36,57 @@ fn test_routine<T>(original: T, modified: Vec<T>)
 where
     T: CachedTreeHash + std::fmt::Debug,
 {
-    let mut hasher = CachedTreeHasher::new(&original).unwrap();
+    let mut cache = TreeHashCache::new(&original).unwrap();
 
     let standard_root = original.tree_hash_root();
-    let cached_root = hasher.tree_hash_root().unwrap();
+    let cached_root = cache.tree_hash_root().unwrap();
     assert_eq!(standard_root, cached_root, "Initial cache build failed.");
 
     for (i, modified) in modified.iter().enumerate() {
         println!("-- Start of modification {} --", i);
 
         // Update the existing hasher.
-        hasher
+        cache
             .update(modified)
             .expect(&format!("Modification {}", i));
 
         // Create a new hasher from the "modified" struct.
-        let modified_hasher = CachedTreeHasher::new(modified).unwrap();
+        let modified_cache = TreeHashCache::new(modified).unwrap();
 
         assert_eq!(
-            hasher.cache.chunk_modified.len(),
-            modified_hasher.cache.chunk_modified.len(),
+            cache.chunk_modified.len(),
+            modified_cache.chunk_modified.len(),
             "Number of chunks is different"
         );
 
         assert_eq!(
-            hasher.cache.cache.len(),
-            modified_hasher.cache.cache.len(),
+            cache.bytes.len(),
+            modified_cache.bytes.len(),
             "Number of bytes is different"
         );
 
-        assert_eq!(
-            hasher.cache.cache, modified_hasher.cache.cache,
-            "Bytes are different"
-        );
+        assert_eq!(cache.bytes, modified_cache.bytes, "Bytes are different");
 
         assert_eq!(
-            hasher.cache.schemas.len(),
-            modified_hasher.cache.schemas.len(),
+            cache.schemas.len(),
+            modified_cache.schemas.len(),
             "Number of schemas is different"
         );
 
         assert_eq!(
-            hasher.cache.schemas, modified_hasher.cache.schemas,
+            cache.schemas, modified_cache.schemas,
             "Schemas are different"
         );
 
         // Test the root generated by the updated hasher matches a non-cached tree hash root.
         let standard_root = modified.tree_hash_root();
-        let cached_root = hasher
+        let cached_root = cache
             .tree_hash_root()
             .expect(&format!("Modification {}", i));
         assert_eq!(
             standard_root, cached_root,
             "Modification {} failed. \n Cache: {:?}",
-            i, hasher
+            i, cache
         );
     }
 }
@@ -194,20 +191,20 @@ fn test_shrinking_vec_of_vec() {
     let original: Vec<Vec<u64>> = vec![vec![1], vec![2], vec![3], vec![4], vec![5]];
     let modified: Vec<Vec<u64>> = original[0..3].to_vec();
 
-    let new_hasher = CachedTreeHasher::new(&modified).unwrap();
+    let new_cache = TreeHashCache::new(&modified).unwrap();
 
-    let mut modified_hasher = CachedTreeHasher::new(&original).unwrap();
-    modified_hasher.update(&modified).unwrap();
+    let mut modified_cache = TreeHashCache::new(&original).unwrap();
+    modified_cache.update(&modified).unwrap();
 
     assert_eq!(
-        new_hasher.cache.schemas.len(),
-        modified_hasher.cache.schemas.len(),
+        new_cache.schemas.len(),
+        modified_cache.schemas.len(),
         "Schema count is different"
     );
 
     assert_eq!(
-        new_hasher.cache.chunk_modified.len(),
-        modified_hasher.cache.chunk_modified.len(),
+        new_cache.chunk_modified.len(),
+        modified_cache.chunk_modified.len(),
         "Chunk count is different"
     );
 }
@@ -601,7 +598,7 @@ fn generic_test(index: usize) {
         d: 4,
     };
 
-    let mut cache = TreeHashCache::new(&inner, 0).unwrap();
+    let mut cache = TreeHashCache::new(&inner).unwrap();
 
     let changed_inner = match index {
         0 => Inner {
@@ -636,7 +633,7 @@ fn generic_test(index: usize) {
 
     let expected = merkleize(join(data));
 
-    let cache_bytes: Vec<u8> = cache.into();
+    let (cache_bytes, _, _) = cache.into_components();
 
     assert_eq!(expected, cache_bytes);
 }
@@ -666,9 +663,9 @@ fn inner_builds() {
         d: 4,
     };
 
-    let cache: Vec<u8> = TreeHashCache::new(&inner, 0).unwrap().into();
+    let (cache_bytes, _, _) = TreeHashCache::new(&inner).unwrap().into_components();
 
-    assert_eq!(expected, cache);
+    assert_eq!(expected, cache_bytes);
 }
 
 fn join(many: Vec<Vec<u8>>) -> Vec<u8> {
diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs
index b111ae7c4..50727a89f 100644
--- a/eth2/utils/tree_hash_derive/src/lib.rs
+++ b/eth2/utils/tree_hash_derive/src/lib.rs
@@ -58,7 +58,7 @@ pub fn subtree_derive(input: TokenStream) -> TokenStream {
     let output = quote! {
         impl cached_tree_hash::CachedTreeHash for #name {
             fn new_tree_hash_cache(&self, depth: usize) -> Result<cached_tree_hash::TreeHashCache, cached_tree_hash::Error> {
-                let tree = cached_tree_hash::TreeHashCache::from_leaves_and_subtrees(
+                let tree = cached_tree_hash::TreeHashCache::from_subtrees(
                     self,
                     vec![
                         #(
diff --git a/eth2/utils/tree_hash_derive/tests/tests.rs b/eth2/utils/tree_hash_derive/tests/tests.rs
index 11eae4e02..d4fd55165 100644
--- a/eth2/utils/tree_hash_derive/tests/tests.rs
+++ b/eth2/utils/tree_hash_derive/tests/tests.rs
@@ -1,4 +1,4 @@
-use cached_tree_hash::{CachedTreeHash, CachedTreeHasher};
+use cached_tree_hash::{CachedTreeHash, TreeHashCache};
 use tree_hash::{merkleize::merkle_root, SignedRoot, TreeHash};
 use tree_hash_derive::{CachedTreeHash, SignedRoot, TreeHash};
 
@@ -12,16 +12,16 @@ pub struct Inner {
 
 fn test_standard_and_cached<T: CachedTreeHash>(original: &T, modified: &T) {
     // let mut cache = original.new_tree_hash_cache().unwrap();
-    let mut hasher = CachedTreeHasher::new(original).unwrap();
+    let mut cache = TreeHashCache::new(original).unwrap();
 
     let standard_root = original.tree_hash_root();
-    let cached_root = hasher.tree_hash_root().unwrap();
+    let cached_root = cache.tree_hash_root().unwrap();
     assert_eq!(standard_root, cached_root);
 
     // Test after a modification
-    hasher.update(modified).unwrap();
+    cache.update(modified).unwrap();
     let standard_root = modified.tree_hash_root();
-    let cached_root = hasher.tree_hash_root().unwrap();
+    let cached_root = cache.tree_hash_root().unwrap();
     assert_eq!(standard_root, cached_root);
 }
 

From c61c4d93c19da51451a7172a83e4943477b9256b Mon Sep 17 00:00:00 2001
From: Luke Anderson <luke@lukeanderson.com.au>
Date: Wed, 1 May 2019 15:13:30 +1000
Subject: [PATCH 136/137] Added a quick 'documentation.md' file, to describe
 where the Lighthouse technical documentation is, and how it can be updated.

---
 docs/documentation.md | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 docs/documentation.md

diff --git a/docs/documentation.md b/docs/documentation.md
new file mode 100644
index 000000000..360055887
--- /dev/null
+++ b/docs/documentation.md
@@ -0,0 +1,14 @@
+# Lighthouse Technical Documentation
+
+The technical documentation, as generated by Rust, is available at [lighthouse-docs.sigmaprime.io](http://lighthouse-docs.sigmaprime.io/).
+
+This documentation is generated from Lighthouse and updated regularly.
+
+
+### How to update:
+
+- `cargo doc`: builds the docs inside the `target/doc/` directory.
+- `aws s3 sync target/doc/ s3://lighthouse-docs.sigmaprime.io/`: Uploads all of the docs, as generated with `cargo doc`, to the S3 bucket.
+
+**Note**: You will need appropriate credentials to make the upload.
+

From 13b23adb0d5f7c3f67f540d7b994301b9687a184 Mon Sep 17 00:00:00 2001
From: Luke Anderson <luke@lukeanderson.com.au>
Date: Wed, 1 May 2019 15:14:51 +1000
Subject: [PATCH 137/137] Added a link to the lighthouse technical
 documentation in the main readme file.

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 7727154e7..abf9acb6a 100644
--- a/README.md
+++ b/README.md
@@ -24,6 +24,7 @@ present-Ethereum functionality.
 - [About Lighthouse](docs/lighthouse.md): Goals, Ideology and Ethos surrounding
 this implementation.
 - [What is Ethereum Serenity](docs/serenity.md): an introduction to Ethereum Serenity.
+- [Lighthouse Technical Documentation](http://lighthouse-docs.sigmaprime.io/): The Rust generated documentation, updated regularly.
 
 If you'd like some background on Sigma Prime, please see the [Lighthouse Update
 \#00](https://lighthouse.sigmaprime.io/update-00.html) blog post or the