From 49639c40ee26ec31dccf03d9ec2dfa3d6dc92d2d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 28 Mar 2019 19:01:31 +1100 Subject: [PATCH] Implement failing cache hash test --- eth2/utils/ssz/src/cached_tree_hash.rs | 111 +++++++++++- eth2/utils/ssz/src/cached_tree_hash/impls.rs | 6 + eth2/utils/ssz/src/cached_tree_hash/tests.rs | 181 ++++++++++++++----- 3 files changed, 248 insertions(+), 50 deletions(-) diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs index caafaa2cf..75598f0b2 100644 --- a/eth2/utils/ssz/src/cached_tree_hash.rs +++ b/eth2/utils/ssz/src/cached_tree_hash.rs @@ -1,4 +1,5 @@ use hashing::hash; +use std::iter::Iterator; mod impls; mod tests; @@ -16,6 +17,8 @@ pub trait CachedTreeHash { /// prefixes. fn num_bytes(&self) -> usize; + fn num_child_nodes(&self) -> usize; + fn cached_hash_tree_root( &self, other: &Self::Item, @@ -81,15 +84,24 @@ impl TreeHashCache { self.chunk_modified.get(chunk).cloned() } - pub fn children_modified(&self, parent_chunk: usize) -> Option { - let children = children(parent_chunk); - - Some(self.changed(children.0)? | self.changed(children.1)?) + pub fn either_modified(&self, children: (&usize, &usize)) -> Option { + dbg!(&self.chunk_modified.len()); + dbg!(&self.cache.len() / BYTES_PER_CHUNK); + Some(self.changed(*children.0)? | self.changed(*children.1)?) } - pub fn hash_children(&self, parent_chunk: usize) -> Option> { + /* + pub fn children_modified(&self, parent_chunk: usize, child_offsets: &[usize]) -> Option { let children = children(parent_chunk); + let a = *child_offsets.get(children.0)?; + let b = *child_offsets.get(children.1)?; + + Some(self.changed(a)? | self.changed(b)?) + } + */ + + pub fn hash_children(&self, children: (&usize, &usize)) -> Option> { let start = children.0 * BYTES_PER_CHUNK; let end = start + BYTES_PER_CHUNK * 2; @@ -97,6 +109,30 @@ impl TreeHashCache { } } +/* +pub struct LocalCache { + offsets: Vec, +} + +impl LocalCache { + +} + +pub struct OffsetBTree { + offsets: Vec, +} + +impl From> for OffsetBTree { + fn from(offsets: Vec) -> Self { + Self { offsets } + } +} + +impl OffsetBTree { + fn +} +*/ + fn children(parent: usize) -> (usize, usize) { ((2 * parent + 1), (2 * parent + 2)) } @@ -105,6 +141,71 @@ fn num_nodes(num_leaves: usize) -> usize { 2 * num_leaves - 1 } +pub struct OffsetHandler { + num_internal_nodes: usize, + num_leaf_nodes: usize, + next_node: usize, + offsets: Vec, +} + +impl OffsetHandler { + fn from_lengths(offset: usize, mut lengths: Vec) -> Self { + // Extend it to the next power-of-two, if it is not already. + let num_leaf_nodes = if lengths.len().is_power_of_two() { + lengths.len() + } else { + let num_leaf_nodes = lengths.len().next_power_of_two(); + lengths.resize(num_leaf_nodes, 1); + num_leaf_nodes + }; + + let num_nodes = num_nodes(num_leaf_nodes); + let num_internal_nodes = num_nodes - num_leaf_nodes; + + let mut offsets = Vec::with_capacity(num_nodes); + offsets.append(&mut (offset..offset + num_internal_nodes).collect()); + + let mut next_node = num_internal_nodes + offset; + for i in 0..num_leaf_nodes { + offsets.push(next_node); + next_node += lengths[i]; + } + + Self { + num_internal_nodes, + num_leaf_nodes, + offsets, + next_node, + } + } + + pub fn total_nodes(&self) -> usize { + self.num_internal_nodes + self.num_leaf_nodes + } + + pub fn first_leaf_node(&self) -> Option { + self.offsets.get(self.num_internal_nodes).cloned() + } + + pub fn next_node(&self) -> usize { + self.next_node + } + + pub fn iter_internal_nodes<'a>( + &'a self, + ) -> impl DoubleEndedIterator { + let internal_nodes = &self.offsets[0..self.num_internal_nodes]; + + internal_nodes.iter().enumerate().map(move |(i, parent)| { + let children = children(i); + ( + parent, + (&self.offsets[children.0], &self.offsets[children.1]), + ) + }) + } +} + /// Split `values` into a power-of-two, identical-length chunks (padding with `0`) and merkleize /// them, returning the entire merkle tree. /// diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs index 1c2bf342e..6fb2d8938 100644 --- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs +++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs @@ -12,6 +12,10 @@ impl CachedTreeHash for u64 { 8 } + fn num_child_nodes(&self) -> usize { + 0 + } + fn cached_hash_tree_root( &self, other: &Self, @@ -27,6 +31,7 @@ impl CachedTreeHash for u64 { } } +/* impl CachedTreeHash for Vec where T: CachedTreeHash + Encodable, @@ -96,3 +101,4 @@ where Some(chunk + num_nodes) } } +*/ diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs index 8b235b3b9..791bc17b6 100644 --- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs +++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs @@ -13,58 +13,87 @@ impl CachedTreeHash for Inner { type Item = Self; fn build_cache_bytes(&self) -> Vec { + let cache_a = self.a.build_cache_bytes(); + let cache_b = self.b.build_cache_bytes(); + let cache_c = self.c.build_cache_bytes(); + let cache_d = self.d.build_cache_bytes(); + let mut leaves = vec![]; + leaves.extend_from_slice(&cache_a[0..32].to_vec()); + leaves.extend_from_slice(&cache_b[0..32].to_vec()); + leaves.extend_from_slice(&cache_c[0..32].to_vec()); + leaves.extend_from_slice(&cache_d[0..32].to_vec()); - leaves.append(&mut self.a.build_cache_bytes()); - leaves.append(&mut self.b.build_cache_bytes()); - leaves.append(&mut self.c.build_cache_bytes()); - leaves.append(&mut self.d.build_cache_bytes()); + let mut merkle = merkleize(leaves); - merkleize(leaves) + let num_leaves = 4; + let num_nodes = num_nodes(num_leaves); + let num_internal_nodes = num_nodes - num_leaves; + + let mut next_hash = num_internal_nodes * HASHSIZE; + merkle.splice(next_hash..next_hash + HASHSIZE, cache_a); + next_hash += HASHSIZE; + merkle.splice(next_hash..next_hash + HASHSIZE, cache_b); + next_hash += HASHSIZE; + merkle.splice(next_hash..next_hash + HASHSIZE, cache_c); + next_hash += HASHSIZE; + merkle.splice(next_hash..next_hash + HASHSIZE, cache_d); + + merkle } fn num_bytes(&self) -> usize { let mut bytes = 0; + bytes += self.a.num_bytes(); bytes += self.b.num_bytes(); bytes += self.c.num_bytes(); bytes += self.d.num_bytes(); + bytes } + fn num_child_nodes(&self) -> usize { + let mut children = 0; + let leaves = 4; + + children += self.a.num_child_nodes(); + children += self.b.num_child_nodes(); + children += self.c.num_child_nodes(); + children += self.d.num_child_nodes(); + + num_nodes(leaves) + children - 1 + } + fn cached_hash_tree_root( &self, other: &Self, cache: &mut TreeHashCache, chunk: usize, ) -> Option { - let mut num_leaves: usize = 0; - num_leaves += num_unsanitized_leaves(self.a.num_bytes()); - num_leaves += num_unsanitized_leaves(self.b.num_bytes()); - num_leaves += num_unsanitized_leaves(self.c.num_bytes()); - num_leaves += num_unsanitized_leaves(self.d.num_bytes()); - - let num_nodes = num_nodes(num_leaves); - let num_internal_nodes = num_nodes - num_leaves; + let mut offsets = vec![]; + offsets.push(self.a.num_child_nodes() + 1); + offsets.push(self.b.num_child_nodes() + 1); + offsets.push(self.c.num_child_nodes() + 1); + offsets.push(self.d.num_child_nodes() + 1); + let offset_handler = OffsetHandler::from_lengths(chunk, offsets); // Skip past the internal nodes and update any changed leaf nodes. { - let chunk = chunk + num_internal_nodes; + let chunk = offset_handler.first_leaf_node()?; let chunk = self.a.cached_hash_tree_root(&other.a, cache, chunk)?; let chunk = self.b.cached_hash_tree_root(&other.b, cache, chunk)?; let chunk = self.c.cached_hash_tree_root(&other.c, cache, chunk)?; let _chunk = self.d.cached_hash_tree_root(&other.d, cache, chunk)?; } - // Iterate backwards through the internal nodes, rehashing any node where it's children - // have changed. - for chunk in (chunk..chunk + num_internal_nodes).into_iter().rev() { - if cache.children_modified(chunk)? { - cache.modify_chunk(chunk, &cache.hash_children(chunk)?)?; + for (&parent, children) in offset_handler.iter_internal_nodes().rev() { + if cache.either_modified(children)? { + cache.modify_chunk(parent, &cache.hash_children(children)?)?; } } - Some(chunk + num_nodes) + Some(offset_handler.next_node()) } } @@ -79,53 +108,79 @@ impl CachedTreeHash for Outer { type Item = Self; fn build_cache_bytes(&self) -> Vec { + let cache_a = self.a.build_cache_bytes(); + let cache_b = self.b.build_cache_bytes(); + let cache_c = self.c.build_cache_bytes(); + let mut leaves = vec![]; + leaves.extend_from_slice(&cache_a[0..32].to_vec()); + leaves.extend_from_slice(&cache_b[0..32].to_vec()); + leaves.extend_from_slice(&cache_c[0..32].to_vec()); - leaves.append(&mut self.a.build_cache_bytes()); - leaves.append(&mut self.b.build_cache_bytes()); - leaves.append(&mut self.c.build_cache_bytes()); + let mut merkle = merkleize(leaves); - merkleize(leaves) + let num_leaves = 4; + let num_nodes = num_nodes(num_leaves); + let num_internal_nodes = num_nodes - num_leaves; + + let mut next_hash = num_internal_nodes * HASHSIZE; + merkle.splice(next_hash..next_hash + HASHSIZE, cache_a); + next_hash += (self.a.num_child_nodes() + 1) * HASHSIZE; + merkle.splice(next_hash..next_hash + HASHSIZE, cache_b); + next_hash += (self.b.num_child_nodes() + 1) * HASHSIZE; + merkle.splice(next_hash..next_hash + HASHSIZE, cache_c); + + merkle } fn num_bytes(&self) -> usize { let mut bytes = 0; bytes += self.a.num_bytes(); bytes += self.b.num_bytes(); + bytes += self.c.num_bytes(); bytes } + fn num_child_nodes(&self) -> usize { + let mut children = 0; + let leaves = 3; + + children += self.a.num_child_nodes(); + children += self.b.num_child_nodes(); + children += self.c.num_child_nodes(); + + num_nodes(leaves) + children - 1 + } + fn cached_hash_tree_root( &self, other: &Self, cache: &mut TreeHashCache, chunk: usize, ) -> Option { - let mut num_leaves: usize = 0; - num_leaves += num_unsanitized_leaves(self.a.num_bytes()); - num_leaves += num_unsanitized_leaves(self.b.num_bytes()); - num_leaves += num_unsanitized_leaves(self.c.num_bytes()); - - let num_nodes = num_nodes(num_leaves); - let num_internal_nodes = num_nodes - num_leaves; + let mut offsets = vec![]; + offsets.push(self.a.num_child_nodes() + 1); + offsets.push(self.b.num_child_nodes() + 1); + offsets.push(self.c.num_child_nodes() + 1); + let offset_handler = OffsetHandler::from_lengths(chunk, offsets); // Skip past the internal nodes and update any changed leaf nodes. { - let chunk = chunk + num_internal_nodes; + let chunk = offset_handler.first_leaf_node()?; let chunk = self.a.cached_hash_tree_root(&other.a, cache, chunk)?; let chunk = self.b.cached_hash_tree_root(&other.b, cache, chunk)?; let _chunk = self.c.cached_hash_tree_root(&other.c, cache, chunk)?; } - // Iterate backwards through the internal nodes, rehashing any node where it's children - // have changed. - for chunk in (chunk..chunk + num_internal_nodes).into_iter().rev() { - if cache.children_modified(chunk)? { - cache.modify_chunk(chunk, &cache.hash_children(chunk)?)?; + for (&parent, children) in offset_handler.iter_internal_nodes().rev() { + if cache.either_modified(children)? { + dbg!(parent); + dbg!(children); + cache.modify_chunk(parent, &cache.hash_children(children)?)?; } } - Some(chunk + num_nodes) + Some(offset_handler.next_node()) } } @@ -163,15 +218,30 @@ fn partial_modification_to_outer() { // Perform a differential hash let mut cache_struct = TreeHashCache::from_bytes(original_cache.clone()).unwrap(); - modified_outer.cached_hash_tree_root(&original_outer, &mut cache_struct, 0); + + modified_outer + .cached_hash_tree_root(&original_outer, &mut cache_struct, 0) + .unwrap(); + let modified_cache: Vec = cache_struct.into(); // Generate reference data. let mut data = vec![]; data.append(&mut int_to_bytes32(0)); - data.append(&mut inner.build_cache_bytes()); - data.append(&mut int_to_bytes32(42)); - let merkle = merkleize(data); + let inner_bytes = inner.build_cache_bytes(); + data.append(&mut int_to_bytes32(5)); + + let leaves = vec![ + int_to_bytes32(0), + inner_bytes[0..32].to_vec(), + int_to_bytes32(5), + vec![0; 32], // padding + ]; + let mut merkle = merkleize(join(leaves)); + merkle.splice(4 * 32..5 * 32, inner_bytes); + + assert_eq!(merkle.len() / HASHSIZE, 13); + assert_eq!(modified_cache.len() / HASHSIZE, 13); assert_eq!(merkle, modified_cache); } @@ -197,13 +267,33 @@ fn outer_builds() { // Generate reference data. let mut data = vec![]; data.append(&mut int_to_bytes32(0)); - data.append(&mut inner.build_cache_bytes()); + let inner_bytes = inner.build_cache_bytes(); data.append(&mut int_to_bytes32(5)); - let merkle = merkleize(data); - assert_eq!(merkle, cache); + let leaves = vec![ + int_to_bytes32(0), + inner_bytes[0..32].to_vec(), + int_to_bytes32(5), + vec![0; 32], // padding + ]; + let mut merkle = merkleize(join(leaves)); + merkle.splice(4 * 32..5 * 32, inner_bytes); + + assert_eq!(merkle.len() / HASHSIZE, 13); + assert_eq!(cache.len() / HASHSIZE, 13); + + for (i, chunk) in cache.chunks(HASHSIZE).enumerate() { + assert_eq!( + merkle[i * HASHSIZE..(i + 1) * HASHSIZE], + *chunk, + "failed on {}", + i + ); + } + // assert_eq!(merkle, cache); } +/* #[test] fn partial_modification_u64_vec() { let n: u64 = 50; @@ -272,6 +362,7 @@ fn vec_of_u64_builds() { assert_eq!(expected, cache); } +*/ #[test] fn merkleize_odd() {