From af39f096e7a635f39acf9851f84f96cda6af8750 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 17 Apr 2019 10:57:36 +1000 Subject: [PATCH] Add vector type to tree hashing --- eth2/types/src/beacon_state.rs | 28 ++++--- eth2/types/src/historical_batch.rs | 6 +- eth2/types/src/lib.rs | 2 + eth2/types/src/test_utils/mod.rs | 5 +- eth2/types/src/tree_hash_vector.rs | 82 +++++++++++++++++++ .../src/cached_tree_hash/impls/vec.rs | 6 +- eth2/utils/tree_hash/src/lib.rs | 32 +++++++- .../utils/tree_hash/src/standard_tree_hash.rs | 15 ++-- .../tree_hash/src/standard_tree_hash/impls.rs | 58 +++++++------ eth2/utils/tree_hash/tests/tests.rs | 34 +++++++- eth2/utils/tree_hash_derive/src/lib.rs | 15 +--- 11 files changed, 211 insertions(+), 72 deletions(-) create mode 100644 eth2/types/src/tree_hash_vector.rs diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 19c1b4c11..c068c4e03 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -60,7 +60,7 @@ pub struct BeaconState { pub validator_registry_update_epoch: Epoch, // Randomness and committees - pub latest_randao_mixes: Vec, + pub latest_randao_mixes: TreeHashVector, pub previous_shuffling_start_shard: u64, pub current_shuffling_start_shard: u64, pub previous_shuffling_epoch: Epoch, @@ -80,11 +80,11 @@ pub struct BeaconState { pub finalized_root: Hash256, // Recent state - pub latest_crosslinks: Vec, - latest_block_roots: Vec, - latest_state_roots: Vec, - latest_active_index_roots: Vec, - latest_slashed_balances: Vec, + pub latest_crosslinks: TreeHashVector, + latest_block_roots: TreeHashVector, + latest_state_roots: TreeHashVector, + latest_active_index_roots: TreeHashVector, + latest_slashed_balances: TreeHashVector, pub latest_block_header: BeaconBlockHeader, pub historical_roots: Vec, @@ -139,7 +139,8 @@ impl BeaconState { validator_registry_update_epoch: spec.genesis_epoch, // Randomness and committees - latest_randao_mixes: vec![spec.zero_hash; spec.latest_randao_mixes_length as usize], + latest_randao_mixes: vec![spec.zero_hash; spec.latest_randao_mixes_length as usize] + .into(), previous_shuffling_start_shard: spec.genesis_start_shard, current_shuffling_start_shard: spec.genesis_start_shard, previous_shuffling_epoch: spec.genesis_epoch, @@ -159,11 +160,12 @@ impl BeaconState { finalized_root: spec.zero_hash, // Recent state - latest_crosslinks: vec![initial_crosslink; spec.shard_count as usize], - latest_block_roots: vec![spec.zero_hash; spec.slots_per_historical_root], - latest_state_roots: vec![spec.zero_hash; spec.slots_per_historical_root], - latest_active_index_roots: vec![spec.zero_hash; spec.latest_active_index_roots_length], - latest_slashed_balances: vec![0; spec.latest_slashed_exit_length], + latest_crosslinks: vec![initial_crosslink; spec.shard_count as usize].into(), + latest_block_roots: vec![spec.zero_hash; spec.slots_per_historical_root].into(), + latest_state_roots: vec![spec.zero_hash; spec.slots_per_historical_root].into(), + latest_active_index_roots: vec![spec.zero_hash; spec.latest_active_index_roots_length] + .into(), + latest_slashed_balances: vec![0; spec.latest_slashed_exit_length].into(), latest_block_header: BeaconBlock::empty(spec).temporary_block_header(spec), historical_roots: vec![], @@ -505,7 +507,7 @@ impl BeaconState { /// Spec v0.5.0 pub fn fill_active_index_roots_with(&mut self, index_root: Hash256, spec: &ChainSpec) { self.latest_active_index_roots = - vec![index_root; spec.latest_active_index_roots_length as usize] + vec![index_root; spec.latest_active_index_roots_length as usize].into() } /// Safely obtains the index for latest state roots, given some `slot`. diff --git a/eth2/types/src/historical_batch.rs b/eth2/types/src/historical_batch.rs index 33dc9c450..23c26901e 100644 --- a/eth2/types/src/historical_batch.rs +++ b/eth2/types/src/historical_batch.rs @@ -1,5 +1,5 @@ use crate::test_utils::TestRandom; -use crate::Hash256; +use crate::{Hash256, TreeHashVector}; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; @@ -11,8 +11,8 @@ use tree_hash_derive::TreeHash; /// Spec v0.5.0 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] pub struct HistoricalBatch { - pub block_roots: Vec, - pub state_roots: Vec, + pub block_roots: TreeHashVector, + pub state_roots: TreeHashVector, } #[cfg(test)] diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index 118e862e8..070ed6745 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -27,6 +27,7 @@ pub mod pending_attestation; pub mod proposer_slashing; pub mod slashable_attestation; pub mod transfer; +pub mod tree_hash_vector; pub mod voluntary_exit; #[macro_use] pub mod slot_epoch_macros; @@ -65,6 +66,7 @@ pub use crate::slashable_attestation::SlashableAttestation; pub use crate::slot_epoch::{Epoch, Slot}; pub use crate::slot_height::SlotHeight; pub use crate::transfer::Transfer; +pub use crate::tree_hash_vector::TreeHashVector; pub use crate::validator::Validator; pub use crate::voluntary_exit::VoluntaryExit; diff --git a/eth2/types/src/test_utils/mod.rs b/eth2/types/src/test_utils/mod.rs index 018b70d15..9d69a48f6 100644 --- a/eth2/types/src/test_utils/mod.rs +++ b/eth2/types/src/test_utils/mod.rs @@ -17,7 +17,10 @@ mod testing_voluntary_exit_builder; pub use generate_deterministic_keypairs::generate_deterministic_keypairs; pub use keypairs_file::KeypairsFile; -pub use rand::{prng::XorShiftRng, SeedableRng}; +pub use rand::{ + RngCore, + {prng::XorShiftRng, SeedableRng}, +}; pub use serde_utils::{fork_from_hex_str, u8_from_hex_str}; pub use test_random::TestRandom; pub use testing_attestation_builder::TestingAttestationBuilder; diff --git a/eth2/types/src/tree_hash_vector.rs b/eth2/types/src/tree_hash_vector.rs new file mode 100644 index 000000000..1cc8e40a5 --- /dev/null +++ b/eth2/types/src/tree_hash_vector.rs @@ -0,0 +1,82 @@ +use crate::test_utils::{RngCore, TestRandom}; +use serde_derive::{Deserialize, Serialize}; +use ssz::{Decodable, DecodeError, Encodable, SszStream}; +use std::ops::{Deref, DerefMut}; +use tree_hash::TreeHash; + +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +pub struct TreeHashVector(Vec); + +impl From> for TreeHashVector { + fn from(vec: Vec) -> TreeHashVector { + TreeHashVector(vec) + } +} + +impl Into> for TreeHashVector { + fn into(self) -> Vec { + self.0 + } +} + +impl Deref for TreeHashVector { + type Target = Vec; + + fn deref(&self) -> &Vec { + &self.0 + } +} + +impl DerefMut for TreeHashVector { + fn deref_mut(&mut self) -> &mut Vec { + &mut self.0 + } +} + +impl tree_hash::TreeHash for TreeHashVector +where + T: TreeHash, +{ + fn tree_hash_type() -> tree_hash::TreeHashType { + tree_hash::TreeHashType::Vector + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("Vector should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("Vector should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + tree_hash::standard_tree_hash::vec_tree_hash_root(self) + } +} + +impl Encodable for TreeHashVector +where + T: Encodable, +{ + fn ssz_append(&self, s: &mut SszStream) { + s.append_vec(self) + } +} + +impl Decodable for TreeHashVector +where + T: Decodable, +{ + fn ssz_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), DecodeError> { + ssz::decode_ssz_list(bytes, index).and_then(|(vec, i)| Ok((vec.into(), i))) + } +} + +impl TestRandom for TreeHashVector +where + U: TestRandom, +{ + fn random_for_test(rng: &mut T) -> Self { + Vec::random_for_test(rng).into() + } +} diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs index 6c0970cef..1cd7eb902 100644 --- a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs +++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs @@ -9,7 +9,7 @@ where TreeHashType::Basic => { TreeHashCache::from_bytes(merkleize(get_packed_leaves(self)?), false) } - TreeHashType::Composite | TreeHashType::List => { + TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => { let subtrees = self .iter() .map(|item| TreeHashCache::new(item)) @@ -23,7 +23,7 @@ where fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result { let lengths = match T::tree_hash_type() { TreeHashType::Basic => vec![1; self.len() / T::tree_hash_packing_factor()], - TreeHashType::Composite | TreeHashType::List => { + TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => { let mut lengths = vec![]; for item in self { @@ -97,7 +97,7 @@ where TreeHashCache::from_bytes(leaves, true)?, ); } - TreeHashType::Composite | TreeHashType::List => { + TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => { let mut i = offset_handler.num_leaf_nodes; for &start_chunk in offset_handler.iter_leaf_nodes().rev() { i -= 1; diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs index 7c74c9f97..fe2001002 100644 --- a/eth2/utils/tree_hash/src/lib.rs +++ b/eth2/utils/tree_hash/src/lib.rs @@ -13,8 +13,9 @@ pub use standard_tree_hash::{merkle_root, TreeHash}; #[derive(Debug, PartialEq, Clone)] pub enum TreeHashType { Basic, + Vector, List, - Composite, + Container, } fn num_sanitized_leaves(num_bytes: usize) -> usize { @@ -31,15 +32,15 @@ macro_rules! impl_tree_hash_for_ssz_bytes { ($type: ident) => { impl tree_hash::TreeHash for $type { fn tree_hash_type() -> tree_hash::TreeHashType { - tree_hash::TreeHashType::List + tree_hash::TreeHashType::Vector } fn tree_hash_packed_encoding(&self) -> Vec { - panic!("bytesN should never be packed.") + unreachable!("Vector should never be packed.") } fn tree_hash_packing_factor() -> usize { - panic!("bytesN should never be packed.") + unreachable!("Vector should never be packed.") } fn tree_hash_root(&self) -> Vec { @@ -48,3 +49,26 @@ macro_rules! impl_tree_hash_for_ssz_bytes { } }; } + +#[macro_export] +macro_rules! impl_vec_as_fixed_len { + ($type: ty) => { + impl tree_hash::TreeHash for $type { + fn tree_hash_type() -> tree_hash::TreeHashType { + tree_hash::TreeHashType::Vector + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("Vector should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("Vector should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + tree_hash::standard_tree_hash::vec_tree_hash_root(self) + } + } + }; +} diff --git a/eth2/utils/tree_hash/src/standard_tree_hash.rs b/eth2/utils/tree_hash/src/standard_tree_hash.rs index 473d2a5f0..130c360ed 100644 --- a/eth2/utils/tree_hash/src/standard_tree_hash.rs +++ b/eth2/utils/tree_hash/src/standard_tree_hash.rs @@ -3,6 +3,8 @@ use hashing::hash; use int_to_bytes::int_to_bytes32; use ssz::ssz_encode; +pub use impls::vec_tree_hash_root; + mod impls; pub trait TreeHash { @@ -16,11 +18,18 @@ pub trait TreeHash { } pub fn merkle_root(bytes: &[u8]) -> Vec { - // TODO: replace this with a _more_ efficient fn which is more memory efficient. + // TODO: replace this with a more memory efficient method. efficient_merkleize(&bytes)[0..32].to_vec() } pub fn efficient_merkleize(bytes: &[u8]) -> Vec { + // If the bytes are just one chunk (or less than one chunk) just return them. + if bytes.len() <= HASHSIZE { + let mut o = bytes.to_vec(); + o.resize(HASHSIZE, 0); + return o; + } + let leaves = num_sanitized_leaves(bytes.len()); let nodes = num_nodes(leaves); let internal_nodes = nodes - leaves; @@ -29,10 +38,6 @@ pub fn efficient_merkleize(bytes: &[u8]) -> Vec { let mut o: Vec = vec![0; internal_nodes * HASHSIZE]; - if o.len() < HASHSIZE { - o.resize(HASHSIZE, 0); - } - o.append(&mut bytes.to_vec()); assert_eq!(o.len(), num_bytes); diff --git a/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs b/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs index 749d5b3bb..c3be8d55b 100644 --- a/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs +++ b/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs @@ -50,7 +50,7 @@ impl TreeHash for [u8; 4] { impl TreeHash for H256 { fn tree_hash_type() -> TreeHashType { - TreeHashType::Basic + TreeHashType::Vector } fn tree_hash_packed_encoding(&self) -> Vec { @@ -62,7 +62,7 @@ impl TreeHash for H256 { } fn tree_hash_root(&self) -> Vec { - ssz_encode(self) + merkle_root(&ssz::ssz_encode(self)) } } @@ -83,37 +83,43 @@ where } fn tree_hash_root(&self) -> Vec { - let leaves = match T::tree_hash_type() { - TreeHashType::Basic => { - let mut leaves = - Vec::with_capacity((HASHSIZE / T::tree_hash_packing_factor()) * self.len()); - - for item in self { - leaves.append(&mut item.tree_hash_packed_encoding()); - } - - leaves - } - TreeHashType::Composite | TreeHashType::List => { - let mut leaves = Vec::with_capacity(self.len() * HASHSIZE); - - for item in self { - leaves.append(&mut item.tree_hash_root()) - } - - leaves - } - }; - - // Mix in the length let mut root_and_len = Vec::with_capacity(HASHSIZE * 2); - root_and_len.append(&mut merkle_root(&leaves)); + root_and_len.append(&mut vec_tree_hash_root(self)); root_and_len.append(&mut int_to_bytes32(self.len() as u64)); hash(&root_and_len) } } +pub fn vec_tree_hash_root(vec: &[T]) -> Vec +where + T: TreeHash, +{ + let leaves = match T::tree_hash_type() { + TreeHashType::Basic => { + let mut leaves = + Vec::with_capacity((HASHSIZE / T::tree_hash_packing_factor()) * vec.len()); + + for item in vec { + leaves.append(&mut item.tree_hash_packed_encoding()); + } + + leaves + } + TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => { + let mut leaves = Vec::with_capacity(vec.len() * HASHSIZE); + + for item in vec { + leaves.append(&mut item.tree_hash_root()) + } + + leaves + } + }; + + merkle_root(&leaves) +} + #[cfg(test)] mod test { use super::*; diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs index db33709ac..4d2c6f282 100644 --- a/eth2/utils/tree_hash/tests/tests.rs +++ b/eth2/utils/tree_hash/tests/tests.rs @@ -13,7 +13,7 @@ pub struct InternalCache { impl TreeHash for InternalCache { fn tree_hash_type() -> TreeHashType { - TreeHashType::Composite + TreeHashType::Container } fn tree_hash_packed_encoding(&self) -> Vec { @@ -146,7 +146,7 @@ pub struct Inner { impl TreeHash for Inner { fn tree_hash_type() -> TreeHashType { - TreeHashType::Composite + TreeHashType::Container } fn tree_hash_packed_encoding(&self) -> Vec { @@ -231,7 +231,7 @@ pub struct Outer { impl TreeHash for Outer { fn tree_hash_type() -> TreeHashType { - TreeHashType::Composite + TreeHashType::Container } fn tree_hash_packed_encoding(&self) -> Vec { @@ -894,11 +894,39 @@ fn vec_of_u64_builds() { let my_vec = vec![1, 2, 3, 4, 5]; + // + // Note: the length is not mixed-in in this example. The user must ensure the length is + // mixed-in. + // + let cache: Vec = TreeHashCache::new(&my_vec).unwrap().into(); assert_eq!(expected, cache); } +#[test] +fn vec_does_mix_in_len() { + let data = join(vec![ + int_to_bytes8(1), + int_to_bytes8(2), + int_to_bytes8(3), + int_to_bytes8(4), + int_to_bytes8(5), + vec![0; 32 - 8], // padding + ]); + + let tree = merkleize(data); + + let my_vec: Vec = vec![1, 2, 3, 4, 5]; + + let mut expected = vec![0; 32]; + expected.copy_from_slice(&tree[0..HASHSIZE]); + expected.append(&mut int_to_bytes32(my_vec.len() as u64)); + let expected = hash(&expected); + + assert_eq!(&expected[0..HASHSIZE], &my_vec.tree_hash_root()[..]); +} + #[test] fn merkleize_odd() { let data = join(vec![ diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs index e3a7b4aaa..4b7761f91 100644 --- a/eth2/utils/tree_hash_derive/src/lib.rs +++ b/eth2/utils/tree_hash_derive/src/lib.rs @@ -129,7 +129,7 @@ pub fn tree_hash_derive(input: TokenStream) -> TokenStream { let output = quote! { impl tree_hash::TreeHash for #name { fn tree_hash_type() -> tree_hash::TreeHashType { - tree_hash::TreeHashType::Composite + tree_hash::TreeHashType::Container } fn tree_hash_packed_encoding(&self) -> Vec { @@ -154,19 +154,6 @@ pub fn tree_hash_derive(input: TokenStream) -> TokenStream { output.into() } -/// Implements `tree_hash::TreeHash` for some `struct`, whilst excluding any fields following and -/// including a field that is of type "Signature" or "AggregateSignature". -/// -/// See: -/// https://github.com/ethereum/eth2.0-specs/blob/master/specs/simple-serialize.md#signed-roots -/// -/// This is a rather horrendous macro, it will read the type of the object as a string and decide -/// if it's a signature by matching that string against "Signature" or "AggregateSignature". So, -/// it's important that you use those exact words as your type -- don't alias it to something else. -/// -/// If you can think of a better way to do this, please make an issue! -/// -/// Fields are processed in the order they are defined. #[proc_macro_derive(SignedRoot, attributes(signed_root))] pub fn tree_hash_signed_root_derive(input: TokenStream) -> TokenStream { let item = parse_macro_input!(input as DeriveInput);