diff --git a/.travis.yml b/.travis.yml index e725aa0ba..f89db54c9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,9 @@ language: rust +cache: + directories: + - /home/travis/.cargo +before_cache: + - rm -rf /home/travis/.cargo/registry before_install: - curl -OL https://github.com/google/protobuf/releases/download/v3.4.0/protoc-3.4.0-linux-x86_64.zip - unzip protoc-3.4.0-linux-x86_64.zip -d protoc3 @@ -6,11 +11,13 @@ before_install: - sudo mv protoc3/include/* /usr/local/include/ - sudo chown $USER /usr/local/bin/protoc - sudo chown -R $USER /usr/local/include/google +env: + - BUILD=--all + - BUILD=--release --all + - BUILD= --manifest-path eth2/state_processing/Cargo.toml --release --features fake_crypto script: - - cargo build --verbose --all - - cargo build --verbose --release --all - - cargo test --verbose --all - - cargo test --verbose --release --all + - cargo build --verbose $BUILD + - cargo test --verbose $BUILD - cargo fmt --all -- --check # No clippy until later... #- cargo clippy @@ -22,6 +29,15 @@ matrix: allow_failures: - rust: nightly fast_finish: true + exclude: + - rust: beta + env: BUILD=--release --all + - rust: beta + env: BUILD= --manifest-path eth2/state_processing/Cargo.toml --release --features fake_crypto + - rust: nightly + env: BUILD=--release --all + - rust: nightly + env: BUILD= --manifest-path eth2/state_processing/Cargo.toml --release --features fake_crypto install: - rustup component add rustfmt - rustup component add clippy diff --git a/Cargo.toml b/Cargo.toml index 5c9593f5a..b419d32e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,8 @@ members = [ "eth2/utils/ssz", "eth2/utils/ssz_derive", "eth2/utils/swap_or_not_shuffle", + "eth2/utils/tree_hash", + "eth2/utils/tree_hash_derive", "eth2/utils/fisher_yates_shuffle", "eth2/utils/test_random_derive", "beacon_node", diff --git a/Jenkinsfile b/Jenkinsfile index d12189941..11cbf0abe 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,6 +23,11 @@ pipeline { steps { sh 'cargo test --verbose --all' sh 'cargo test --verbose --all --release' + sh 'cargo test --manifest-path eth2/state_processing/Cargo.toml --verbose \ + --release --features fake_crypto' + sh 'cargo test --manifest-path eth2/state_processing/Cargo.toml --verbose \ + --release --features fake_crypto -- --ignored' + } } } diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index 55d4bacfd..e2a4527a9 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -23,4 +23,5 @@ serde_json = "1.0" slot_clock = { path = "../../eth2/utils/slot_clock" } ssz = { path = "../../eth2/utils/ssz" } state_processing = { path = "../../eth2/state_processing" } +tree_hash = { path = "../../eth2/utils/tree_hash" } types = { path = "../../eth2/types" } diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index a22f4179e..41a718655 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -303,8 +303,6 @@ where /// then having it iteratively updated -- in such a case it's possible for another thread to /// find the state at an old slot. pub fn update_state(&self, mut state: BeaconState) -> Result<(), Error> { - let latest_block_header = self.head().beacon_block.block_header(); - let present_slot = match self.slot_clock.present_slot() { Ok(Some(slot)) => slot, _ => return Err(Error::UnableToReadSlot), @@ -312,7 +310,7 @@ where // If required, transition the new state to the present slot. for _ in state.slot.as_u64()..present_slot.as_u64() { - per_slot_processing(&mut state, &latest_block_header, &self.spec)?; + per_slot_processing(&mut state, &self.spec)?; } state.build_all_caches(&self.spec)?; @@ -324,8 +322,6 @@ where /// Ensures the current canonical `BeaconState` has been transitioned to match the `slot_clock`. pub fn catchup_state(&self) -> Result<(), Error> { - let latest_block_header = self.head().beacon_block.block_header(); - let present_slot = match self.slot_clock.present_slot() { Ok(Some(slot)) => slot, _ => return Err(Error::UnableToReadSlot), @@ -339,7 +335,7 @@ where state.build_epoch_cache(RelativeEpoch::NextWithoutRegistryChange, &self.spec)?; state.build_epoch_cache(RelativeEpoch::NextWithRegistryChange, &self.spec)?; - per_slot_processing(&mut *state, &latest_block_header, &self.spec)?; + per_slot_processing(&mut *state, &self.spec)?; } state.build_all_caches(&self.spec)?; @@ -617,9 +613,8 @@ where // Transition the parent state to the block slot. let mut state = parent_state; - let previous_block_header = parent_block.block_header(); for _ in state.slot.as_u64()..block.slot.as_u64() { - if let Err(e) = per_slot_processing(&mut state, &previous_block_header, &self.spec) { + if let Err(e) = per_slot_processing(&mut state, &self.spec) { return Ok(BlockProcessingOutcome::InvalidBlock( InvalidBlock::SlotProcessingError(e), )); diff --git a/beacon_node/beacon_chain/src/initialise.rs b/beacon_node/beacon_chain/src/initialise.rs index 0951e06fb..c66dd63b1 100644 --- a/beacon_node/beacon_chain/src/initialise.rs +++ b/beacon_node/beacon_chain/src/initialise.rs @@ -7,9 +7,9 @@ use db::stores::{BeaconBlockStore, BeaconStateStore}; use db::{DiskDB, MemoryDB}; use fork_choice::BitwiseLMDGhost; use slot_clock::SystemTimeSlotClock; -use ssz::TreeHash; use std::path::PathBuf; use std::sync::Arc; +use tree_hash::TreeHash; use types::test_utils::TestingBeaconStateBuilder; use types::{BeaconBlock, ChainSpec, Hash256}; @@ -32,7 +32,7 @@ pub fn initialise_beacon_chain( let (genesis_state, _keypairs) = state_builder.build(); let mut genesis_block = BeaconBlock::empty(&spec); - genesis_block.state_root = Hash256::from_slice(&genesis_state.hash_tree_root()); + genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root()); // Slot clock let slot_clock = SystemTimeSlotClock::new( @@ -73,7 +73,7 @@ pub fn initialise_test_beacon_chain( let (genesis_state, _keypairs) = state_builder.build(); let mut genesis_block = BeaconBlock::empty(spec); - genesis_block.state_root = Hash256::from_slice(&genesis_state.hash_tree_root()); + genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root()); // Slot clock let slot_clock = SystemTimeSlotClock::new( diff --git a/beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs b/beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs index 5c5477e55..d174670c0 100644 --- a/beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs +++ b/beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs @@ -5,8 +5,8 @@ use db::{ }; use fork_choice::BitwiseLMDGhost; use slot_clock::TestingSlotClock; -use ssz::TreeHash; use std::sync::Arc; +use tree_hash::TreeHash; use types::test_utils::TestingBeaconStateBuilder; use types::*; @@ -27,7 +27,7 @@ impl TestingBeaconChainBuilder { let (genesis_state, _keypairs) = self.state_builder.build(); let mut genesis_block = BeaconBlock::empty(&spec); - genesis_block.state_root = Hash256::from_slice(&genesis_state.hash_tree_root()); + genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root()); // Create the Beacon Chain BeaconChain::from_genesis( diff --git a/beacon_node/beacon_chain/test_harness/Cargo.toml b/beacon_node/beacon_chain/test_harness/Cargo.toml index 50d154732..a2abf6c5a 100644 --- a/beacon_node/beacon_chain/test_harness/Cargo.toml +++ b/beacon_node/beacon_chain/test_harness/Cargo.toml @@ -38,5 +38,6 @@ serde_json = "1.0" serde_yaml = "0.8" slot_clock = { path = "../../../eth2/utils/slot_clock" } ssz = { path = "../../../eth2/utils/ssz" } +tree_hash = { path = "../../../eth2/utils/tree_hash" } types = { path = "../../../eth2/types" } yaml-rust = "0.4.2" diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index aeb734a4e..34b559478 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -9,8 +9,8 @@ use fork_choice::BitwiseLMDGhost; use log::debug; use rayon::prelude::*; use slot_clock::TestingSlotClock; -use ssz::TreeHash; use std::sync::Arc; +use tree_hash::TreeHash; use types::{test_utils::TestingBeaconStateBuilder, *}; type TestingBeaconChain = BeaconChain>; @@ -54,7 +54,7 @@ impl BeaconChainHarness { let (mut genesis_state, keypairs) = state_builder.build(); let mut genesis_block = BeaconBlock::empty(&spec); - genesis_block.state_root = Hash256::from_slice(&genesis_state.hash_tree_root()); + genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root()); genesis_state .build_epoch_cache(RelativeEpoch::Previous, &spec) @@ -163,7 +163,7 @@ impl BeaconChainHarness { data: data.clone(), custody_bit: false, } - .hash_tree_root(); + .tree_hash_root(); let domain = self.spec.get_domain( state.slot.epoch(self.spec.slots_per_epoch), Domain::Attestation, diff --git a/beacon_node/beacon_chain/test_harness/src/lib.rs b/beacon_node/beacon_chain/test_harness/src/lib.rs index 0703fd4a5..e93fa7003 100644 --- a/beacon_node/beacon_chain/test_harness/src/lib.rs +++ b/beacon_node/beacon_chain/test_harness/src/lib.rs @@ -8,7 +8,7 @@ //! producing blocks and attestations. //! //! Example: -//! ``` +//! ```rust,no_run //! use test_harness::BeaconChainHarness; //! use types::ChainSpec; //! diff --git a/beacon_node/beacon_chain/test_harness/src/test_case.rs b/beacon_node/beacon_chain/test_harness/src/test_case.rs index f65b45505..28c7ae8a8 100644 --- a/beacon_node/beacon_chain/test_harness/src/test_case.rs +++ b/beacon_node/beacon_chain/test_harness/src/test_case.rs @@ -4,7 +4,7 @@ use crate::beacon_chain_harness::BeaconChainHarness; use beacon_chain::CheckPoint; use log::{info, warn}; -use ssz::SignedRoot; +use tree_hash::SignedRoot; use types::*; use types::test_utils::*; diff --git a/beacon_node/network/Cargo.toml b/beacon_node/network/Cargo.toml index cd2c2269a..36bf1f141 100644 --- a/beacon_node/network/Cargo.toml +++ b/beacon_node/network/Cargo.toml @@ -15,6 +15,7 @@ version = { path = "../version" } types = { path = "../../eth2/types" } slog = { version = "^2.2.3" , features = ["max_level_trace", "release_max_level_debug"] } ssz = { path = "../../eth2/utils/ssz" } +tree_hash = { path = "../../eth2/utils/tree_hash" } futures = "0.1.25" error-chain = "0.12.0" crossbeam-channel = "0.3.8" diff --git a/beacon_node/network/src/sync/import_queue.rs b/beacon_node/network/src/sync/import_queue.rs index 0026347eb..106e3eb66 100644 --- a/beacon_node/network/src/sync/import_queue.rs +++ b/beacon_node/network/src/sync/import_queue.rs @@ -2,9 +2,9 @@ use crate::beacon_chain::BeaconChain; use eth2_libp2p::rpc::methods::*; use eth2_libp2p::PeerId; use slog::{debug, error}; -use ssz::TreeHash; use std::sync::Arc; use std::time::{Duration, Instant}; +use tree_hash::TreeHash; use types::{BeaconBlock, BeaconBlockBody, BeaconBlockHeader, Hash256, Slot}; /// Provides a queue for fully and partially built `BeaconBlock`s. @@ -15,7 +15,7 @@ use types::{BeaconBlock, BeaconBlockBody, BeaconBlockHeader, Hash256, Slot}; /// /// - When we receive a `BeaconBlockBody`, the only way we can find it's matching /// `BeaconBlockHeader` is to find a header such that `header.beacon_block_body == -/// hash_tree_root(body)`. Therefore, if we used a `HashMap` we would need to use the root of +/// tree_hash_root(body)`. Therefore, if we used a `HashMap` we would need to use the root of /// `BeaconBlockBody` as the key. /// - It is possible for multiple distinct blocks to have identical `BeaconBlockBodies`. Therefore /// we cannot use a `HashMap` keyed by the root of `BeaconBlockBody`. @@ -166,7 +166,7 @@ impl ImportQueue { let mut required_bodies: Vec = vec![]; for header in headers { - let block_root = Hash256::from_slice(&header.hash_tree_root()[..]); + let block_root = Hash256::from_slice(&header.tree_hash_root()[..]); if self.chain_has_not_seen_block(&block_root) { self.insert_header(block_root, header, sender.clone()); @@ -230,7 +230,7 @@ impl ImportQueue { /// /// If the body already existed, the `inserted` time is set to `now`. fn insert_body(&mut self, body: BeaconBlockBody, sender: PeerId) { - let body_root = Hash256::from_slice(&body.hash_tree_root()[..]); + let body_root = Hash256::from_slice(&body.tree_hash_root()[..]); self.partials.iter_mut().for_each(|mut p| { if let Some(header) = &mut p.header { @@ -250,7 +250,7 @@ impl ImportQueue { /// /// If the partial already existed, the `inserted` time is set to `now`. fn insert_full_block(&mut self, block: BeaconBlock, sender: PeerId) { - let block_root = Hash256::from_slice(&block.hash_tree_root()[..]); + let block_root = Hash256::from_slice(&block.tree_hash_root()[..]); let partial = PartialBeaconBlock { slot: block.slot, diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index 824458b89..1b57fbc00 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -5,10 +5,10 @@ use eth2_libp2p::rpc::methods::*; use eth2_libp2p::rpc::{RPCRequest, RPCResponse, RequestId}; use eth2_libp2p::PeerId; use slog::{debug, error, info, o, warn}; -use ssz::TreeHash; use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; +use tree_hash::TreeHash; use types::{Attestation, BeaconBlock, Epoch, Hash256, Slot}; /// The number of slots that we can import blocks ahead of us, before going into full Sync mode. @@ -565,7 +565,7 @@ impl SimpleSync { return false; } - let block_root = Hash256::from_slice(&block.hash_tree_root()); + let block_root = Hash256::from_slice(&block.tree_hash_root()); // Ignore any block that the chain already knows about. if self.chain_has_seen_block(&block_root) { diff --git a/eth2/attester/Cargo.toml b/eth2/attester/Cargo.toml index 956ecf565..41824274d 100644 --- a/eth2/attester/Cargo.toml +++ b/eth2/attester/Cargo.toml @@ -7,4 +7,5 @@ edition = "2018" [dependencies] slot_clock = { path = "../../eth2/utils/slot_clock" } ssz = { path = "../../eth2/utils/ssz" } +tree_hash = { path = "../../eth2/utils/tree_hash" } types = { path = "../../eth2/types" } diff --git a/eth2/attester/src/lib.rs b/eth2/attester/src/lib.rs index a4295f005..1bbbd6b43 100644 --- a/eth2/attester/src/lib.rs +++ b/eth2/attester/src/lib.rs @@ -2,8 +2,8 @@ pub mod test_utils; mod traits; use slot_clock::SlotClock; -use ssz::TreeHash; use std::sync::Arc; +use tree_hash::TreeHash; use types::{AttestationData, AttestationDataAndCustodyBit, FreeAttestation, Signature, Slot}; pub use self::traits::{ @@ -141,7 +141,7 @@ impl Attester BlockProducer bool { if bitfield.num_bytes() != ((committee_size + 7) / 8) { return false; @@ -18,3 +18,62 @@ pub fn verify_bitfield_length(bitfield: &Bitfield, committee_size: usize) -> boo true } + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn bitfield_length() { + assert_eq!( + verify_bitfield_length(&Bitfield::from_bytes(&[0b0000_0001]), 4), + true + ); + + assert_eq!( + verify_bitfield_length(&Bitfield::from_bytes(&[0b0001_0001]), 4), + false + ); + + assert_eq!( + verify_bitfield_length(&Bitfield::from_bytes(&[0b0000_0000]), 4), + true + ); + + assert_eq!( + verify_bitfield_length(&Bitfield::from_bytes(&[0b1000_0000]), 8), + true + ); + + assert_eq!( + verify_bitfield_length(&Bitfield::from_bytes(&[0b1000_0000, 0b0000_0000]), 16), + true + ); + + assert_eq!( + verify_bitfield_length(&Bitfield::from_bytes(&[0b1000_0000, 0b0000_0000]), 15), + false + ); + + assert_eq!( + verify_bitfield_length(&Bitfield::from_bytes(&[0b0000_0000, 0b0000_0000]), 8), + false + ); + + assert_eq!( + verify_bitfield_length( + &Bitfield::from_bytes(&[0b0000_0000, 0b0000_0000, 0b0000_0000]), + 8 + ), + false + ); + + assert_eq!( + verify_bitfield_length( + &Bitfield::from_bytes(&[0b0000_0000, 0b0000_0000, 0b0000_0000]), + 24 + ), + true + ); + } +} diff --git a/eth2/state_processing/src/get_genesis_state.rs b/eth2/state_processing/src/get_genesis_state.rs index 7c4d4cafd..4e9fb6caf 100644 --- a/eth2/state_processing/src/get_genesis_state.rs +++ b/eth2/state_processing/src/get_genesis_state.rs @@ -1,5 +1,5 @@ use super::per_block_processing::{errors::BlockProcessingError, process_deposits}; -use ssz::TreeHash; +use tree_hash::TreeHash; use types::*; pub enum GenesisError { @@ -9,13 +9,13 @@ pub enum GenesisError { /// Returns the genesis `BeaconState` /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn get_genesis_state( genesis_validator_deposits: &[Deposit], genesis_time: u64, genesis_eth1_data: Eth1Data, spec: &ChainSpec, -) -> Result<(), BlockProcessingError> { +) -> Result { // Get the genesis `BeaconState` let mut state = BeaconState::genesis(genesis_time, genesis_eth1_data, spec); @@ -36,13 +36,13 @@ pub fn get_genesis_state( let active_validator_indices = state .get_cached_active_validator_indices(RelativeEpoch::Current, spec)? .to_vec(); - let genesis_active_index_root = Hash256::from_slice(&active_validator_indices.hash_tree_root()); + let genesis_active_index_root = Hash256::from_slice(&active_validator_indices.tree_hash_root()); state.fill_active_index_roots_with(genesis_active_index_root, spec); // Generate the current shuffling seed. state.current_shuffling_seed = state.generate_seed(spec.genesis_epoch, spec)?; - Ok(()) + Ok(state) } impl From for GenesisError { diff --git a/eth2/state_processing/src/per_block_processing.rs b/eth2/state_processing/src/per_block_processing.rs index 6c52a2676..58b948f62 100644 --- a/eth2/state_processing/src/per_block_processing.rs +++ b/eth2/state_processing/src/per_block_processing.rs @@ -1,7 +1,7 @@ use crate::common::slash_validator; use errors::{BlockInvalid as Invalid, BlockProcessingError as Error, IntoWithIndex}; use rayon::prelude::*; -use ssz::{SignedRoot, TreeHash}; +use tree_hash::{SignedRoot, TreeHash}; use types::*; pub use self::verify_attester_slashing::{ @@ -39,7 +39,7 @@ const VERIFY_DEPOSIT_MERKLE_PROOFS: bool = false; /// Returns `Ok(())` if the block is valid and the state was successfully updated. Otherwise /// returns an error describing why the block was invalid or how the function failed to execute. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn per_block_processing( state: &mut BeaconState, block: &BeaconBlock, @@ -54,7 +54,7 @@ pub fn per_block_processing( /// Returns `Ok(())` if the block is valid and the state was successfully updated. Otherwise /// returns an error describing why the block was invalid or how the function failed to execute. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn per_block_processing_without_verifying_block_signature( state: &mut BeaconState, block: &BeaconBlock, @@ -69,7 +69,7 @@ pub fn per_block_processing_without_verifying_block_signature( /// Returns `Ok(())` if the block is valid and the state was successfully updated. Otherwise /// returns an error describing why the block was invalid or how the function failed to execute. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn per_block_processing_signature_optional( mut state: &mut BeaconState, block: &BeaconBlock, @@ -99,7 +99,7 @@ fn per_block_processing_signature_optional( /// Processes the block header. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_block_header( state: &mut BeaconState, block: &BeaconBlock, @@ -107,12 +107,14 @@ pub fn process_block_header( ) -> Result<(), Error> { verify!(block.slot == state.slot, Invalid::StateSlotMismatch); - // NOTE: this is not to spec. I think spec is broken. See: - // - // https://github.com/ethereum/eth2.0-specs/issues/797 + let expected_previous_block_root = + Hash256::from_slice(&state.latest_block_header.signed_root()); verify!( - block.previous_block_root == *state.get_block_root(state.slot - 1, spec)?, - Invalid::ParentBlockRootMismatch + block.previous_block_root == expected_previous_block_root, + Invalid::ParentBlockRootMismatch { + state: expected_previous_block_root, + block: block.previous_block_root, + } ); state.latest_block_header = block.temporary_block_header(spec); @@ -122,7 +124,7 @@ pub fn process_block_header( /// Verifies the signature of a block. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn verify_block_signature( state: &BeaconState, block: &BeaconBlock, @@ -150,7 +152,7 @@ pub fn verify_block_signature( /// Verifies the `randao_reveal` against the block's proposer pubkey and updates /// `state.latest_randao_mixes`. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_randao( state: &mut BeaconState, block: &BeaconBlock, @@ -162,7 +164,7 @@ pub fn process_randao( // Verify the RANDAO is a valid signature of the proposer. verify!( block.body.randao_reveal.verify( - &state.current_epoch(spec).hash_tree_root()[..], + &state.current_epoch(spec).tree_hash_root()[..], spec.get_domain( block.slot.epoch(spec.slots_per_epoch), Domain::Randao, @@ -181,7 +183,7 @@ pub fn process_randao( /// Update the `state.eth1_data_votes` based upon the `eth1_data` provided. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_eth1_data(state: &mut BeaconState, eth1_data: &Eth1Data) -> Result<(), Error> { // Attempt to find a `Eth1DataVote` with matching `Eth1Data`. let matching_eth1_vote_index = state @@ -207,7 +209,7 @@ pub fn process_eth1_data(state: &mut BeaconState, eth1_data: &Eth1Data) -> Resul /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns /// an `Err` describing the invalid object or cause of failure. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_proposer_slashings( state: &mut BeaconState, proposer_slashings: &[ProposerSlashing], @@ -240,7 +242,7 @@ pub fn process_proposer_slashings( /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns /// an `Err` describing the invalid object or cause of failure. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_attester_slashings( state: &mut BeaconState, attester_slashings: &[AttesterSlashing], @@ -298,7 +300,7 @@ pub fn process_attester_slashings( /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns /// an `Err` describing the invalid object or cause of failure. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_attestations( state: &mut BeaconState, attestations: &[Attestation], @@ -340,7 +342,7 @@ pub fn process_attestations( /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns /// an `Err` describing the invalid object or cause of failure. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_deposits( state: &mut BeaconState, deposits: &[Deposit], @@ -410,7 +412,7 @@ pub fn process_deposits( /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns /// an `Err` describing the invalid object or cause of failure. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_exits( state: &mut BeaconState, voluntary_exits: &[VoluntaryExit], @@ -442,7 +444,7 @@ pub fn process_exits( /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns /// an `Err` describing the invalid object or cause of failure. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_transfers( state: &mut BeaconState, transfers: &[Transfer], diff --git a/eth2/state_processing/src/per_block_processing/errors.rs b/eth2/state_processing/src/per_block_processing/errors.rs index 6614f6f60..d8627d359 100644 --- a/eth2/state_processing/src/per_block_processing/errors.rs +++ b/eth2/state_processing/src/per_block_processing/errors.rs @@ -67,7 +67,10 @@ impl_from_beacon_state_error!(BlockProcessingError); #[derive(Debug, PartialEq)] pub enum BlockInvalid { StateSlotMismatch, - ParentBlockRootMismatch, + ParentBlockRootMismatch { + state: Hash256, + block: Hash256, + }, BadSignature, BadRandaoSignature, MaxAttestationsExceeded, @@ -271,10 +274,10 @@ pub enum ProposerSlashingValidationError { pub enum ProposerSlashingInvalid { /// The proposer index is not a known validator. ProposerUnknown(u64), - /// The two proposal have different slots. + /// The two proposal have different epochs. /// /// (proposal_1_slot, proposal_2_slot) - ProposalSlotMismatch(Slot, Slot), + ProposalEpochMismatch(Slot, Slot), /// The proposals are identical and therefore not slashable. ProposalsIdentical, /// The specified proposer has already been slashed. diff --git a/eth2/state_processing/src/per_block_processing/validate_attestation.rs b/eth2/state_processing/src/per_block_processing/validate_attestation.rs index 3b89bec99..438a75c94 100644 --- a/eth2/state_processing/src/per_block_processing/validate_attestation.rs +++ b/eth2/state_processing/src/per_block_processing/validate_attestation.rs @@ -1,6 +1,6 @@ use super::errors::{AttestationInvalid as Invalid, AttestationValidationError as Error}; use crate::common::verify_bitfield_length; -use ssz::TreeHash; +use tree_hash::TreeHash; use types::*; /// Indicates if an `Attestation` is valid to be included in a block in the current epoch of the @@ -8,7 +8,7 @@ use types::*; /// /// Returns `Ok(())` if the `Attestation` is valid, otherwise indicates the reason for invalidity. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn validate_attestation( state: &BeaconState, attestation: &Attestation, @@ -31,7 +31,7 @@ pub fn validate_attestation_time_independent_only( /// /// Returns `Ok(())` if the `Attestation` is valid, otherwise indicates the reason for invalidity. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn validate_attestation_without_signature( state: &BeaconState, attestation: &Attestation, @@ -44,7 +44,7 @@ pub fn validate_attestation_without_signature( /// given state, optionally validating the aggregate signature. /// /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn validate_attestation_parametric( state: &BeaconState, attestation: &Attestation, @@ -167,7 +167,7 @@ fn validate_attestation_parametric( /// Verify that the `source_epoch` and `source_root` of an `Attestation` correctly /// match the current (or previous) justified epoch and root from the state. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn verify_justified_epoch_and_root( attestation: &Attestation, state: &BeaconState, @@ -222,7 +222,7 @@ fn verify_justified_epoch_and_root( /// - `custody_bitfield` does not have a bit for each index of `committee`. /// - A `validator_index` in `committee` is not in `state.validator_registry`. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn verify_attestation_signature( state: &BeaconState, committee: &[usize], @@ -270,14 +270,14 @@ fn verify_attestation_signature( data: a.data.clone(), custody_bit: false, } - .hash_tree_root(); + .tree_hash_root(); // Message when custody bitfield is `true` let message_1 = AttestationDataAndCustodyBit { data: a.data.clone(), custody_bit: true, } - .hash_tree_root(); + .tree_hash_root(); let mut messages = vec![]; let mut keys = vec![]; diff --git a/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs b/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs index abf99da64..3527b62e3 100644 --- a/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs +++ b/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs @@ -7,7 +7,7 @@ use types::*; /// /// Returns `Ok(())` if the `AttesterSlashing` is valid, otherwise indicates the reason for invalidity. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn verify_attester_slashing( state: &BeaconState, attester_slashing: &AttesterSlashing, @@ -41,7 +41,7 @@ pub fn verify_attester_slashing( /// /// Returns Ok(indices) if `indices.len() > 0`. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn gather_attester_slashing_indices( state: &BeaconState, attester_slashing: &AttesterSlashing, diff --git a/eth2/state_processing/src/per_block_processing/verify_deposit.rs b/eth2/state_processing/src/per_block_processing/verify_deposit.rs index a3a0f5734..22a62a321 100644 --- a/eth2/state_processing/src/per_block_processing/verify_deposit.rs +++ b/eth2/state_processing/src/per_block_processing/verify_deposit.rs @@ -15,7 +15,7 @@ use types::*; /// /// Note: this function is incomplete. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn verify_deposit( state: &BeaconState, deposit: &Deposit, @@ -46,7 +46,7 @@ pub fn verify_deposit( /// Verify that the `Deposit` index is correct. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn verify_deposit_index(state: &BeaconState, deposit: &Deposit) -> Result<(), Error> { verify!( deposit.index == state.deposit_index, @@ -88,7 +88,7 @@ pub fn get_existing_validator_index( /// Verify that a deposit is included in the state's eth1 deposit root. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn verify_deposit_merkle_proof(state: &BeaconState, deposit: &Deposit, spec: &ChainSpec) -> bool { let leaf = hash(&get_serialized_deposit_data(deposit)); verify_merkle_proof( @@ -102,7 +102,7 @@ fn verify_deposit_merkle_proof(state: &BeaconState, deposit: &Deposit, spec: &Ch /// Helper struct for easily getting the serialized data generated by the deposit contract. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive(Encode)] struct SerializedDepositData { amount: u64, @@ -113,7 +113,7 @@ struct SerializedDepositData { /// Return the serialized data generated by the deposit contract that is used to generate the /// merkle proof. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn get_serialized_deposit_data(deposit: &Deposit) -> Vec { let serialized_deposit_data = SerializedDepositData { amount: deposit.deposit_data.amount, diff --git a/eth2/state_processing/src/per_block_processing/verify_exit.rs b/eth2/state_processing/src/per_block_processing/verify_exit.rs index a3b694395..697188ee9 100644 --- a/eth2/state_processing/src/per_block_processing/verify_exit.rs +++ b/eth2/state_processing/src/per_block_processing/verify_exit.rs @@ -1,5 +1,5 @@ use super::errors::{ExitInvalid as Invalid, ExitValidationError as Error}; -use ssz::SignedRoot; +use tree_hash::SignedRoot; use types::*; /// Indicates if an `Exit` is valid to be included in a block in the current epoch of the given @@ -7,7 +7,7 @@ use types::*; /// /// Returns `Ok(())` if the `Exit` is valid, otherwise indicates the reason for invalidity. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn verify_exit( state: &BeaconState, exit: &VoluntaryExit, diff --git a/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs b/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs index dffb9d898..bbc03dd62 100644 --- a/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs +++ b/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs @@ -1,5 +1,5 @@ use super::errors::{ProposerSlashingInvalid as Invalid, ProposerSlashingValidationError as Error}; -use ssz::SignedRoot; +use tree_hash::SignedRoot; use types::*; /// Indicates if a `ProposerSlashing` is valid to be included in a block in the current epoch of the given @@ -7,7 +7,7 @@ use types::*; /// /// Returns `Ok(())` if the `ProposerSlashing` is valid, otherwise indicates the reason for invalidity. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn verify_proposer_slashing( proposer_slashing: &ProposerSlashing, state: &BeaconState, @@ -21,8 +21,9 @@ pub fn verify_proposer_slashing( })?; verify!( - proposer_slashing.header_1.slot == proposer_slashing.header_2.slot, - Invalid::ProposalSlotMismatch( + proposer_slashing.header_1.slot.epoch(spec.slots_per_epoch) + == proposer_slashing.header_2.slot.epoch(spec.slots_per_epoch), + Invalid::ProposalEpochMismatch( proposer_slashing.header_1.slot, proposer_slashing.header_2.slot ) @@ -66,7 +67,7 @@ pub fn verify_proposer_slashing( /// /// Returns `true` if the signature is valid. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn verify_header_signature( header: &BeaconBlockHeader, pubkey: &PublicKey, diff --git a/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs b/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs index d3ab5e398..89cb93ce5 100644 --- a/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs +++ b/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs @@ -2,7 +2,7 @@ use super::errors::{ SlashableAttestationInvalid as Invalid, SlashableAttestationValidationError as Error, }; use crate::common::verify_bitfield_length; -use ssz::TreeHash; +use tree_hash::TreeHash; use types::*; /// Indicates if a `SlashableAttestation` is valid to be included in a block in the current epoch of the given @@ -10,7 +10,7 @@ use types::*; /// /// Returns `Ok(())` if the `SlashableAttestation` is valid, otherwise indicates the reason for invalidity. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn verify_slashable_attestation( state: &BeaconState, slashable_attestation: &SlashableAttestation, @@ -77,12 +77,12 @@ pub fn verify_slashable_attestation( data: slashable_attestation.data.clone(), custody_bit: false, } - .hash_tree_root(); + .tree_hash_root(); let message_1 = AttestationDataAndCustodyBit { data: slashable_attestation.data.clone(), custody_bit: true, } - .hash_tree_root(); + .tree_hash_root(); let mut messages = vec![]; let mut keys = vec![]; diff --git a/eth2/state_processing/src/per_block_processing/verify_transfer.rs b/eth2/state_processing/src/per_block_processing/verify_transfer.rs index ac9e9aa09..8b0415508 100644 --- a/eth2/state_processing/src/per_block_processing/verify_transfer.rs +++ b/eth2/state_processing/src/per_block_processing/verify_transfer.rs @@ -1,6 +1,6 @@ use super::errors::{TransferInvalid as Invalid, TransferValidationError as Error}; use bls::get_withdrawal_credentials; -use ssz::SignedRoot; +use tree_hash::SignedRoot; use types::*; /// Indicates if a `Transfer` is valid to be included in a block in the current epoch of the given @@ -10,7 +10,7 @@ use types::*; /// /// Note: this function is incomplete. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn verify_transfer( state: &BeaconState, transfer: &Transfer, @@ -122,7 +122,7 @@ fn verify_transfer_parametric( /// /// Does not check that the transfer is valid, however checks for overflow in all actions. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn execute_transfer( state: &mut BeaconState, transfer: &Transfer, diff --git a/eth2/state_processing/src/per_epoch_processing.rs b/eth2/state_processing/src/per_epoch_processing.rs index fcdc668f4..87c9b9398 100644 --- a/eth2/state_processing/src/per_epoch_processing.rs +++ b/eth2/state_processing/src/per_epoch_processing.rs @@ -3,8 +3,8 @@ use errors::EpochProcessingError as Error; use process_ejections::process_ejections; use process_exit_queue::process_exit_queue; use process_slashings::process_slashings; -use ssz::TreeHash; use std::collections::HashMap; +use tree_hash::TreeHash; use types::*; use update_registry_and_shuffling_data::update_registry_and_shuffling_data; use validator_statuses::{TotalBalances, ValidatorStatuses}; @@ -32,7 +32,7 @@ pub type WinningRootHashSet = HashMap; /// Mutates the given `BeaconState`, returning early if an error is encountered. If an error is /// returned, a state might be "half-processed" and therefore in an invalid state. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn per_epoch_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> { // Ensure the previous and next epoch caches are built. state.build_epoch_cache(RelativeEpoch::Previous, spec)?; @@ -86,7 +86,7 @@ pub fn per_epoch_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result /// Maybe resets the eth1 period. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn maybe_reset_eth1_period(state: &mut BeaconState, spec: &ChainSpec) { let next_epoch = state.next_epoch(spec); let voting_period = spec.epochs_per_eth1_voting_period; @@ -108,7 +108,7 @@ pub fn maybe_reset_eth1_period(state: &mut BeaconState, spec: &ChainSpec) { /// - `justified_epoch` /// - `previous_justified_epoch` /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn update_justification_and_finalization( state: &mut BeaconState, total_balances: &TotalBalances, @@ -178,7 +178,7 @@ pub fn update_justification_and_finalization( /// /// Also returns a `WinningRootHashSet` for later use during epoch processing. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_crosslinks( state: &mut BeaconState, spec: &ChainSpec, @@ -221,7 +221,7 @@ pub fn process_crosslinks( /// Finish up an epoch update. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn finish_epoch_update(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> { let current_epoch = state.current_epoch(spec); let next_epoch = state.next_epoch(spec); @@ -236,7 +236,7 @@ pub fn finish_epoch_update(state: &mut BeaconState, spec: &ChainSpec) -> Result< let active_index_root = Hash256::from_slice( &state .get_active_validator_indices(next_epoch + spec.activation_exit_delay) - .hash_tree_root()[..], + .tree_hash_root()[..], ); state.set_active_index_root(next_epoch, active_index_root, spec)?; @@ -261,7 +261,7 @@ pub fn finish_epoch_update(state: &mut BeaconState, spec: &ChainSpec) -> Result< let historical_batch: HistoricalBatch = state.historical_batch(); state .historical_roots - .push(Hash256::from_slice(&historical_batch.hash_tree_root()[..])); + .push(Hash256::from_slice(&historical_batch.tree_hash_root()[..])); } state.previous_epoch_attestations = state.current_epoch_attestations.clone(); diff --git a/eth2/state_processing/src/per_epoch_processing/apply_rewards.rs b/eth2/state_processing/src/per_epoch_processing/apply_rewards.rs index ce5fccb21..9af1ee8c3 100644 --- a/eth2/state_processing/src/per_epoch_processing/apply_rewards.rs +++ b/eth2/state_processing/src/per_epoch_processing/apply_rewards.rs @@ -32,7 +32,7 @@ impl std::ops::AddAssign for Delta { /// Apply attester and proposer rewards. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn apply_rewards( state: &mut BeaconState, validator_statuses: &mut ValidatorStatuses, @@ -79,7 +79,7 @@ pub fn apply_rewards( /// Applies the attestation inclusion reward to each proposer for every validator who included an /// attestation in the previous epoch. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn get_proposer_deltas( deltas: &mut Vec, state: &mut BeaconState, @@ -120,7 +120,7 @@ fn get_proposer_deltas( /// Apply rewards for participation in attestations during the previous epoch. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn get_justification_and_finalization_deltas( deltas: &mut Vec, state: &BeaconState, @@ -163,7 +163,7 @@ fn get_justification_and_finalization_deltas( /// Determine the delta for a single validator, if the chain is finalizing normally. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn compute_normal_justification_and_finalization_delta( validator: &ValidatorStatus, total_balances: &TotalBalances, @@ -215,7 +215,7 @@ fn compute_normal_justification_and_finalization_delta( /// Determine the delta for a single delta, assuming the chain is _not_ finalizing normally. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn compute_inactivity_leak_delta( validator: &ValidatorStatus, base_reward: u64, @@ -261,7 +261,7 @@ fn compute_inactivity_leak_delta( /// Calculate the deltas based upon the winning roots for attestations during the previous epoch. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn get_crosslink_deltas( deltas: &mut Vec, state: &BeaconState, @@ -295,7 +295,7 @@ fn get_crosslink_deltas( /// Returns the base reward for some validator. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn get_base_reward( state: &BeaconState, index: usize, @@ -312,7 +312,7 @@ fn get_base_reward( /// Returns the inactivity penalty for some validator. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn get_inactivity_penalty( state: &BeaconState, index: usize, @@ -328,7 +328,7 @@ fn get_inactivity_penalty( /// Returns the epochs since the last finalized epoch. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn epochs_since_finality(state: &BeaconState, spec: &ChainSpec) -> Epoch { state.current_epoch(spec) + 1 - state.finalized_epoch } diff --git a/eth2/state_processing/src/per_epoch_processing/get_attestation_participants.rs b/eth2/state_processing/src/per_epoch_processing/get_attestation_participants.rs index 52ba0274b..bea772204 100644 --- a/eth2/state_processing/src/per_epoch_processing/get_attestation_participants.rs +++ b/eth2/state_processing/src/per_epoch_processing/get_attestation_participants.rs @@ -3,7 +3,7 @@ use types::*; /// Returns validator indices which participated in the attestation. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn get_attestation_participants( state: &BeaconState, attestation_data: &AttestationData, diff --git a/eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs b/eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs index b52485947..6b221f513 100644 --- a/eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs +++ b/eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs @@ -5,7 +5,7 @@ use types::*; /// Returns the distance between the first included attestation for some validator and this /// slot. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn inclusion_distance( state: &BeaconState, attestations: &[&PendingAttestation], @@ -18,7 +18,7 @@ pub fn inclusion_distance( /// Returns the slot of the earliest included attestation for some validator. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn inclusion_slot( state: &BeaconState, attestations: &[&PendingAttestation], @@ -31,7 +31,7 @@ pub fn inclusion_slot( /// Finds the earliest included attestation for some validator. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn earliest_included_attestation( state: &BeaconState, attestations: &[&PendingAttestation], diff --git a/eth2/state_processing/src/per_epoch_processing/process_ejections.rs b/eth2/state_processing/src/per_epoch_processing/process_ejections.rs index a60d92187..6f64c46f7 100644 --- a/eth2/state_processing/src/per_epoch_processing/process_ejections.rs +++ b/eth2/state_processing/src/per_epoch_processing/process_ejections.rs @@ -4,7 +4,7 @@ use types::{BeaconStateError as Error, *}; /// Iterate through the validator registry and eject active validators with balance below /// ``EJECTION_BALANCE``. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_ejections(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> { // There is an awkward double (triple?) loop here because we can't loop across the borrowed // active validator indices and mutate state in the one loop. diff --git a/eth2/state_processing/src/per_epoch_processing/process_exit_queue.rs b/eth2/state_processing/src/per_epoch_processing/process_exit_queue.rs index 074db1d08..a6362188d 100644 --- a/eth2/state_processing/src/per_epoch_processing/process_exit_queue.rs +++ b/eth2/state_processing/src/per_epoch_processing/process_exit_queue.rs @@ -2,7 +2,7 @@ use types::*; /// Process the exit queue. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_exit_queue(state: &mut BeaconState, spec: &ChainSpec) { let current_epoch = state.current_epoch(spec); @@ -31,7 +31,7 @@ pub fn process_exit_queue(state: &mut BeaconState, spec: &ChainSpec) { /// Initiate an exit for the validator of the given `index`. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn prepare_validator_for_withdrawal( state: &mut BeaconState, validator_index: usize, diff --git a/eth2/state_processing/src/per_epoch_processing/process_slashings.rs b/eth2/state_processing/src/per_epoch_processing/process_slashings.rs index 88777472c..89a7dd484 100644 --- a/eth2/state_processing/src/per_epoch_processing/process_slashings.rs +++ b/eth2/state_processing/src/per_epoch_processing/process_slashings.rs @@ -2,7 +2,7 @@ use types::{BeaconStateError as Error, *}; /// Process slashings. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn process_slashings( state: &mut BeaconState, current_total_balance: u64, diff --git a/eth2/state_processing/src/per_epoch_processing/update_registry_and_shuffling_data.rs b/eth2/state_processing/src/per_epoch_processing/update_registry_and_shuffling_data.rs index 0b18c2571..d290d2987 100644 --- a/eth2/state_processing/src/per_epoch_processing/update_registry_and_shuffling_data.rs +++ b/eth2/state_processing/src/per_epoch_processing/update_registry_and_shuffling_data.rs @@ -4,7 +4,7 @@ use types::*; /// Peforms a validator registry update, if required. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn update_registry_and_shuffling_data( state: &mut BeaconState, current_total_balance: u64, @@ -49,7 +49,7 @@ pub fn update_registry_and_shuffling_data( /// Returns `true` if the validator registry should be updated during an epoch processing. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn should_update_validator_registry( state: &BeaconState, spec: &ChainSpec, @@ -78,7 +78,7 @@ pub fn should_update_validator_registry( /// /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn update_validator_registry( state: &mut BeaconState, current_total_balance: u64, @@ -133,7 +133,7 @@ pub fn update_validator_registry( /// Activate the validator of the given ``index``. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn activate_validator( state: &mut BeaconState, validator_index: usize, diff --git a/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs b/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs index 02149cc5a..afa78c9c0 100644 --- a/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs +++ b/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs @@ -160,7 +160,7 @@ impl ValidatorStatuses { /// - Active validators /// - Total balances for the current and previous epochs. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn new(state: &BeaconState, spec: &ChainSpec) -> Result { let mut statuses = Vec::with_capacity(state.validator_registry.len()); let mut total_balances = TotalBalances::default(); @@ -195,7 +195,7 @@ impl ValidatorStatuses { /// Process some attestations from the given `state` updating the `statuses` and /// `total_balances` fields. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn process_attestations( &mut self, state: &BeaconState, @@ -261,7 +261,7 @@ impl ValidatorStatuses { /// Update the `statuses` for each validator based upon whether or not they attested to the /// "winning" shard block root for the previous epoch. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn process_winning_roots( &mut self, state: &BeaconState, @@ -297,14 +297,14 @@ impl ValidatorStatuses { /// Returns the distance between when the attestation was created and when it was included in a /// block. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn inclusion_distance(a: &PendingAttestation) -> Slot { a.inclusion_slot - a.data.slot } /// Returns `true` if some `PendingAttestation` is from the supplied `epoch`. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn is_from_epoch(a: &PendingAttestation, epoch: Epoch, spec: &ChainSpec) -> bool { a.data.slot.epoch(spec.slots_per_epoch) == epoch } @@ -312,7 +312,7 @@ fn is_from_epoch(a: &PendingAttestation, epoch: Epoch, spec: &ChainSpec) -> bool /// Returns `true` if a `PendingAttestation` and `BeaconState` share the same beacon block hash for /// the first slot of the given epoch. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn has_common_epoch_boundary_root( a: &PendingAttestation, state: &BeaconState, @@ -328,7 +328,7 @@ fn has_common_epoch_boundary_root( /// Returns `true` if a `PendingAttestation` and `BeaconState` share the same beacon block hash for /// the current slot of the `PendingAttestation`. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn has_common_beacon_block_root( a: &PendingAttestation, state: &BeaconState, diff --git a/eth2/state_processing/src/per_epoch_processing/winning_root.rs b/eth2/state_processing/src/per_epoch_processing/winning_root.rs index 97cff3e13..5d31dff31 100644 --- a/eth2/state_processing/src/per_epoch_processing/winning_root.rs +++ b/eth2/state_processing/src/per_epoch_processing/winning_root.rs @@ -16,7 +16,7 @@ impl WinningRoot { /// A winning root is "better" than another if it has a higher `total_attesting_balance`. Ties /// are broken by favouring the higher `crosslink_data_root` value. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn is_better_than(&self, other: &Self) -> bool { if self.total_attesting_balance > other.total_attesting_balance { true @@ -34,7 +34,7 @@ impl WinningRoot { /// The `WinningRoot` object also contains additional fields that are useful in later stages of /// per-epoch processing. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn winning_root( state: &BeaconState, shard: u64, @@ -89,7 +89,7 @@ pub fn winning_root( /// Returns `true` if pending attestation `a` is eligible to become a winning root. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn is_eligible_for_winning_root(state: &BeaconState, a: &PendingAttestation, shard: Shard) -> bool { if shard >= state.latest_crosslinks.len() as u64 { return false; @@ -100,7 +100,7 @@ fn is_eligible_for_winning_root(state: &BeaconState, a: &PendingAttestation, sha /// Returns all indices which voted for a given crosslink. Does not contain duplicates. /// -/// Spec v0.5.0 +/// Spec v0.5.1 fn get_attesting_validator_indices( state: &BeaconState, shard: u64, diff --git a/eth2/state_processing/src/per_slot_processing.rs b/eth2/state_processing/src/per_slot_processing.rs index c6b5312c7..a68f98c6d 100644 --- a/eth2/state_processing/src/per_slot_processing.rs +++ b/eth2/state_processing/src/per_slot_processing.rs @@ -1,5 +1,5 @@ use crate::*; -use ssz::TreeHash; +use tree_hash::{SignedRoot, TreeHash}; use types::*; #[derive(Debug, PartialEq)] @@ -10,13 +10,9 @@ pub enum Error { /// Advances a state forward by one slot, performing per-epoch processing if required. /// -/// Spec v0.5.0 -pub fn per_slot_processing( - state: &mut BeaconState, - latest_block_header: &BeaconBlockHeader, - spec: &ChainSpec, -) -> Result<(), Error> { - cache_state(state, latest_block_header, spec)?; +/// Spec v0.5.1 +pub fn per_slot_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> { + cache_state(state, spec)?; if (state.slot + 1) % spec.slots_per_epoch == 0 { per_epoch_processing(state, spec)?; @@ -27,12 +23,8 @@ pub fn per_slot_processing( Ok(()) } -fn cache_state( - state: &mut BeaconState, - latest_block_header: &BeaconBlockHeader, - spec: &ChainSpec, -) -> Result<(), Error> { - let previous_slot_state_root = Hash256::from_slice(&state.hash_tree_root()[..]); +fn cache_state(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> { + let previous_slot_state_root = Hash256::from_slice(&state.tree_hash_root()[..]); // Note: increment the state slot here to allow use of our `state_root` and `block_root` // getter/setter functions. @@ -46,7 +38,10 @@ fn cache_state( state.latest_block_header.state_root = previous_slot_state_root } - let latest_block_root = Hash256::from_slice(&latest_block_header.hash_tree_root()[..]); + // Store the previous slot's post state transition root. + state.set_state_root(previous_slot, previous_slot_state_root, spec)?; + + let latest_block_root = Hash256::from_slice(&state.latest_block_header.signed_root()[..]); state.set_block_root(previous_slot, latest_block_root, spec)?; // Set the state slot back to what it should be. diff --git a/eth2/state_processing/tests/tests.rs b/eth2/state_processing/tests/tests.rs index 25613f9a7..e80340e9a 100644 --- a/eth2/state_processing/tests/tests.rs +++ b/eth2/state_processing/tests/tests.rs @@ -2,14 +2,58 @@ use serde_derive::Deserialize; use serde_yaml; -use state_processing::{ - per_block_processing, per_block_processing_without_verifying_block_signature, - per_slot_processing, -}; +use state_processing::{per_block_processing, per_slot_processing}; use std::{fs::File, io::prelude::*, path::PathBuf}; use types::*; -#[allow(unused_imports)] -use yaml_utils; + +#[derive(Debug, Deserialize)] +pub struct ExpectedState { + pub slot: Option, + pub genesis_time: Option, + pub fork: Option, + pub validator_registry: Option>, + pub validator_balances: Option>, + pub previous_epoch_attestations: Option>, + pub current_epoch_attestations: Option>, + pub historical_roots: Option>, + pub finalized_epoch: Option, + pub latest_block_roots: Option>, +} + +impl ExpectedState { + // Return a list of fields that differ, and a string representation of the beacon state's field. + fn check(&self, state: &BeaconState) -> Vec<(&str, String)> { + // Check field equality + macro_rules! cfe { + ($field_name:ident) => { + if self.$field_name.as_ref().map_or(true, |$field_name| { + println!(" > Checking {}", stringify!($field_name)); + $field_name == &state.$field_name + }) { + vec![] + } else { + vec![(stringify!($field_name), format!("{:#?}", state.$field_name))] + } + }; + } + + vec![ + cfe!(slot), + cfe!(genesis_time), + cfe!(fork), + cfe!(validator_registry), + cfe!(validator_balances), + cfe!(previous_epoch_attestations), + cfe!(current_epoch_attestations), + cfe!(historical_roots), + cfe!(finalized_epoch), + cfe!(latest_block_roots), + ] + .into_iter() + .flat_map(|x| x) + .collect() + } +} #[derive(Debug, Deserialize)] pub struct TestCase { @@ -18,6 +62,7 @@ pub struct TestCase { pub verify_signatures: bool, pub initial_state: BeaconState, pub blocks: Vec, + pub expected_state: ExpectedState, } #[derive(Debug, Deserialize)] @@ -28,81 +73,81 @@ pub struct TestDoc { pub test_cases: Vec, } -#[test] -fn test_read_yaml() { - // Test sanity-check_small-config_32-vals.yaml +fn load_test_case(test_name: &str) -> TestDoc { let mut file = { let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - file_path_buf.push("yaml_utils/specs/sanity-check_small-config_32-vals.yaml"); + file_path_buf.push(format!("yaml_utils/specs/{}", test_name)); File::open(file_path_buf).unwrap() }; - let mut yaml_str = String::new(); - file.read_to_string(&mut yaml_str).unwrap(); - yaml_str = yaml_str.to_lowercase(); - let _doc: TestDoc = serde_yaml::from_str(&yaml_str.as_str()).unwrap(); - - // Test sanity-check_default-config_100-vals.yaml - file = { - let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - file_path_buf.push("yaml_utils/specs/sanity-check_default-config_100-vals.yaml"); - - File::open(file_path_buf).unwrap() - }; - - yaml_str = String::new(); - - file.read_to_string(&mut yaml_str).unwrap(); - - yaml_str = yaml_str.to_lowercase(); - - let _doc: TestDoc = serde_yaml::from_str(&yaml_str.as_str()).unwrap(); + serde_yaml::from_str(&yaml_str.as_str()).unwrap() } -#[test] -fn run_state_transition_tests_small() { - // Test sanity-check_small-config_32-vals.yaml - let mut file = { - let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - file_path_buf.push("yaml_utils/specs/sanity-check_small-config_32-vals.yaml"); - - File::open(file_path_buf).unwrap() - }; - let mut yaml_str = String::new(); - file.read_to_string(&mut yaml_str).unwrap(); - yaml_str = yaml_str.to_lowercase(); - - let doc: TestDoc = serde_yaml::from_str(&yaml_str.as_str()).unwrap(); +fn run_state_transition_test(test_name: &str) { + let doc = load_test_case(test_name); // Run Tests + let mut ok = true; for (i, test_case) in doc.test_cases.iter().enumerate() { + let fake_crypto = cfg!(feature = "fake_crypto"); + if !test_case.verify_signatures == fake_crypto { + println!("Running {}", test_case.name); + } else { + println!( + "Skipping {} (fake_crypto: {}, need fake: {})", + test_case.name, fake_crypto, !test_case.verify_signatures + ); + continue; + } let mut state = test_case.initial_state.clone(); - for block in test_case.blocks.iter() { + for (j, block) in test_case.blocks.iter().enumerate() { while block.slot > state.slot { - let latest_block_header = state.latest_block_header.clone(); - per_slot_processing(&mut state, &latest_block_header, &test_case.config).unwrap(); + per_slot_processing(&mut state, &test_case.config).unwrap(); } - if test_case.verify_signatures { - let res = per_block_processing(&mut state, &block, &test_case.config); - if res.is_err() { - println!("{:?}", i); - println!("{:?}", res); - }; - } else { - let res = per_block_processing_without_verifying_block_signature( - &mut state, - &block, - &test_case.config, - ); - if res.is_err() { - println!("{:?}", i); - println!("{:?}", res); - } + let res = per_block_processing(&mut state, &block, &test_case.config); + if res.is_err() { + println!("Error in {} (#{}), on block {}", test_case.name, i, j); + println!("{:?}", res); + ok = false; } } + + let mismatched_fields = test_case.expected_state.check(&state); + if !mismatched_fields.is_empty() { + println!( + "Error in expected state, these fields didn't match: {:?}", + mismatched_fields.iter().map(|(f, _)| f).collect::>() + ); + for (field_name, state_val) in mismatched_fields { + println!("state.{} was: {}", field_name, state_val); + } + ok = false; + } } + + assert!(ok, "one or more tests failed, see above"); +} + +#[test] +#[cfg(not(debug_assertions))] +fn test_read_yaml() { + load_test_case("sanity-check_small-config_32-vals.yaml"); + load_test_case("sanity-check_default-config_100-vals.yaml"); +} + +#[test] +#[cfg(not(debug_assertions))] +fn run_state_transition_tests_small() { + run_state_transition_test("sanity-check_small-config_32-vals.yaml"); +} + +// Run with --ignored to run this test +#[test] +#[ignore] +fn run_state_transition_tests_large() { + run_state_transition_test("sanity-check_default-config_100-vals.yaml"); } diff --git a/eth2/state_processing/yaml_utils/Cargo.toml b/eth2/state_processing/yaml_utils/Cargo.toml index 4a7ae5b89..5f216fe1a 100644 --- a/eth2/state_processing/yaml_utils/Cargo.toml +++ b/eth2/state_processing/yaml_utils/Cargo.toml @@ -6,7 +6,6 @@ edition = "2018" [build-dependencies] reqwest = "0.9" -tempdir = "0.3" [dependencies] diff --git a/eth2/state_processing/yaml_utils/build.rs b/eth2/state_processing/yaml_utils/build.rs index 3b7f31471..7fb652cc1 100644 --- a/eth2/state_processing/yaml_utils/build.rs +++ b/eth2/state_processing/yaml_utils/build.rs @@ -1,5 +1,4 @@ extern crate reqwest; -extern crate tempdir; use std::fs::File; use std::io::copy; diff --git a/eth2/state_processing/yaml_utils/expected_state_fields.py b/eth2/state_processing/yaml_utils/expected_state_fields.py new file mode 100755 index 000000000..df4cb83f7 --- /dev/null +++ b/eth2/state_processing/yaml_utils/expected_state_fields.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 + +# Script to extract all the fields of the state mentioned in `expected_state` fields of tests +# in the `spec` directory. These fields can then be added to the `ExpectedState` struct. +# Might take a while to run. + +import os, yaml + +if __name__ == "__main__": + yaml_files = (filename for filename in os.listdir("specs") if filename.endswith(".yaml")) + parsed_yaml = (yaml.load(open("specs/" + filename, "r")) for filename in yaml_files) + all_fields = set() + for y in parsed_yaml: + all_fields.update(*({key for key in case["expected_state"]} for case in y["test_cases"])) + print(all_fields) diff --git a/eth2/types/Cargo.toml b/eth2/types/Cargo.toml index 613eb7936..b88e1d4cf 100644 --- a/eth2/types/Cargo.toml +++ b/eth2/types/Cargo.toml @@ -26,6 +26,8 @@ ssz = { path = "../utils/ssz" } ssz_derive = { path = "../utils/ssz_derive" } swap_or_not_shuffle = { path = "../utils/swap_or_not_shuffle" } test_random_derive = { path = "../utils/test_random_derive" } +tree_hash = { path = "../utils/tree_hash" } +tree_hash_derive = { path = "../utils/tree_hash_derive" } libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "b3c32d9a821ae6cc89079499cc6e8a6bab0bffc3" } [dev-dependencies] diff --git a/eth2/types/src/attestation.rs b/eth2/types/src/attestation.rs index a8eeea909..f7bfdaab9 100644 --- a/eth2/types/src/attestation.rs +++ b/eth2/types/src/attestation.rs @@ -2,13 +2,14 @@ use super::{AggregateSignature, AttestationData, Bitfield}; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz::TreeHash; -use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash::TreeHash; +use tree_hash_derive::{SignedRoot, TreeHash}; /// Details an attestation that can be slashable. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( Debug, Clone, diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data.rs index 4a6b57823..f8a0ecd15 100644 --- a/eth2/types/src/attestation_data.rs +++ b/eth2/types/src/attestation_data.rs @@ -2,13 +2,14 @@ use crate::test_utils::TestRandom; use crate::{Crosslink, Epoch, Hash256, Slot}; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz::TreeHash; -use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash::TreeHash; +use tree_hash_derive::{SignedRoot, TreeHash}; /// The data upon which an attestation is based. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( Debug, Clone, diff --git a/eth2/types/src/attestation_data_and_custody_bit.rs b/eth2/types/src/attestation_data_and_custody_bit.rs index 2cc6bc80c..e5dc920dc 100644 --- a/eth2/types/src/attestation_data_and_custody_bit.rs +++ b/eth2/types/src/attestation_data_and_custody_bit.rs @@ -2,11 +2,12 @@ use super::AttestationData; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; +use tree_hash_derive::TreeHash; /// Used for pairing an attestation with a proof-of-custody. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode, TreeHash)] pub struct AttestationDataAndCustodyBit { pub data: AttestationData, diff --git a/eth2/types/src/attester_slashing.rs b/eth2/types/src/attester_slashing.rs index 6fc404f42..b5e851dbd 100644 --- a/eth2/types/src/attester_slashing.rs +++ b/eth2/types/src/attester_slashing.rs @@ -1,12 +1,13 @@ use crate::{test_utils::TestRandom, SlashableAttestation}; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::TreeHash; /// Two conflicting attestations. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] pub struct AttesterSlashing { pub slashable_attestation_1: SlashableAttestation, diff --git a/eth2/types/src/beacon_block.rs b/eth2/types/src/beacon_block.rs index 77c1620f3..b4d2752d6 100644 --- a/eth2/types/src/beacon_block.rs +++ b/eth2/types/src/beacon_block.rs @@ -3,13 +3,14 @@ use crate::*; use bls::Signature; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz::TreeHash; -use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash::TreeHash; +use tree_hash_derive::{SignedRoot, TreeHash}; /// A block of the `BeaconChain`. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( Debug, PartialEq, @@ -34,7 +35,7 @@ pub struct BeaconBlock { impl BeaconBlock { /// Returns an empty block to be used during genesis. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn empty(spec: &ChainSpec) -> BeaconBlock { BeaconBlock { slot: spec.genesis_slot, @@ -57,11 +58,11 @@ impl BeaconBlock { } } - /// Returns the `hash_tree_root` of the block. + /// Returns the `tree_hash_root | update` of the block. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn canonical_root(&self) -> Hash256 { - Hash256::from_slice(&self.hash_tree_root()[..]) + Hash256::from_slice(&self.tree_hash_root()[..]) } /// Returns a full `BeaconBlockHeader` of this block. @@ -71,20 +72,20 @@ impl BeaconBlock { /// /// Note: performs a full tree-hash of `self.body`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn block_header(&self) -> BeaconBlockHeader { BeaconBlockHeader { slot: self.slot, previous_block_root: self.previous_block_root, state_root: self.state_root, - block_body_root: Hash256::from_slice(&self.body.hash_tree_root()[..]), + block_body_root: Hash256::from_slice(&self.body.tree_hash_root()[..]), signature: self.signature.clone(), } } /// Returns a "temporary" header, where the `state_root` is `spec.zero_hash`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn temporary_block_header(&self, spec: &ChainSpec) -> BeaconBlockHeader { BeaconBlockHeader { state_root: spec.zero_hash, diff --git a/eth2/types/src/beacon_block_body.rs b/eth2/types/src/beacon_block_body.rs index 677e24cec..de4951f1f 100644 --- a/eth2/types/src/beacon_block_body.rs +++ b/eth2/types/src/beacon_block_body.rs @@ -2,12 +2,13 @@ use crate::test_utils::TestRandom; use crate::*; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::TreeHash; /// The body of a `BeaconChain` block, containing operations. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] pub struct BeaconBlockBody { pub randao_reveal: Signature, diff --git a/eth2/types/src/beacon_block_header.rs b/eth2/types/src/beacon_block_header.rs index 090d0a965..e4db3a721 100644 --- a/eth2/types/src/beacon_block_header.rs +++ b/eth2/types/src/beacon_block_header.rs @@ -3,13 +3,14 @@ use crate::*; use bls::Signature; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz::TreeHash; -use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash::{SignedRoot, TreeHash}; +use tree_hash_derive::{SignedRoot, TreeHash}; /// A header of a `BeaconBlock`. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( Debug, PartialEq, @@ -32,16 +33,16 @@ pub struct BeaconBlockHeader { } impl BeaconBlockHeader { - /// Returns the `hash_tree_root` of the header. + /// Returns the `tree_hash_root` of the header. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn canonical_root(&self) -> Hash256 { - Hash256::from_slice(&self.hash_tree_root()[..]) + Hash256::from_slice(&self.signed_root()[..]) } /// Given a `body`, consumes `self` and returns a complete `BeaconBlock`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn into_block(self, body: BeaconBlockBody) -> BeaconBlock { BeaconBlock { slot: self.slot, diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 774e8eb76..eef408308 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -5,9 +5,11 @@ use int_to_bytes::int_to_bytes32; use pubkey_cache::PubkeyCache; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz::{hash, ssz_encode, TreeHash}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz::{hash, ssz_encode}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash::TreeHash; +use tree_hash_derive::TreeHash; mod epoch_cache; mod pubkey_cache; @@ -44,7 +46,7 @@ pub enum Error { /// The state of the `BeaconChain` at some slot. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, TestRandom, Encode, Decode, TreeHash)] pub struct BeaconState { // Misc @@ -58,7 +60,7 @@ pub struct BeaconState { pub validator_registry_update_epoch: Epoch, // Randomness and committees - pub latest_randao_mixes: Vec, + pub latest_randao_mixes: TreeHashVector, pub previous_shuffling_start_shard: u64, pub current_shuffling_start_shard: u64, pub previous_shuffling_epoch: Epoch, @@ -78,11 +80,11 @@ pub struct BeaconState { pub finalized_root: Hash256, // Recent state - pub latest_crosslinks: Vec, - latest_block_roots: Vec, - latest_state_roots: Vec, - latest_active_index_roots: Vec, - latest_slashed_balances: Vec, + pub latest_crosslinks: TreeHashVector, + pub latest_block_roots: TreeHashVector, + latest_state_roots: TreeHashVector, + latest_active_index_roots: TreeHashVector, + latest_slashed_balances: TreeHashVector, pub latest_block_header: BeaconBlockHeader, pub historical_roots: Vec, @@ -118,7 +120,7 @@ impl BeaconState { /// This does not fully build a genesis beacon state, it omits processing of initial validator /// deposits. To obtain a full genesis beacon state, use the `BeaconStateBuilder`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn genesis(genesis_time: u64, latest_eth1_data: Eth1Data, spec: &ChainSpec) -> BeaconState { let initial_crosslink = Crosslink { epoch: spec.genesis_epoch, @@ -137,7 +139,8 @@ impl BeaconState { validator_registry_update_epoch: spec.genesis_epoch, // Randomness and committees - latest_randao_mixes: vec![spec.zero_hash; spec.latest_randao_mixes_length as usize], + latest_randao_mixes: vec![spec.zero_hash; spec.latest_randao_mixes_length as usize] + .into(), previous_shuffling_start_shard: spec.genesis_start_shard, current_shuffling_start_shard: spec.genesis_start_shard, previous_shuffling_epoch: spec.genesis_epoch, @@ -157,11 +160,12 @@ impl BeaconState { finalized_root: spec.zero_hash, // Recent state - latest_crosslinks: vec![initial_crosslink; spec.shard_count as usize], - latest_block_roots: vec![spec.zero_hash; spec.slots_per_historical_root], - latest_state_roots: vec![spec.zero_hash; spec.slots_per_historical_root], - latest_active_index_roots: vec![spec.zero_hash; spec.latest_active_index_roots_length], - latest_slashed_balances: vec![0; spec.latest_slashed_exit_length], + latest_crosslinks: vec![initial_crosslink; spec.shard_count as usize].into(), + latest_block_roots: vec![spec.zero_hash; spec.slots_per_historical_root].into(), + latest_state_roots: vec![spec.zero_hash; spec.slots_per_historical_root].into(), + latest_active_index_roots: vec![spec.zero_hash; spec.latest_active_index_roots_length] + .into(), + latest_slashed_balances: vec![0; spec.latest_slashed_exit_length].into(), latest_block_header: BeaconBlock::empty(spec).temporary_block_header(spec), historical_roots: vec![], @@ -186,11 +190,11 @@ impl BeaconState { } } - /// Returns the `hash_tree_root` of the state. + /// Returns the `tree_hash_root` of the state. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn canonical_root(&self) -> Hash256 { - Hash256::from_slice(&self.hash_tree_root()[..]) + Hash256::from_slice(&self.tree_hash_root()[..]) } pub fn historical_batch(&self) -> HistoricalBatch { @@ -217,7 +221,7 @@ impl BeaconState { /// The epoch corresponding to `self.slot`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn current_epoch(&self, spec: &ChainSpec) -> Epoch { self.slot.epoch(spec.slots_per_epoch) } @@ -226,14 +230,14 @@ impl BeaconState { /// /// If the current epoch is the genesis epoch, the genesis_epoch is returned. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn previous_epoch(&self, spec: &ChainSpec) -> Epoch { self.current_epoch(&spec) - 1 } /// The epoch following `self.current_epoch()`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn next_epoch(&self, spec: &ChainSpec) -> Epoch { self.current_epoch(spec) + 1 } @@ -246,7 +250,7 @@ impl BeaconState { /// /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_cached_active_validator_indices( &self, relative_epoch: RelativeEpoch, @@ -261,7 +265,7 @@ impl BeaconState { /// /// Does not utilize the cache, performs a full iteration over the validator registry. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_active_validator_indices(&self, epoch: Epoch) -> Vec { get_active_validator_indices(&self.validator_registry, epoch) } @@ -270,7 +274,7 @@ impl BeaconState { /// /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_crosslink_committees_at_slot( &self, slot: Slot, @@ -295,7 +299,7 @@ impl BeaconState { /// /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_crosslink_committee_for_shard( &self, epoch: Epoch, @@ -321,7 +325,7 @@ impl BeaconState { /// /// If the state does not contain an index for a beacon proposer at the requested `slot`, then `None` is returned. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_beacon_proposer_index( &self, slot: Slot, @@ -350,7 +354,7 @@ impl BeaconState { /// Safely obtains the index for latest block roots, given some `slot`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 fn get_latest_block_roots_index(&self, slot: Slot, spec: &ChainSpec) -> Result { if (slot < self.slot) && (self.slot <= slot + spec.slots_per_historical_root as u64) { let i = slot.as_usize() % spec.slots_per_historical_root; @@ -366,7 +370,7 @@ impl BeaconState { /// Return the block root at a recent `slot`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_block_root( &self, slot: Slot, @@ -378,7 +382,7 @@ impl BeaconState { /// Sets the block root for some given slot. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn set_block_root( &mut self, slot: Slot, @@ -392,7 +396,7 @@ impl BeaconState { /// Safely obtains the index for `latest_randao_mixes` /// - /// Spec v0.5.0 + /// Spec v0.5.1 fn get_randao_mix_index(&self, epoch: Epoch, spec: &ChainSpec) -> Result { let current_epoch = self.current_epoch(spec); @@ -416,7 +420,7 @@ impl BeaconState { /// /// See `Self::get_randao_mix`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn update_randao_mix( &mut self, epoch: Epoch, @@ -434,7 +438,7 @@ impl BeaconState { /// Return the randao mix at a recent ``epoch``. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_randao_mix(&self, epoch: Epoch, spec: &ChainSpec) -> Result<&Hash256, Error> { let i = self.get_randao_mix_index(epoch, spec)?; Ok(&self.latest_randao_mixes[i]) @@ -442,7 +446,7 @@ impl BeaconState { /// Set the randao mix at a recent ``epoch``. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn set_randao_mix( &mut self, epoch: Epoch, @@ -456,7 +460,7 @@ impl BeaconState { /// Safely obtains the index for `latest_active_index_roots`, given some `epoch`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 fn get_active_index_root_index(&self, epoch: Epoch, spec: &ChainSpec) -> Result { let current_epoch = self.current_epoch(spec); @@ -478,7 +482,7 @@ impl BeaconState { /// Return the `active_index_root` at a recent `epoch`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_active_index_root(&self, epoch: Epoch, spec: &ChainSpec) -> Result { let i = self.get_active_index_root_index(epoch, spec)?; Ok(self.latest_active_index_roots[i]) @@ -486,7 +490,7 @@ impl BeaconState { /// Set the `active_index_root` at a recent `epoch`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn set_active_index_root( &mut self, epoch: Epoch, @@ -500,15 +504,15 @@ impl BeaconState { /// Replace `active_index_roots` with clones of `index_root`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn fill_active_index_roots_with(&mut self, index_root: Hash256, spec: &ChainSpec) { self.latest_active_index_roots = - vec![index_root; spec.latest_active_index_roots_length as usize] + vec![index_root; spec.latest_active_index_roots_length as usize].into() } /// Safely obtains the index for latest state roots, given some `slot`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 fn get_latest_state_roots_index(&self, slot: Slot, spec: &ChainSpec) -> Result { if (slot < self.slot) && (self.slot <= slot + spec.slots_per_historical_root as u64) { let i = slot.as_usize() % spec.slots_per_historical_root; @@ -524,7 +528,7 @@ impl BeaconState { /// Gets the state root for some slot. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_state_root(&mut self, slot: Slot, spec: &ChainSpec) -> Result<&Hash256, Error> { let i = self.get_latest_state_roots_index(slot, spec)?; Ok(&self.latest_state_roots[i]) @@ -532,7 +536,7 @@ impl BeaconState { /// Sets the latest state root for slot. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn set_state_root( &mut self, slot: Slot, @@ -546,7 +550,7 @@ impl BeaconState { /// Safely obtains the index for `latest_slashed_balances`, given some `epoch`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 fn get_slashed_balance_index(&self, epoch: Epoch, spec: &ChainSpec) -> Result { let i = epoch.as_usize() % spec.latest_slashed_exit_length; @@ -561,7 +565,7 @@ impl BeaconState { /// Gets the total slashed balances for some epoch. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_slashed_balance(&self, epoch: Epoch, spec: &ChainSpec) -> Result { let i = self.get_slashed_balance_index(epoch, spec)?; Ok(self.latest_slashed_balances[i]) @@ -569,7 +573,7 @@ impl BeaconState { /// Sets the total slashed balances for some epoch. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn set_slashed_balance( &mut self, epoch: Epoch, @@ -583,7 +587,7 @@ impl BeaconState { /// Generate a seed for the given `epoch`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn generate_seed(&self, epoch: Epoch, spec: &ChainSpec) -> Result { let mut input = self .get_randao_mix(epoch - spec.min_seed_lookahead, spec)? @@ -599,7 +603,7 @@ impl BeaconState { /// Return the effective balance (also known as "balance at stake") for a validator with the given ``index``. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_effective_balance( &self, validator_index: usize, @@ -614,14 +618,14 @@ impl BeaconState { /// Return the epoch at which an activation or exit triggered in ``epoch`` takes effect. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_delayed_activation_exit_epoch(&self, epoch: Epoch, spec: &ChainSpec) -> Epoch { epoch + 1 + spec.activation_exit_delay } /// Initiate an exit for the validator of the given `index`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn initiate_validator_exit(&mut self, validator_index: usize) { self.validator_registry[validator_index].initiated_exit = true; } @@ -633,7 +637,7 @@ impl BeaconState { /// /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_attestation_duties( &self, validator_index: usize, @@ -649,7 +653,7 @@ impl BeaconState { /// Return the combined effective balance of an array of validators. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_total_balance( &self, validator_indices: &[usize], diff --git a/eth2/types/src/beacon_state/epoch_cache.rs b/eth2/types/src/beacon_state/epoch_cache.rs index 62df90271..1a63e9eb9 100644 --- a/eth2/types/src/beacon_state/epoch_cache.rs +++ b/eth2/types/src/beacon_state/epoch_cache.rs @@ -138,7 +138,7 @@ impl EpochCache { /// Returns a list of all `validator_registry` indices where the validator is active at the given /// `epoch`. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub fn get_active_validator_indices(validators: &[Validator], epoch: Epoch) -> Vec { let mut active = Vec::with_capacity(validators.len()); @@ -288,7 +288,7 @@ impl EpochCrosslinkCommitteesBuilder { self.active_validator_indices, spec.shuffle_round_count, &self.shuffling_seed[..], - true, + false, ) .ok_or_else(|| Error::UnableToShuffle)? }; diff --git a/eth2/types/src/beacon_state/epoch_cache/tests.rs b/eth2/types/src/beacon_state/epoch_cache/tests.rs index 5643776e2..5b1e53338 100644 --- a/eth2/types/src/beacon_state/epoch_cache/tests.rs +++ b/eth2/types/src/beacon_state/epoch_cache/tests.rs @@ -27,7 +27,7 @@ fn do_sane_cache_test( active_indices, spec.shuffle_round_count, &expected_seed[..], - true, + false, ) .unwrap(); diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs index 0042304f8..f3c92b42c 100644 --- a/eth2/types/src/chain_spec.rs +++ b/eth2/types/src/chain_spec.rs @@ -8,7 +8,7 @@ const GWEI: u64 = 1_000_000_000; /// Each of the BLS signature domains. /// -/// Spec v0.5.0 +/// Spec v0.5.1 pub enum Domain { BeaconBlock, Randao, @@ -20,7 +20,7 @@ pub enum Domain { /// Holds all the "constants" for a BeaconChain. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive(PartialEq, Debug, Clone, Deserialize)] #[serde(default)] pub struct ChainSpec { @@ -126,7 +126,7 @@ pub struct ChainSpec { impl ChainSpec { /// Return the number of committees in one epoch. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_epoch_committee_count(&self, active_validator_count: usize) -> u64 { std::cmp::max( 1, @@ -139,7 +139,7 @@ impl ChainSpec { /// Get the domain number that represents the fork meta and signature domain. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_domain(&self, epoch: Epoch, domain: Domain, fork: &Fork) -> u64 { let domain_constant = match domain { Domain::BeaconBlock => self.domain_beacon_block, @@ -161,7 +161,7 @@ impl ChainSpec { /// Returns a `ChainSpec` compatible with the Ethereum Foundation specification. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn foundation() -> Self { let genesis_slot = Slot::new(2_u64.pow(32)); let slots_per_epoch = 64; diff --git a/eth2/types/src/crosslink.rs b/eth2/types/src/crosslink.rs index f91680c75..623226ad6 100644 --- a/eth2/types/src/crosslink.rs +++ b/eth2/types/src/crosslink.rs @@ -2,12 +2,13 @@ use crate::test_utils::TestRandom; use crate::{Epoch, Hash256}; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::TreeHash; /// Specifies the block hash for a shard at an epoch. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( Debug, Clone, diff --git a/eth2/types/src/crosslink_committee.rs b/eth2/types/src/crosslink_committee.rs index af1778a1b..e8fc1b96d 100644 --- a/eth2/types/src/crosslink_committee.rs +++ b/eth2/types/src/crosslink_committee.rs @@ -1,6 +1,7 @@ use crate::*; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; +use tree_hash_derive::TreeHash; #[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize, Decode, Encode, TreeHash)] pub struct CrosslinkCommittee { diff --git a/eth2/types/src/deposit.rs b/eth2/types/src/deposit.rs index ff8d83d77..291173d34 100644 --- a/eth2/types/src/deposit.rs +++ b/eth2/types/src/deposit.rs @@ -1,16 +1,17 @@ -use super::{DepositData, Hash256}; +use super::{DepositData, Hash256, TreeHashVector}; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::TreeHash; /// A deposit to potentially become a beacon chain validator. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] pub struct Deposit { - pub proof: Vec, + pub proof: TreeHashVector, pub index: u64, pub deposit_data: DepositData, } diff --git a/eth2/types/src/deposit_data.rs b/eth2/types/src/deposit_data.rs index a1e30032f..bc96ac7c4 100644 --- a/eth2/types/src/deposit_data.rs +++ b/eth2/types/src/deposit_data.rs @@ -2,12 +2,13 @@ use super::DepositInput; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::TreeHash; /// Data generated by the deposit contract. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] pub struct DepositData { pub amount: u64, diff --git a/eth2/types/src/deposit_input.rs b/eth2/types/src/deposit_input.rs index 380528dc0..be2106cb4 100644 --- a/eth2/types/src/deposit_input.rs +++ b/eth2/types/src/deposit_input.rs @@ -3,13 +3,14 @@ use crate::*; use bls::{PublicKey, Signature}; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz::{SignedRoot, TreeHash}; -use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash::{SignedRoot, TreeHash}; +use tree_hash_derive::{SignedRoot, TreeHash}; /// The data supplied by the user to the deposit contract. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( Debug, PartialEq, @@ -32,7 +33,7 @@ pub struct DepositInput { impl DepositInput { /// Generate the 'proof_of_posession' signature for a given DepositInput details. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn create_proof_of_possession( &self, secret_key: &SecretKey, @@ -48,7 +49,7 @@ impl DepositInput { /// Verify that proof-of-possession is valid. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn validate_proof_of_possession( &self, epoch: Epoch, diff --git a/eth2/types/src/epoch_cache.rs b/eth2/types/src/epoch_cache.rs deleted file mode 100644 index e69de29bb..000000000 diff --git a/eth2/types/src/eth1_data.rs b/eth2/types/src/eth1_data.rs index deced19fb..2ad460d13 100644 --- a/eth2/types/src/eth1_data.rs +++ b/eth2/types/src/eth1_data.rs @@ -2,12 +2,13 @@ use super::Hash256; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::TreeHash; /// Contains data obtained from the Eth1 chain. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( Debug, PartialEq, Clone, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, )] diff --git a/eth2/types/src/eth1_data_vote.rs b/eth2/types/src/eth1_data_vote.rs index 2f3a1ade1..7a77c8ff0 100644 --- a/eth2/types/src/eth1_data_vote.rs +++ b/eth2/types/src/eth1_data_vote.rs @@ -2,12 +2,13 @@ use super::Eth1Data; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::TreeHash; /// A summation of votes for some `Eth1Data`. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( Debug, PartialEq, Clone, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, )] diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs index b9d16c333..d99842855 100644 --- a/eth2/types/src/fork.rs +++ b/eth2/types/src/fork.rs @@ -5,12 +5,13 @@ use crate::{ use int_to_bytes::int_to_bytes4; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::TreeHash; /// Specifies a fork of the `BeaconChain`, to prevent replay attacks. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( Debug, Clone, PartialEq, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, )] @@ -25,7 +26,7 @@ pub struct Fork { impl Fork { /// Initialize the `Fork` from the genesis parameters in the `spec`. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn genesis(spec: &ChainSpec) -> Self { let mut current_version: [u8; 4] = [0; 4]; current_version.copy_from_slice(&int_to_bytes4(spec.genesis_fork_version)); @@ -39,7 +40,7 @@ impl Fork { /// Return the fork version of the given ``epoch``. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn get_fork_version(&self, epoch: Epoch) -> [u8; 4] { if epoch < self.epoch { return self.previous_version; diff --git a/eth2/types/src/historical_batch.rs b/eth2/types/src/historical_batch.rs index 77859ed1a..c4f62fcfc 100644 --- a/eth2/types/src/historical_batch.rs +++ b/eth2/types/src/historical_batch.rs @@ -1,17 +1,18 @@ use crate::test_utils::TestRandom; -use crate::Hash256; +use crate::{Hash256, TreeHashVector}; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::TreeHash; /// Historical block and state roots. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] pub struct HistoricalBatch { - pub block_roots: Vec, - pub state_roots: Vec, + pub block_roots: TreeHashVector, + pub state_roots: TreeHashVector, } #[cfg(test)] diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index 118e862e8..070ed6745 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -27,6 +27,7 @@ pub mod pending_attestation; pub mod proposer_slashing; pub mod slashable_attestation; pub mod transfer; +pub mod tree_hash_vector; pub mod voluntary_exit; #[macro_use] pub mod slot_epoch_macros; @@ -65,6 +66,7 @@ pub use crate::slashable_attestation::SlashableAttestation; pub use crate::slot_epoch::{Epoch, Slot}; pub use crate::slot_height::SlotHeight; pub use crate::transfer::Transfer; +pub use crate::tree_hash_vector::TreeHashVector; pub use crate::validator::Validator; pub use crate::voluntary_exit::VoluntaryExit; diff --git a/eth2/types/src/pending_attestation.rs b/eth2/types/src/pending_attestation.rs index 938e59bef..ce9ce3d77 100644 --- a/eth2/types/src/pending_attestation.rs +++ b/eth2/types/src/pending_attestation.rs @@ -2,12 +2,13 @@ use crate::test_utils::TestRandom; use crate::{Attestation, AttestationData, Bitfield, Slot}; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::TreeHash; /// An attestation that has been included in the state but not yet fully processed. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] pub struct PendingAttestation { pub aggregation_bitfield: Bitfield, diff --git a/eth2/types/src/proposer_slashing.rs b/eth2/types/src/proposer_slashing.rs index 02216a2fc..a3501a5bd 100644 --- a/eth2/types/src/proposer_slashing.rs +++ b/eth2/types/src/proposer_slashing.rs @@ -2,12 +2,13 @@ use super::BeaconBlockHeader; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::TreeHash; /// Two conflicting proposals from the same proposer (validator). /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] pub struct ProposerSlashing { pub proposer_index: u64, diff --git a/eth2/types/src/relative_epoch.rs b/eth2/types/src/relative_epoch.rs index 8f895e97a..6538ca4aa 100644 --- a/eth2/types/src/relative_epoch.rs +++ b/eth2/types/src/relative_epoch.rs @@ -10,7 +10,7 @@ pub enum Error { /// Defines the epochs relative to some epoch. Most useful when referring to the committees prior /// to and following some epoch. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive(Debug, PartialEq, Clone, Copy)] pub enum RelativeEpoch { /// The prior epoch. @@ -32,7 +32,7 @@ pub enum RelativeEpoch { impl RelativeEpoch { /// Returns the `epoch` that `self` refers to, with respect to the `base` epoch. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn into_epoch(self, base: Epoch) -> Epoch { match self { RelativeEpoch::Previous => base - 1, @@ -51,7 +51,7 @@ impl RelativeEpoch { /// - `AmbiguiousNextEpoch` whenever `other` is one after `base`, because it's unknowable if /// there will be a registry change. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn from_epoch(base: Epoch, other: Epoch) -> Result { if other == base - 1 { Ok(RelativeEpoch::Previous) diff --git a/eth2/types/src/slashable_attestation.rs b/eth2/types/src/slashable_attestation.rs index e557285b8..9c460e482 100644 --- a/eth2/types/src/slashable_attestation.rs +++ b/eth2/types/src/slashable_attestation.rs @@ -1,15 +1,16 @@ use crate::{test_utils::TestRandom, AggregateSignature, AttestationData, Bitfield, ChainSpec}; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz::TreeHash; -use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash::TreeHash; +use tree_hash_derive::{SignedRoot, TreeHash}; /// Details an attestation that can be slashable. /// /// To be included in an `AttesterSlashing`. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( Debug, PartialEq, @@ -34,14 +35,14 @@ pub struct SlashableAttestation { impl SlashableAttestation { /// Check if ``attestation_data_1`` and ``attestation_data_2`` have the same target. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn is_double_vote(&self, other: &SlashableAttestation, spec: &ChainSpec) -> bool { self.data.slot.epoch(spec.slots_per_epoch) == other.data.slot.epoch(spec.slots_per_epoch) } /// Check if ``attestation_data_1`` surrounds ``attestation_data_2``. /// - /// Spec v0.5.0 + /// Spec v0.5.1 pub fn is_surround_vote(&self, other: &SlashableAttestation, spec: &ChainSpec) -> bool { let source_epoch_1 = self.data.source_epoch; let source_epoch_2 = other.data.source_epoch; diff --git a/eth2/types/src/slot_epoch.rs b/eth2/types/src/slot_epoch.rs index d334177e5..6c6a92ecb 100644 --- a/eth2/types/src/slot_epoch.rs +++ b/eth2/types/src/slot_epoch.rs @@ -14,7 +14,7 @@ use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; use slog; -use ssz::{hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream}; use std::cmp::{Ord, Ordering}; use std::fmt; use std::hash::{Hash, Hasher}; diff --git a/eth2/types/src/slot_epoch_macros.rs b/eth2/types/src/slot_epoch_macros.rs index 300ad3f6f..b3ca5c4bc 100644 --- a/eth2/types/src/slot_epoch_macros.rs +++ b/eth2/types/src/slot_epoch_macros.rs @@ -206,11 +206,21 @@ macro_rules! impl_ssz { } } - impl TreeHash for $type { - fn hash_tree_root(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.0.hash_tree_root()); - hash(&result) + impl tree_hash::TreeHash for $type { + fn tree_hash_type() -> tree_hash::TreeHashType { + tree_hash::TreeHashType::Basic + } + + fn tree_hash_packed_encoding(&self) -> Vec { + ssz_encode(self) + } + + fn tree_hash_packing_factor() -> usize { + 32 / 8 + } + + fn tree_hash_root(&self) -> Vec { + int_to_bytes::int_to_bytes32(self.0) } } diff --git a/eth2/types/src/slot_height.rs b/eth2/types/src/slot_height.rs index 4a783d4a0..f7a34cbba 100644 --- a/eth2/types/src/slot_height.rs +++ b/eth2/types/src/slot_height.rs @@ -2,7 +2,7 @@ use crate::slot_epoch::{Epoch, Slot}; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream}; use std::cmp::{Ord, Ordering}; use std::fmt; use std::hash::{Hash, Hasher}; diff --git a/eth2/types/src/test_utils/macros.rs b/eth2/types/src/test_utils/macros.rs index d580fd818..d5711e96e 100644 --- a/eth2/types/src/test_utils/macros.rs +++ b/eth2/types/src/test_utils/macros.rs @@ -17,14 +17,14 @@ macro_rules! ssz_tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_tree_hash_root() { use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::TreeHash; + use tree_hash::TreeHash; let mut rng = XorShiftRng::from_seed([42; 16]); let original = $type::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.tree_hash_root(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/test_utils/mod.rs b/eth2/types/src/test_utils/mod.rs index 018b70d15..9d69a48f6 100644 --- a/eth2/types/src/test_utils/mod.rs +++ b/eth2/types/src/test_utils/mod.rs @@ -17,7 +17,10 @@ mod testing_voluntary_exit_builder; pub use generate_deterministic_keypairs::generate_deterministic_keypairs; pub use keypairs_file::KeypairsFile; -pub use rand::{prng::XorShiftRng, SeedableRng}; +pub use rand::{ + RngCore, + {prng::XorShiftRng, SeedableRng}, +}; pub use serde_utils::{fork_from_hex_str, u8_from_hex_str}; pub use test_random::TestRandom; pub use testing_attestation_builder::TestingAttestationBuilder; diff --git a/eth2/types/src/test_utils/testing_attestation_builder.rs b/eth2/types/src/test_utils/testing_attestation_builder.rs index 60624b48d..162facc8e 100644 --- a/eth2/types/src/test_utils/testing_attestation_builder.rs +++ b/eth2/types/src/test_utils/testing_attestation_builder.rs @@ -1,6 +1,6 @@ use crate::test_utils::TestingAttestationDataBuilder; use crate::*; -use ssz::TreeHash; +use tree_hash::TreeHash; /// Builds an attestation to be used for testing purposes. /// @@ -74,7 +74,7 @@ impl TestingAttestationBuilder { data: self.attestation.data.clone(), custody_bit: false, } - .hash_tree_root(); + .tree_hash_root(); let domain = spec.get_domain( self.attestation.data.slot.epoch(spec.slots_per_epoch), diff --git a/eth2/types/src/test_utils/testing_attester_slashing_builder.rs b/eth2/types/src/test_utils/testing_attester_slashing_builder.rs index fcaa3285b..dc01f7fb0 100644 --- a/eth2/types/src/test_utils/testing_attester_slashing_builder.rs +++ b/eth2/types/src/test_utils/testing_attester_slashing_builder.rs @@ -1,5 +1,5 @@ use crate::*; -use ssz::TreeHash; +use tree_hash::TreeHash; /// Builds an `AttesterSlashing`. /// @@ -66,7 +66,7 @@ impl TestingAttesterSlashingBuilder { data: attestation.data.clone(), custody_bit: false, }; - let message = attestation_data_and_custody_bit.hash_tree_root(); + let message = attestation_data_and_custody_bit.tree_hash_root(); for (i, validator_index) in validator_indices.iter().enumerate() { attestation.custody_bitfield.set(i, false); diff --git a/eth2/types/src/test_utils/testing_beacon_block_builder.rs b/eth2/types/src/test_utils/testing_beacon_block_builder.rs index c5cd22ed4..549c00ac0 100644 --- a/eth2/types/src/test_utils/testing_beacon_block_builder.rs +++ b/eth2/types/src/test_utils/testing_beacon_block_builder.rs @@ -6,7 +6,7 @@ use crate::{ *, }; use rayon::prelude::*; -use ssz::{SignedRoot, TreeHash}; +use tree_hash::{SignedRoot, TreeHash}; /// Builds a beacon block to be used for testing purposes. /// @@ -43,7 +43,7 @@ impl TestingBeaconBlockBuilder { /// Modifying the block's slot after signing may invalidate the signature. pub fn set_randao_reveal(&mut self, sk: &SecretKey, fork: &Fork, spec: &ChainSpec) { let epoch = self.block.slot.epoch(spec.slots_per_epoch); - let message = epoch.hash_tree_root(); + let message = epoch.tree_hash_root(); let domain = spec.get_domain(epoch, Domain::Randao, fork); self.block.body.randao_reveal = Signature::new(&message, domain, sk); } diff --git a/eth2/types/src/test_utils/testing_deposit_builder.rs b/eth2/types/src/test_utils/testing_deposit_builder.rs index 326858c31..080ed5cfb 100644 --- a/eth2/types/src/test_utils/testing_deposit_builder.rs +++ b/eth2/types/src/test_utils/testing_deposit_builder.rs @@ -12,7 +12,7 @@ impl TestingDepositBuilder { /// Instantiates a new builder. pub fn new(pubkey: PublicKey, amount: u64) -> Self { let deposit = Deposit { - proof: vec![], + proof: vec![].into(), index: 0, deposit_data: DepositData { amount, diff --git a/eth2/types/src/test_utils/testing_proposer_slashing_builder.rs b/eth2/types/src/test_utils/testing_proposer_slashing_builder.rs index 2cfebd915..03c257b2d 100644 --- a/eth2/types/src/test_utils/testing_proposer_slashing_builder.rs +++ b/eth2/types/src/test_utils/testing_proposer_slashing_builder.rs @@ -1,5 +1,5 @@ use crate::*; -use ssz::SignedRoot; +use tree_hash::SignedRoot; /// Builds a `ProposerSlashing`. /// diff --git a/eth2/types/src/test_utils/testing_transfer_builder.rs b/eth2/types/src/test_utils/testing_transfer_builder.rs index 354e29aa5..2680f7b66 100644 --- a/eth2/types/src/test_utils/testing_transfer_builder.rs +++ b/eth2/types/src/test_utils/testing_transfer_builder.rs @@ -1,5 +1,5 @@ use crate::*; -use ssz::SignedRoot; +use tree_hash::SignedRoot; /// Builds a transfer to be used for testing purposes. /// diff --git a/eth2/types/src/test_utils/testing_voluntary_exit_builder.rs b/eth2/types/src/test_utils/testing_voluntary_exit_builder.rs index fe5c8325a..8583bc451 100644 --- a/eth2/types/src/test_utils/testing_voluntary_exit_builder.rs +++ b/eth2/types/src/test_utils/testing_voluntary_exit_builder.rs @@ -1,5 +1,5 @@ use crate::*; -use ssz::SignedRoot; +use tree_hash::SignedRoot; /// Builds an exit to be used for testing purposes. /// diff --git a/eth2/types/src/transfer.rs b/eth2/types/src/transfer.rs index f291190b2..82ead03d5 100644 --- a/eth2/types/src/transfer.rs +++ b/eth2/types/src/transfer.rs @@ -4,13 +4,14 @@ use bls::{PublicKey, Signature}; use derivative::Derivative; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz::TreeHash; -use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash::TreeHash; +use tree_hash_derive::{SignedRoot, TreeHash}; /// The data submitted to the deposit contract. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( Debug, Clone, diff --git a/eth2/types/src/tree_hash_vector.rs b/eth2/types/src/tree_hash_vector.rs new file mode 100644 index 000000000..1cc8e40a5 --- /dev/null +++ b/eth2/types/src/tree_hash_vector.rs @@ -0,0 +1,82 @@ +use crate::test_utils::{RngCore, TestRandom}; +use serde_derive::{Deserialize, Serialize}; +use ssz::{Decodable, DecodeError, Encodable, SszStream}; +use std::ops::{Deref, DerefMut}; +use tree_hash::TreeHash; + +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +pub struct TreeHashVector(Vec); + +impl From> for TreeHashVector { + fn from(vec: Vec) -> TreeHashVector { + TreeHashVector(vec) + } +} + +impl Into> for TreeHashVector { + fn into(self) -> Vec { + self.0 + } +} + +impl Deref for TreeHashVector { + type Target = Vec; + + fn deref(&self) -> &Vec { + &self.0 + } +} + +impl DerefMut for TreeHashVector { + fn deref_mut(&mut self) -> &mut Vec { + &mut self.0 + } +} + +impl tree_hash::TreeHash for TreeHashVector +where + T: TreeHash, +{ + fn tree_hash_type() -> tree_hash::TreeHashType { + tree_hash::TreeHashType::Vector + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("Vector should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("Vector should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + tree_hash::standard_tree_hash::vec_tree_hash_root(self) + } +} + +impl Encodable for TreeHashVector +where + T: Encodable, +{ + fn ssz_append(&self, s: &mut SszStream) { + s.append_vec(self) + } +} + +impl Decodable for TreeHashVector +where + T: Decodable, +{ + fn ssz_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), DecodeError> { + ssz::decode_ssz_list(bytes, index).and_then(|(vec, i)| Ok((vec.into(), i))) + } +} + +impl TestRandom for TreeHashVector +where + U: TestRandom, +{ + fn random_for_test(rng: &mut T) -> Self { + Vec::random_for_test(rng).into() + } +} diff --git a/eth2/types/src/validator.rs b/eth2/types/src/validator.rs index f57261175..bbd68ed2b 100644 --- a/eth2/types/src/validator.rs +++ b/eth2/types/src/validator.rs @@ -1,12 +1,13 @@ use crate::{test_utils::TestRandom, Epoch, Hash256, PublicKey}; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash_derive::TreeHash; /// Information about a `BeaconChain` validator. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TestRandom, TreeHash)] pub struct Validator { pub pubkey: PublicKey, diff --git a/eth2/types/src/voluntary_exit.rs b/eth2/types/src/voluntary_exit.rs index 0cdc63149..cb872cb98 100644 --- a/eth2/types/src/voluntary_exit.rs +++ b/eth2/types/src/voluntary_exit.rs @@ -2,13 +2,14 @@ use crate::{test_utils::TestRandom, Epoch}; use bls::Signature; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; -use ssz::TreeHash; -use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; +use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; +use tree_hash::TreeHash; +use tree_hash_derive::{SignedRoot, TreeHash}; /// An exit voluntarily submitted a validator who wishes to withdraw. /// -/// Spec v0.5.0 +/// Spec v0.5.1 #[derive( Debug, PartialEq, diff --git a/eth2/utils/bls/Cargo.toml b/eth2/utils/bls/Cargo.toml index 4230a06ea..4ce499580 100644 --- a/eth2/utils/bls/Cargo.toml +++ b/eth2/utils/bls/Cargo.toml @@ -12,3 +12,7 @@ serde = "1.0" serde_derive = "1.0" serde_hex = { path = "../serde_hex" } ssz = { path = "../ssz" } +tree_hash = { path = "../tree_hash" } + +[features] +fake_crypto = [] diff --git a/eth2/utils/bls/build.rs b/eth2/utils/bls/build.rs new file mode 100644 index 000000000..7f08a1ed5 --- /dev/null +++ b/eth2/utils/bls/build.rs @@ -0,0 +1,19 @@ +// This build script is symlinked from each project that requires BLS's "fake crypto", +// so that the `fake_crypto` feature of every sub-crate can be turned on by running +// with FAKE_CRYPTO=1 from the top-level workspace. +// At some point in the future it might be possible to do: +// $ cargo test --all --release --features fake_crypto +// but at the present time this doesn't work. +// Related: https://github.com/rust-lang/cargo/issues/5364 +fn main() { + if let Ok(fake_crypto) = std::env::var("FAKE_CRYPTO") { + if fake_crypto == "1" { + println!("cargo:rustc-cfg=feature=\"fake_crypto\""); + println!("cargo:rerun-if-env-changed=FAKE_CRYPTO"); + println!( + "cargo:warning=[{}]: Compiled with fake BLS cryptography. DO NOT USE, TESTING ONLY", + std::env::var("CARGO_PKG_NAME").unwrap() + ); + } + } +} diff --git a/eth2/utils/bls/src/aggregate_signature.rs b/eth2/utils/bls/src/aggregate_signature.rs index 8c7ae5222..0fbcc3493 100644 --- a/eth2/utils/bls/src/aggregate_signature.rs +++ b/eth2/utils/bls/src/aggregate_signature.rs @@ -5,7 +5,8 @@ use bls_aggregates::{ use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; use serde_hex::{encode as hex_encode, HexVisitor}; -use ssz::{decode, hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{decode, Decodable, DecodeError, Encodable, SszStream}; +use tree_hash::tree_hash_ssz_encoding_as_vector; /// A BLS aggregate signature. /// @@ -165,11 +166,7 @@ impl<'de> Deserialize<'de> for AggregateSignature { } } -impl TreeHash for AggregateSignature { - fn hash_tree_root(&self) -> Vec { - hash(&self.as_bytes()) - } -} +tree_hash_ssz_encoding_as_vector!(AggregateSignature); #[cfg(test)] mod tests { diff --git a/eth2/utils/bls/src/fake_aggregate_signature.rs b/eth2/utils/bls/src/fake_aggregate_signature.rs index 3f0ec0d6d..f201eba3e 100644 --- a/eth2/utils/bls/src/fake_aggregate_signature.rs +++ b/eth2/utils/bls/src/fake_aggregate_signature.rs @@ -2,7 +2,8 @@ use super::{fake_signature::FakeSignature, AggregatePublicKey, BLS_AGG_SIG_BYTE_ use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; use serde_hex::{encode as hex_encode, PrefixedHexVisitor}; -use ssz::{hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream}; +use tree_hash::tree_hash_ssz_encoding_as_vector; /// A BLS aggregate signature. /// @@ -98,11 +99,7 @@ impl<'de> Deserialize<'de> for FakeAggregateSignature { } } -impl TreeHash for FakeAggregateSignature { - fn hash_tree_root(&self) -> Vec { - hash(&self.bytes) - } -} +tree_hash_ssz_encoding_as_vector!(FakeAggregateSignature); #[cfg(test)] mod tests { diff --git a/eth2/utils/bls/src/fake_signature.rs b/eth2/utils/bls/src/fake_signature.rs index 3c9f3a9f4..3208ed992 100644 --- a/eth2/utils/bls/src/fake_signature.rs +++ b/eth2/utils/bls/src/fake_signature.rs @@ -3,7 +3,8 @@ use hex::encode as hex_encode; use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; use serde_hex::HexVisitor; -use ssz::{hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream}; +use tree_hash::tree_hash_ssz_encoding_as_vector; /// A single BLS signature. /// @@ -73,11 +74,7 @@ impl Decodable for FakeSignature { } } -impl TreeHash for FakeSignature { - fn hash_tree_root(&self) -> Vec { - hash(&self.bytes) - } -} +tree_hash_ssz_encoding_as_vector!(FakeSignature); impl Serialize for FakeSignature { fn serialize(&self, serializer: S) -> Result diff --git a/eth2/utils/bls/src/lib.rs b/eth2/utils/bls/src/lib.rs index b9a4d5c1d..fae41aeed 100644 --- a/eth2/utils/bls/src/lib.rs +++ b/eth2/utils/bls/src/lib.rs @@ -6,22 +6,22 @@ mod keypair; mod public_key; mod secret_key; -#[cfg(not(debug_assertions))] +#[cfg(not(feature = "fake_crypto"))] mod aggregate_signature; -#[cfg(not(debug_assertions))] +#[cfg(not(feature = "fake_crypto"))] mod signature; -#[cfg(not(debug_assertions))] +#[cfg(not(feature = "fake_crypto"))] pub use crate::aggregate_signature::AggregateSignature; -#[cfg(not(debug_assertions))] +#[cfg(not(feature = "fake_crypto"))] pub use crate::signature::Signature; -#[cfg(debug_assertions)] +#[cfg(feature = "fake_crypto")] mod fake_aggregate_signature; -#[cfg(debug_assertions)] +#[cfg(feature = "fake_crypto")] mod fake_signature; -#[cfg(debug_assertions)] +#[cfg(feature = "fake_crypto")] pub use crate::fake_aggregate_signature::FakeAggregateSignature as AggregateSignature; -#[cfg(debug_assertions)] +#[cfg(feature = "fake_crypto")] pub use crate::fake_signature::FakeSignature as Signature; pub use crate::aggregate_public_key::AggregatePublicKey; diff --git a/eth2/utils/bls/src/public_key.rs b/eth2/utils/bls/src/public_key.rs index 177a735c4..dcbbc622a 100644 --- a/eth2/utils/bls/src/public_key.rs +++ b/eth2/utils/bls/src/public_key.rs @@ -3,10 +3,11 @@ use bls_aggregates::PublicKey as RawPublicKey; use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; use serde_hex::{encode as hex_encode, HexVisitor}; -use ssz::{decode, hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{decode, ssz_encode, Decodable, DecodeError, Encodable, SszStream}; use std::default; use std::fmt; use std::hash::{Hash, Hasher}; +use tree_hash::tree_hash_ssz_encoding_as_vector; /// A single BLS signature. /// @@ -104,11 +105,7 @@ impl<'de> Deserialize<'de> for PublicKey { } } -impl TreeHash for PublicKey { - fn hash_tree_root(&self) -> Vec { - hash(&self.0.as_bytes()) - } -} +tree_hash_ssz_encoding_as_vector!(PublicKey); impl PartialEq for PublicKey { fn eq(&self, other: &PublicKey) -> bool { diff --git a/eth2/utils/bls/src/secret_key.rs b/eth2/utils/bls/src/secret_key.rs index 40c469513..d1aaa96da 100644 --- a/eth2/utils/bls/src/secret_key.rs +++ b/eth2/utils/bls/src/secret_key.rs @@ -4,7 +4,8 @@ use hex::encode as hex_encode; use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; use serde_hex::HexVisitor; -use ssz::{decode, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{decode, ssz_encode, Decodable, DecodeError, Encodable, SszStream}; +use tree_hash::tree_hash_ssz_encoding_as_vector; /// A single BLS signature. /// @@ -69,11 +70,7 @@ impl<'de> Deserialize<'de> for SecretKey { } } -impl TreeHash for SecretKey { - fn hash_tree_root(&self) -> Vec { - self.0.as_bytes().clone() - } -} +tree_hash_ssz_encoding_as_vector!(SecretKey); #[cfg(test)] mod tests { diff --git a/eth2/utils/bls/src/signature.rs b/eth2/utils/bls/src/signature.rs index d19af545f..3fb68dc53 100644 --- a/eth2/utils/bls/src/signature.rs +++ b/eth2/utils/bls/src/signature.rs @@ -4,7 +4,8 @@ use hex::encode as hex_encode; use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; use serde_hex::HexVisitor; -use ssz::{decode, hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{decode, ssz_encode, Decodable, DecodeError, Encodable, SszStream}; +use tree_hash::tree_hash_ssz_encoding_as_vector; /// A single BLS signature. /// @@ -114,11 +115,7 @@ impl Decodable for Signature { } } -impl TreeHash for Signature { - fn hash_tree_root(&self) -> Vec { - hash(&self.as_bytes()) - } -} +tree_hash_ssz_encoding_as_vector!(Signature); impl Serialize for Signature { /// Serde serialization is compliant the Ethereum YAML test format. diff --git a/eth2/utils/boolean-bitfield/Cargo.toml b/eth2/utils/boolean-bitfield/Cargo.toml index cf037c5d7..61bbc60a8 100644 --- a/eth2/utils/boolean-bitfield/Cargo.toml +++ b/eth2/utils/boolean-bitfield/Cargo.toml @@ -8,5 +8,10 @@ edition = "2018" serde_hex = { path = "../serde_hex" } ssz = { path = "../ssz" } bit-vec = "0.5.0" +bit_reverse = "0.1" serde = "1.0" serde_derive = "1.0" +tree_hash = { path = "../tree_hash" } + +[dev-dependencies] +serde_yaml = "0.8" diff --git a/eth2/utils/boolean-bitfield/src/lib.rs b/eth2/utils/boolean-bitfield/src/lib.rs index d04516dba..1d0f1c02e 100644 --- a/eth2/utils/boolean-bitfield/src/lib.rs +++ b/eth2/utils/boolean-bitfield/src/lib.rs @@ -1,8 +1,8 @@ extern crate bit_vec; extern crate ssz; +use bit_reverse::LookupReverse; use bit_vec::BitVec; - use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; use serde_hex::{encode, PrefixedHexVisitor}; @@ -53,10 +53,15 @@ impl BooleanBitfield { /// Create a new bitfield using the supplied `bytes` as input pub fn from_bytes(bytes: &[u8]) -> Self { Self { - 0: BitVec::from_bytes(bytes), + 0: BitVec::from_bytes(&reverse_bit_order(bytes.to_vec())), } } + /// Returns a vector of bytes representing the bitfield + pub fn to_bytes(&self) -> Vec { + reverse_bit_order(self.0.to_bytes().to_vec()) + } + /// Read the value of a bit. /// /// If the index is in bounds, then result is Ok(value) where value is `true` if the bit is 1 and `false` if the bit is 0. @@ -85,11 +90,6 @@ impl BooleanBitfield { previous } - /// Returns the index of the highest set bit. Some(n) if some bit is set, None otherwise. - pub fn highest_set_bit(&self) -> Option { - self.0.iter().rposition(|bit| bit) - } - /// Returns the number of bits in this bitfield. pub fn len(&self) -> usize { self.0.len() @@ -115,12 +115,6 @@ impl BooleanBitfield { self.0.iter().filter(|&bit| bit).count() } - /// Returns a vector of bytes representing the bitfield - /// Note that this returns the bit layout of the underlying implementation in the `bit-vec` crate. - pub fn to_bytes(&self) -> Vec { - self.0.to_bytes() - } - /// Compute the intersection (binary-and) of this bitfield with another. Lengths must match. pub fn intersection(&self, other: &Self) -> Self { let mut res = self.clone(); @@ -217,17 +211,7 @@ impl Decodable for BooleanBitfield { Ok((BooleanBitfield::new(), index + ssz::LENGTH_BYTES)) } else { let bytes = &bytes[(index + 4)..(index + len + 4)]; - - let count = len * 8; - let mut field = BooleanBitfield::with_capacity(count); - for (byte_index, byte) in bytes.iter().enumerate() { - for i in 0..8 { - let bit = byte & (128 >> i); - if bit != 0 { - field.set(8 * byte_index + i, true); - } - } - } + let field = BooleanBitfield::from_bytes(bytes); let index = index + ssz::LENGTH_BYTES + len; Ok((field, index)) @@ -235,36 +219,61 @@ impl Decodable for BooleanBitfield { } } +// Reverse the bit order of a whole byte vec, so that the ith bit +// of the input vec is placed in the (N - i)th bit of the output vec. +// This function is necessary for converting bitfields to and from YAML, +// as the BitVec library and the hex-parser use opposing bit orders. +fn reverse_bit_order(mut bytes: Vec) -> Vec { + bytes.reverse(); + bytes.into_iter().map(|b| b.swap_bits()).collect() +} + impl Serialize for BooleanBitfield { - /// Serde serialization is compliant the Ethereum YAML test format. + /// Serde serialization is compliant with the Ethereum YAML test format. fn serialize(&self, serializer: S) -> Result where S: Serializer, { - serializer.serialize_str(&encode(&self.to_bytes())) + serializer.serialize_str(&encode(self.to_bytes())) } } impl<'de> Deserialize<'de> for BooleanBitfield { - /// Serde serialization is compliant the Ethereum YAML test format. + /// Serde serialization is compliant with the Ethereum YAML test format. fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { + // We reverse the bit-order so that the BitVec library can read its 0th + // bit from the end of the hex string, e.g. + // "0xef01" => [0xef, 0x01] => [0b1000_0000, 0b1111_1110] let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?; Ok(BooleanBitfield::from_bytes(&bytes)) } } -impl ssz::TreeHash for BooleanBitfield { - fn hash_tree_root(&self) -> Vec { - self.to_bytes().hash_tree_root() +impl tree_hash::TreeHash for BooleanBitfield { + fn tree_hash_type() -> tree_hash::TreeHashType { + tree_hash::TreeHashType::List + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("List should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("List should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + self.to_bytes().tree_hash_root() } } #[cfg(test)] mod tests { use super::*; + use serde_yaml; use ssz::{decode, ssz_encode, SszStream}; #[test] @@ -312,7 +321,7 @@ mod tests { assert_eq!(field.num_set_bits(), 100); } - const INPUT: &[u8] = &[0b0000_0010, 0b0000_0010]; + const INPUT: &[u8] = &[0b0100_0000, 0b0100_0000]; #[test] fn test_get_from_bitfield() { @@ -338,18 +347,6 @@ mod tests { assert!(!previous); } - #[test] - fn test_highest_set_bit() { - let field = BooleanBitfield::from_bytes(INPUT); - assert_eq!(field.highest_set_bit().unwrap(), 14); - - let field = BooleanBitfield::from_bytes(&[0b0000_0011]); - assert_eq!(field.highest_set_bit().unwrap(), 7); - - let field = BooleanBitfield::new(); - assert_eq!(field.highest_set_bit(), None); - } - #[test] fn test_len() { let field = BooleanBitfield::from_bytes(INPUT); @@ -430,15 +427,30 @@ mod tests { #[test] fn test_ssz_encode() { let field = create_test_bitfield(); - let mut stream = SszStream::new(); stream.append(&field); - assert_eq!(stream.drain(), vec![2, 0, 0, 0, 225, 192]); + assert_eq!(stream.drain(), vec![2, 0, 0, 0, 0b0000_0011, 0b1000_0111]); let field = BooleanBitfield::from_elem(18, true); let mut stream = SszStream::new(); stream.append(&field); - assert_eq!(stream.drain(), vec![3, 0, 0, 0, 255, 255, 192]); + assert_eq!( + stream.drain(), + vec![3, 0, 0, 0, 0b0000_0011, 0b1111_1111, 0b1111_1111] + ); + + let mut b = BooleanBitfield::new(); + b.set(1, true); + assert_eq!( + ssz_encode(&b), + vec![ + 0b0000_0001, + 0b0000_0000, + 0b0000_0000, + 0b0000_0000, + 0b0000_0010 + ] + ); } fn create_test_bitfield() -> BooleanBitfield { @@ -454,7 +466,7 @@ mod tests { #[test] fn test_ssz_decode() { - let encoded = vec![2, 0, 0, 0, 225, 192]; + let encoded = vec![2, 0, 0, 0, 0b0000_0011, 0b1000_0111]; let field = decode::(&encoded).unwrap(); let expected = create_test_bitfield(); assert_eq!(field, expected); @@ -465,6 +477,27 @@ mod tests { assert_eq!(field, expected); } + #[test] + fn test_serialize_deserialize() { + use serde_yaml::Value; + + let data: &[(_, &[_])] = &[ + ("0x01", &[0b00000001]), + ("0xf301", &[0b11110011, 0b00000001]), + ]; + for (hex_data, bytes) in data { + let bitfield = BooleanBitfield::from_bytes(bytes); + assert_eq!( + serde_yaml::from_str::(hex_data).unwrap(), + bitfield + ); + assert_eq!( + serde_yaml::to_value(&bitfield).unwrap(), + Value::String(hex_data.to_string()) + ); + } + } + #[test] fn test_ssz_round_trip() { let original = BooleanBitfield::from_bytes(&vec![18; 12][..]); diff --git a/eth2/utils/ssz/Cargo.toml b/eth2/utils/ssz/Cargo.toml index fa042a8ac..ecefd69fe 100644 --- a/eth2/utils/ssz/Cargo.toml +++ b/eth2/utils/ssz/Cargo.toml @@ -8,5 +8,6 @@ edition = "2018" bytes = "0.4.9" ethereum-types = "0.5" hashing = { path = "../hashing" } +int_to_bytes = { path = "../int_to_bytes" } hex = "0.3" yaml-rust = "0.4" diff --git a/eth2/utils/ssz/src/impl_tree_hash.rs b/eth2/utils/ssz/src/impl_tree_hash.rs deleted file mode 100644 index 03976f637..000000000 --- a/eth2/utils/ssz/src/impl_tree_hash.rs +++ /dev/null @@ -1,85 +0,0 @@ -use super::ethereum_types::{Address, H256}; -use super::{merkle_hash, ssz_encode, TreeHash}; -use hashing::hash; - -impl TreeHash for u8 { - fn hash_tree_root(&self) -> Vec { - ssz_encode(self) - } -} - -impl TreeHash for u16 { - fn hash_tree_root(&self) -> Vec { - ssz_encode(self) - } -} - -impl TreeHash for u32 { - fn hash_tree_root(&self) -> Vec { - ssz_encode(self) - } -} - -impl TreeHash for u64 { - fn hash_tree_root(&self) -> Vec { - ssz_encode(self) - } -} - -impl TreeHash for usize { - fn hash_tree_root(&self) -> Vec { - ssz_encode(self) - } -} - -impl TreeHash for bool { - fn hash_tree_root(&self) -> Vec { - ssz_encode(self) - } -} - -impl TreeHash for Address { - fn hash_tree_root(&self) -> Vec { - ssz_encode(self) - } -} - -impl TreeHash for H256 { - fn hash_tree_root(&self) -> Vec { - ssz_encode(self) - } -} - -impl TreeHash for [u8] { - fn hash_tree_root(&self) -> Vec { - if self.len() > 32 { - return hash(&self); - } - self.to_vec() - } -} - -impl TreeHash for Vec -where - T: TreeHash, -{ - /// Returns the merkle_hash of a list of hash_tree_root values created - /// from the given list. - /// Note: A byte vector, Vec, must be converted to a slice (as_slice()) - /// to be handled properly (i.e. hashed) as byte array. - fn hash_tree_root(&self) -> Vec { - let mut tree_hashes = self.iter().map(|x| x.hash_tree_root()).collect(); - merkle_hash(&mut tree_hashes) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_impl_tree_hash_vec() { - let result = vec![1u32, 2, 3, 4, 5, 6, 7].hash_tree_root(); - assert_eq!(result.len(), 32); - } -} diff --git a/eth2/utils/ssz/src/lib.rs b/eth2/utils/ssz/src/lib.rs index cb3f63c48..0a00efa5d 100644 --- a/eth2/utils/ssz/src/lib.rs +++ b/eth2/utils/ssz/src/lib.rs @@ -12,17 +12,12 @@ extern crate ethereum_types; pub mod decode; pub mod encode; -mod signed_root; -pub mod tree_hash; mod impl_decode; mod impl_encode; -mod impl_tree_hash; pub use crate::decode::{decode, decode_ssz_list, Decodable, DecodeError}; pub use crate::encode::{Encodable, SszStream}; -pub use crate::signed_root::SignedRoot; -pub use crate::tree_hash::{merkle_hash, TreeHash}; pub use hashing::hash; diff --git a/eth2/utils/ssz/src/tree_hash.rs b/eth2/utils/ssz/src/tree_hash.rs deleted file mode 100644 index 85e56924c..000000000 --- a/eth2/utils/ssz/src/tree_hash.rs +++ /dev/null @@ -1,107 +0,0 @@ -use hashing::hash; - -const BYTES_PER_CHUNK: usize = 32; -const HASHSIZE: usize = 32; - -pub trait TreeHash { - fn hash_tree_root(&self) -> Vec; -} - -/// Returns a 32 byte hash of 'list' - a vector of byte vectors. -/// Note that this will consume 'list'. -pub fn merkle_hash(list: &mut Vec>) -> Vec { - // flatten list - let mut chunkz = list_to_blob(list); - - // get data_len as bytes. It will hashed will the merkle root - let mut datalen = list.len().to_le_bytes().to_vec(); - zpad(&mut datalen, 32); - - // merklelize - while chunkz.len() > HASHSIZE { - let mut new_chunkz: Vec = Vec::new(); - - for two_chunks in chunkz.chunks(BYTES_PER_CHUNK * 2) { - // Hash two chuncks together - new_chunkz.append(&mut hash(two_chunks)); - } - - chunkz = new_chunkz; - } - - chunkz.append(&mut datalen); - hash(&chunkz) -} - -fn list_to_blob(list: &mut Vec>) -> Vec { - // pack - fit as many many items per chunk as we can and then - // right pad to BYTES_PER_CHUNCK - let (items_per_chunk, chunk_count) = if list.is_empty() { - (1, 1) - } else { - let items_per_chunk = BYTES_PER_CHUNK / list[0].len(); - let chunk_count = list.len() / items_per_chunk; - (items_per_chunk, chunk_count) - }; - - let mut chunkz = Vec::new(); - if list.is_empty() { - // handle and empty list - chunkz.append(&mut vec![0; BYTES_PER_CHUNK * 2]); - } else if list[0].len() <= BYTES_PER_CHUNK { - // just create a blob here; we'll divide into - // chunked slices when we merklize - let mut chunk = Vec::with_capacity(BYTES_PER_CHUNK); - let mut item_count_in_chunk = 0; - chunkz.reserve(chunk_count * BYTES_PER_CHUNK); - for item in list.iter_mut() { - item_count_in_chunk += 1; - chunk.append(item); - - // completed chunk? - if item_count_in_chunk == items_per_chunk { - zpad(&mut chunk, BYTES_PER_CHUNK); - chunkz.append(&mut chunk); - item_count_in_chunk = 0; - } - } - - // left-over uncompleted chunk? - if item_count_in_chunk != 0 { - zpad(&mut chunk, BYTES_PER_CHUNK); - chunkz.append(&mut chunk); - } - } - - // extend the number of chunks to a power of two if necessary - if !chunk_count.is_power_of_two() { - let zero_chunks_count = chunk_count.next_power_of_two() - chunk_count; - chunkz.append(&mut vec![0; zero_chunks_count * BYTES_PER_CHUNK]); - } - - chunkz -} - -/// right pads with zeros making 'bytes' 'size' in length -fn zpad(bytes: &mut Vec, size: usize) { - if bytes.len() < size { - bytes.resize(size, 0); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_merkle_hash() { - let data1 = vec![1; 32]; - let data2 = vec![2; 32]; - let data3 = vec![3; 32]; - let mut list = vec![data1, data2, data3]; - let result = merkle_hash(&mut list); - - //note: should test againt a known test hash value - assert_eq!(HASHSIZE, result.len()); - } -} diff --git a/eth2/utils/ssz_derive/src/lib.rs b/eth2/utils/ssz_derive/src/lib.rs index ce2538785..c5b53e934 100644 --- a/eth2/utils/ssz_derive/src/lib.rs +++ b/eth2/utils/ssz_derive/src/lib.rs @@ -38,7 +38,7 @@ extern crate proc_macro; use proc_macro::TokenStream; -use quote::{quote, ToTokens}; +use quote::quote; use syn::{parse_macro_input, DeriveInput}; /// Returns a Vec of `syn::Ident` for each named field in the struct. @@ -188,157 +188,3 @@ pub fn ssz_decode_derive(input: TokenStream) -> TokenStream { }; output.into() } - -/// Returns a Vec of `syn::Ident` for each named field in the struct, whilst filtering out fields -/// that should not be tree hashed. -/// -/// # Panics -/// Any unnamed struct field (like in a tuple struct) will raise a panic at compile time. -fn get_tree_hashable_named_field_idents<'a>( - struct_data: &'a syn::DataStruct, -) -> Vec<&'a syn::Ident> { - struct_data - .fields - .iter() - .filter_map(|f| { - if should_skip_tree_hash(&f) { - None - } else { - Some(match &f.ident { - Some(ref ident) => ident, - _ => panic!("ssz_derive only supports named struct fields."), - }) - } - }) - .collect() -} - -/// Returns true if some field has an attribute declaring it should not be tree-hashed. -/// -/// The field attribute is: `#[tree_hash(skip_hashing)]` -fn should_skip_tree_hash(field: &syn::Field) -> bool { - for attr in &field.attrs { - if attr.into_token_stream().to_string() == "# [ tree_hash ( skip_hashing ) ]" { - return true; - } - } - false -} - -/// Implements `ssz::TreeHash` for some `struct`. -/// -/// Fields are processed in the order they are defined. -#[proc_macro_derive(TreeHash, attributes(tree_hash))] -pub fn ssz_tree_hash_derive(input: TokenStream) -> TokenStream { - let item = parse_macro_input!(input as DeriveInput); - - let name = &item.ident; - - let struct_data = match &item.data { - syn::Data::Struct(s) => s, - _ => panic!("ssz_derive only supports structs."), - }; - - let field_idents = get_tree_hashable_named_field_idents(&struct_data); - - let output = quote! { - impl ssz::TreeHash for #name { - fn hash_tree_root(&self) -> Vec { - let mut list: Vec> = Vec::new(); - #( - list.push(self.#field_idents.hash_tree_root()); - )* - - ssz::merkle_hash(&mut list) - } - } - }; - output.into() -} - -/// Returns `true` if some `Ident` should be considered to be a signature type. -fn type_ident_is_signature(ident: &syn::Ident) -> bool { - match ident.to_string().as_ref() { - "Signature" => true, - "AggregateSignature" => true, - _ => false, - } -} - -/// Takes a `Field` where the type (`ty`) portion is a path (e.g., `types::Signature`) and returns -/// the final `Ident` in that path. -/// -/// E.g., for `types::Signature` returns `Signature`. -fn final_type_ident(field: &syn::Field) -> &syn::Ident { - match &field.ty { - syn::Type::Path(path) => &path.path.segments.last().unwrap().value().ident, - _ => panic!("ssz_derive only supports Path types."), - } -} - -/// Implements `ssz::TreeHash` for some `struct`, whilst excluding any fields following and -/// including a field that is of type "Signature" or "AggregateSignature". -/// -/// See: -/// https://github.com/ethereum/eth2.0-specs/blob/master/specs/simple-serialize.md#signed-roots -/// -/// This is a rather horrendous macro, it will read the type of the object as a string and decide -/// if it's a signature by matching that string against "Signature" or "AggregateSignature". So, -/// it's important that you use those exact words as your type -- don't alias it to something else. -/// -/// If you can think of a better way to do this, please make an issue! -/// -/// Fields are processed in the order they are defined. -#[proc_macro_derive(SignedRoot, attributes(signed_root))] -pub fn ssz_signed_root_derive(input: TokenStream) -> TokenStream { - let item = parse_macro_input!(input as DeriveInput); - - let name = &item.ident; - - let struct_data = match &item.data { - syn::Data::Struct(s) => s, - _ => panic!("ssz_derive only supports structs."), - }; - - let mut field_idents: Vec<&syn::Ident> = vec![]; - - let field_idents = get_signed_root_named_field_idents(&struct_data); - - let output = quote! { - impl ssz::SignedRoot for #name { - fn signed_root(&self) -> Vec { - let mut list: Vec> = Vec::new(); - #( - list.push(self.#field_idents.hash_tree_root()); - )* - - ssz::merkle_hash(&mut list) - } - } - }; - output.into() -} - -fn get_signed_root_named_field_idents(struct_data: &syn::DataStruct) -> Vec<&syn::Ident> { - struct_data - .fields - .iter() - .filter_map(|f| { - if should_skip_signed_root(&f) { - None - } else { - Some(match &f.ident { - Some(ref ident) => ident, - _ => panic!("ssz_derive only supports named struct fields"), - }) - } - }) - .collect() -} - -fn should_skip_signed_root(field: &syn::Field) -> bool { - field - .attrs - .iter() - .any(|attr| attr.into_token_stream().to_string() == "# [ signed_root ( skip_hashing ) ]") -} diff --git a/eth2/utils/ssz_derive/tests/test_derives.rs b/eth2/utils/ssz_derive/tests/test_derives.rs deleted file mode 100644 index e025dc3a5..000000000 --- a/eth2/utils/ssz_derive/tests/test_derives.rs +++ /dev/null @@ -1,94 +0,0 @@ -use ssz::{SignedRoot, TreeHash}; -use ssz_derive::{SignedRoot, TreeHash}; - -#[derive(TreeHash, SignedRoot)] -struct CryptoKitties { - best_kitty: u64, - worst_kitty: u8, - kitties: Vec, -} - -impl CryptoKitties { - fn new() -> Self { - CryptoKitties { - best_kitty: 9999, - worst_kitty: 1, - kitties: vec![2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43], - } - } - - fn hash(&self) -> Vec { - let mut list: Vec> = Vec::new(); - list.push(self.best_kitty.hash_tree_root()); - list.push(self.worst_kitty.hash_tree_root()); - list.push(self.kitties.hash_tree_root()); - ssz::merkle_hash(&mut list) - } -} - -#[test] -fn test_cryptokitties_hash() { - let kitties = CryptoKitties::new(); - let expected_hash = vec![ - 201, 9, 139, 14, 24, 247, 21, 55, 132, 211, 51, 125, 183, 186, 177, 33, 147, 210, 42, 108, - 174, 162, 221, 227, 157, 179, 15, 7, 97, 239, 82, 220, - ]; - assert_eq!(kitties.hash(), expected_hash); -} - -#[test] -fn test_simple_tree_hash_derive() { - let kitties = CryptoKitties::new(); - assert_eq!(kitties.hash_tree_root(), kitties.hash()); -} - -#[test] -fn test_simple_signed_root_derive() { - let kitties = CryptoKitties::new(); - assert_eq!(kitties.signed_root(), kitties.hash()); -} - -#[derive(TreeHash, SignedRoot)] -struct Casper { - friendly: bool, - #[tree_hash(skip_hashing)] - friends: Vec, - #[signed_root(skip_hashing)] - dead: bool, -} - -impl Casper { - fn new() -> Self { - Casper { - friendly: true, - friends: vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - dead: true, - } - } - - fn expected_signed_hash(&self) -> Vec { - let mut list = Vec::new(); - list.push(self.friendly.hash_tree_root()); - list.push(self.friends.hash_tree_root()); - ssz::merkle_hash(&mut list) - } - - fn expected_tree_hash(&self) -> Vec { - let mut list = Vec::new(); - list.push(self.friendly.hash_tree_root()); - list.push(self.dead.hash_tree_root()); - ssz::merkle_hash(&mut list) - } -} - -#[test] -fn test_annotated_tree_hash_derive() { - let casper = Casper::new(); - assert_eq!(casper.hash_tree_root(), casper.expected_tree_hash()); -} - -#[test] -fn test_annotated_signed_root_derive() { - let casper = Casper::new(); - assert_eq!(casper.signed_root(), casper.expected_signed_hash()); -} diff --git a/eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs b/eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs index e7e1e18e6..f60d793f2 100644 --- a/eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs +++ b/eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs @@ -18,6 +18,8 @@ const TOTAL_SIZE: usize = SEED_SIZE + ROUND_SIZE + POSITION_WINDOW_SIZE; /// Credits to [@protolambda](https://github.com/protolambda) for defining this algorithm. /// /// Shuffles if `forwards == true`, otherwise un-shuffles. +/// It holds that: shuffle_list(shuffle_list(l, r, s, true), r, s, false) == l +/// and: shuffle_list(shuffle_list(l, r, s, false), r, s, true) == l /// /// Returns `None` under any of the following conditions: /// - `list_size == 0` diff --git a/eth2/utils/tree_hash/Cargo.toml b/eth2/utils/tree_hash/Cargo.toml new file mode 100644 index 000000000..328d91577 --- /dev/null +++ b/eth2/utils/tree_hash/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "tree_hash" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" + +[dependencies] +ethereum-types = "0.5" +hashing = { path = "../hashing" } +int_to_bytes = { path = "../int_to_bytes" } diff --git a/eth2/utils/tree_hash/src/cached_tree_hash.rs b/eth2/utils/tree_hash/src/cached_tree_hash.rs new file mode 100644 index 000000000..e093b2dd7 --- /dev/null +++ b/eth2/utils/tree_hash/src/cached_tree_hash.rs @@ -0,0 +1,309 @@ +use super::*; +use hashing::hash; +use int_to_bytes::int_to_bytes32; +use std::ops::Range; + +pub mod btree_overlay; +pub mod impls; +pub mod resize; + +pub use btree_overlay::BTreeOverlay; + +#[derive(Debug, PartialEq, Clone)] +pub enum Error { + ShouldNotProduceBTreeOverlay, + NoFirstNode, + NoBytesForRoot, + UnableToObtainSlices, + UnableToGrowMerkleTree, + UnableToShrinkMerkleTree, + ShouldNeverBePacked(TreeHashType), + BytesAreNotEvenChunks(usize), + NoModifiedFieldForChunk(usize), + NoBytesForChunk(usize), +} + +pub trait CachedTreeHash: CachedTreeHashSubTree + Sized { + fn update_internal_tree_hash_cache(self, old: T) -> Result<(Self, Self), Error>; + + fn cached_tree_hash_root(&self) -> Option>; + + fn clone_without_tree_hash_cache(&self) -> Self; +} + +pub trait CachedTreeHashSubTree: TreeHash { + fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result; + + fn new_tree_hash_cache(&self) -> Result; + + fn update_tree_hash_cache( + &self, + other: &Item, + cache: &mut TreeHashCache, + chunk: usize, + ) -> Result; +} + +fn children(parent: usize) -> (usize, usize) { + ((2 * parent + 1), (2 * parent + 2)) +} + +fn node_range_to_byte_range(node_range: &Range) -> Range { + node_range.start * HASHSIZE..node_range.end * HASHSIZE +} + +/// Split `values` into a power-of-two, identical-length chunks (padding with `0`) and merkleize +/// them, returning the entire merkle tree. +/// +/// The root hash is `merkleize(values)[0..BYTES_PER_CHUNK]`. +pub fn merkleize(values: Vec) -> Vec { + let values = sanitise_bytes(values); + + let leaves = values.len() / HASHSIZE; + + if leaves == 0 { + panic!("No full leaves"); + } + + if !leaves.is_power_of_two() { + panic!("leaves is not power of two"); + } + + let mut o: Vec = vec![0; (num_nodes(leaves) - leaves) * HASHSIZE]; + o.append(&mut values.to_vec()); + + let mut i = o.len(); + let mut j = o.len() - values.len(); + + while i >= MERKLE_HASH_CHUNCK { + i -= MERKLE_HASH_CHUNCK; + let hash = hash(&o[i..i + MERKLE_HASH_CHUNCK]); + + j -= HASHSIZE; + o[j..j + HASHSIZE].copy_from_slice(&hash); + } + + o +} + +pub fn sanitise_bytes(mut bytes: Vec) -> Vec { + let present_leaves = num_unsanitized_leaves(bytes.len()); + let required_leaves = present_leaves.next_power_of_two(); + + if (present_leaves != required_leaves) | last_leaf_needs_padding(bytes.len()) { + bytes.resize(num_bytes(required_leaves), 0); + } + + bytes +} + +fn pad_for_leaf_count(num_leaves: usize, bytes: &mut Vec) { + let required_leaves = num_leaves.next_power_of_two(); + + bytes.resize( + bytes.len() + (required_leaves - num_leaves) * BYTES_PER_CHUNK, + 0, + ); +} + +fn last_leaf_needs_padding(num_bytes: usize) -> bool { + num_bytes % HASHSIZE != 0 +} + +/// Rounds up +fn num_unsanitized_leaves(num_bytes: usize) -> usize { + (num_bytes + HASHSIZE - 1) / HASHSIZE +} + +fn num_bytes(num_leaves: usize) -> usize { + num_leaves * HASHSIZE +} + +#[derive(Debug, PartialEq, Clone)] +pub struct TreeHashCache { + cache: Vec, + chunk_modified: Vec, +} + +impl Into> for TreeHashCache { + fn into(self) -> Vec { + self.cache + } +} + +impl TreeHashCache { + pub fn new(item: &T) -> Result + where + T: CachedTreeHashSubTree, + { + item.new_tree_hash_cache() + } + + pub fn from_elems(cache: Vec, chunk_modified: Vec) -> Self { + Self { + cache, + chunk_modified, + } + } + + pub fn from_leaves_and_subtrees( + item: &T, + leaves_and_subtrees: Vec, + ) -> Result + where + T: CachedTreeHashSubTree, + { + let offset_handler = BTreeOverlay::new(item, 0)?; + + // Note how many leaves were provided. If is not a power-of-two, we'll need to pad it out + // later. + let num_provided_leaf_nodes = leaves_and_subtrees.len(); + + // Allocate enough bytes to store the internal nodes and the leaves and subtrees, then fill + // all the to-be-built internal nodes with zeros and append the leaves and subtrees. + let internal_node_bytes = offset_handler.num_internal_nodes * BYTES_PER_CHUNK; + let leaves_and_subtrees_bytes = leaves_and_subtrees + .iter() + .fold(0, |acc, t| acc + t.bytes_len()); + let mut cache = Vec::with_capacity(leaves_and_subtrees_bytes + internal_node_bytes); + cache.resize(internal_node_bytes, 0); + + // Allocate enough bytes to store all the leaves. + let mut leaves = Vec::with_capacity(offset_handler.num_leaf_nodes * HASHSIZE); + + // Iterate through all of the leaves/subtrees, adding their root as a leaf node and then + // concatenating their merkle trees. + for t in leaves_and_subtrees { + leaves.append(&mut t.root().ok_or_else(|| Error::NoBytesForRoot)?.to_vec()); + cache.append(&mut t.into_merkle_tree()); + } + + // Pad the leaves to an even power-of-two, using zeros. + pad_for_leaf_count(num_provided_leaf_nodes, &mut cache); + + // Merkleize the leaves, then split the leaf nodes off them. Then, replace all-zeros + // internal nodes created earlier with the internal nodes generated by `merkleize`. + let mut merkleized = merkleize(leaves); + merkleized.split_off(internal_node_bytes); + cache.splice(0..internal_node_bytes, merkleized); + + Ok(Self { + chunk_modified: vec![false; cache.len() / BYTES_PER_CHUNK], + cache, + }) + } + + pub fn from_bytes(bytes: Vec, initial_modified_state: bool) -> Result { + if bytes.len() % BYTES_PER_CHUNK > 0 { + return Err(Error::BytesAreNotEvenChunks(bytes.len())); + } + + Ok(Self { + chunk_modified: vec![initial_modified_state; bytes.len() / BYTES_PER_CHUNK], + cache: bytes, + }) + } + + pub fn bytes_len(&self) -> usize { + self.cache.len() + } + + pub fn root(&self) -> Option<&[u8]> { + self.cache.get(0..HASHSIZE) + } + + pub fn splice(&mut self, chunk_range: Range, replace_with: Self) { + let (bytes, bools) = replace_with.into_components(); + + // Update the `chunk_modified` vec, marking all spliced-in nodes as changed. + self.chunk_modified.splice(chunk_range.clone(), bools); + self.cache + .splice(node_range_to_byte_range(&chunk_range), bytes); + } + + pub fn maybe_update_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> { + let start = chunk * BYTES_PER_CHUNK; + let end = start + BYTES_PER_CHUNK; + + if !self.chunk_equals(chunk, to)? { + self.cache + .get_mut(start..end) + .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))? + .copy_from_slice(to); + self.chunk_modified[chunk] = true; + } + + Ok(()) + } + + pub fn slices(&self, chunk_range: Range) -> Option<(&[u8], &[bool])> { + Some(( + self.cache.get(node_range_to_byte_range(&chunk_range))?, + self.chunk_modified.get(chunk_range)?, + )) + } + + pub fn modify_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> { + let start = chunk * BYTES_PER_CHUNK; + let end = start + BYTES_PER_CHUNK; + + self.cache + .get_mut(start..end) + .ok_or_else(|| Error::NoBytesForChunk(chunk))? + .copy_from_slice(to); + + self.chunk_modified[chunk] = true; + + Ok(()) + } + + pub fn get_chunk(&self, chunk: usize) -> Result<&[u8], Error> { + let start = chunk * BYTES_PER_CHUNK; + let end = start + BYTES_PER_CHUNK; + + Ok(self + .cache + .get(start..end) + .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))?) + } + + pub fn chunk_equals(&mut self, chunk: usize, other: &[u8]) -> Result { + Ok(self.get_chunk(chunk)? == other) + } + + pub fn changed(&self, chunk: usize) -> Result { + self.chunk_modified + .get(chunk) + .cloned() + .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk)) + } + + pub fn either_modified(&self, children: (&usize, &usize)) -> Result { + Ok(self.changed(*children.0)? | self.changed(*children.1)?) + } + + pub fn hash_children(&self, children: (&usize, &usize)) -> Result, Error> { + let mut child_bytes = Vec::with_capacity(BYTES_PER_CHUNK * 2); + child_bytes.append(&mut self.get_chunk(*children.0)?.to_vec()); + child_bytes.append(&mut self.get_chunk(*children.1)?.to_vec()); + + Ok(hash(&child_bytes)) + } + + pub fn mix_in_length(&self, chunk: usize, length: usize) -> Result, Error> { + let mut bytes = Vec::with_capacity(2 * BYTES_PER_CHUNK); + + bytes.append(&mut self.get_chunk(chunk)?.to_vec()); + bytes.append(&mut int_to_bytes32(length as u64)); + + Ok(hash(&bytes)) + } + + pub fn into_merkle_tree(self) -> Vec { + self.cache + } + + pub fn into_components(self) -> (Vec, Vec) { + (self.cache, self.chunk_modified) + } +} diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs b/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs new file mode 100644 index 000000000..e8c04a91e --- /dev/null +++ b/eth2/utils/tree_hash/src/cached_tree_hash/btree_overlay.rs @@ -0,0 +1,100 @@ +use super::*; + +#[derive(Debug)] +pub struct BTreeOverlay { + pub num_internal_nodes: usize, + pub num_leaf_nodes: usize, + pub first_node: usize, + pub next_node: usize, + offsets: Vec, +} + +impl BTreeOverlay { + pub fn new(item: &T, initial_offset: usize) -> Result + where + T: CachedTreeHashSubTree, + { + item.tree_hash_cache_overlay(initial_offset) + } + + pub fn from_lengths(offset: usize, mut lengths: Vec) -> Result { + // Extend it to the next power-of-two, if it is not already. + let num_leaf_nodes = if lengths.len().is_power_of_two() { + lengths.len() + } else { + let num_leaf_nodes = lengths.len().next_power_of_two(); + lengths.resize(num_leaf_nodes, 1); + num_leaf_nodes + }; + + let num_nodes = num_nodes(num_leaf_nodes); + let num_internal_nodes = num_nodes - num_leaf_nodes; + + let mut offsets = Vec::with_capacity(num_nodes); + offsets.append(&mut (offset..offset + num_internal_nodes).collect()); + + let mut next_node = num_internal_nodes + offset; + for i in 0..num_leaf_nodes { + offsets.push(next_node); + next_node += lengths[i]; + } + + Ok(Self { + num_internal_nodes, + num_leaf_nodes, + offsets, + first_node: offset, + next_node, + }) + } + + pub fn root(&self) -> usize { + self.first_node + } + + pub fn height(&self) -> usize { + self.num_leaf_nodes.trailing_zeros() as usize + } + + pub fn chunk_range(&self) -> Range { + self.first_node..self.next_node + } + + pub fn total_chunks(&self) -> usize { + self.next_node - self.first_node + } + + pub fn total_nodes(&self) -> usize { + self.num_internal_nodes + self.num_leaf_nodes + } + + pub fn first_leaf_node(&self) -> Result { + self.offsets + .get(self.num_internal_nodes) + .cloned() + .ok_or_else(|| Error::NoFirstNode) + } + + /// Returns an iterator visiting each internal node, providing the left and right child chunks + /// for the node. + pub fn iter_internal_nodes<'a>( + &'a self, + ) -> impl DoubleEndedIterator { + let internal_nodes = &self.offsets[0..self.num_internal_nodes]; + + internal_nodes.iter().enumerate().map(move |(i, parent)| { + let children = children(i); + ( + parent, + (&self.offsets[children.0], &self.offsets[children.1]), + ) + }) + } + + /// Returns an iterator visiting each leaf node, providing the chunk for that node. + pub fn iter_leaf_nodes<'a>(&'a self) -> impl DoubleEndedIterator { + let leaf_nodes = &self.offsets[self.num_internal_nodes..]; + + leaf_nodes.iter() + } +} diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs new file mode 100644 index 000000000..6500e4eff --- /dev/null +++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls.rs @@ -0,0 +1,31 @@ +use super::resize::{grow_merkle_cache, shrink_merkle_cache}; +use super::*; + +mod vec; + +impl CachedTreeHashSubTree for u64 { + fn new_tree_hash_cache(&self) -> Result { + Ok(TreeHashCache::from_bytes( + merkleize(self.to_le_bytes().to_vec()), + false, + )?) + } + + fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result { + BTreeOverlay::from_lengths(chunk_offset, vec![1]) + } + + fn update_tree_hash_cache( + &self, + other: &Self, + cache: &mut TreeHashCache, + chunk: usize, + ) -> Result { + if self != other { + let leaf = merkleize(self.to_le_bytes().to_vec()); + cache.modify_chunk(chunk, &leaf)?; + } + + Ok(chunk + 1) + } +} diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs new file mode 100644 index 000000000..1cd7eb902 --- /dev/null +++ b/eth2/utils/tree_hash/src/cached_tree_hash/impls/vec.rs @@ -0,0 +1,171 @@ +use super::*; + +impl CachedTreeHashSubTree> for Vec +where + T: CachedTreeHashSubTree + TreeHash, +{ + fn new_tree_hash_cache(&self) -> Result { + match T::tree_hash_type() { + TreeHashType::Basic => { + TreeHashCache::from_bytes(merkleize(get_packed_leaves(self)?), false) + } + TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => { + let subtrees = self + .iter() + .map(|item| TreeHashCache::new(item)) + .collect::, _>>()?; + + TreeHashCache::from_leaves_and_subtrees(self, subtrees) + } + } + } + + fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result { + let lengths = match T::tree_hash_type() { + TreeHashType::Basic => vec![1; self.len() / T::tree_hash_packing_factor()], + TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => { + let mut lengths = vec![]; + + for item in self { + lengths.push(BTreeOverlay::new(item, 0)?.total_nodes()) + } + + lengths + } + }; + + BTreeOverlay::from_lengths(chunk_offset, lengths) + } + + fn update_tree_hash_cache( + &self, + other: &Vec, + cache: &mut TreeHashCache, + chunk: usize, + ) -> Result { + let offset_handler = BTreeOverlay::new(self, chunk)?; + let old_offset_handler = BTreeOverlay::new(other, chunk)?; + + if offset_handler.num_leaf_nodes != old_offset_handler.num_leaf_nodes { + let old_offset_handler = BTreeOverlay::new(other, chunk)?; + + // Get slices of the exsiting tree from the cache. + let (old_bytes, old_flags) = cache + .slices(old_offset_handler.chunk_range()) + .ok_or_else(|| Error::UnableToObtainSlices)?; + + let (new_bytes, new_flags) = + if offset_handler.num_leaf_nodes > old_offset_handler.num_leaf_nodes { + grow_merkle_cache( + old_bytes, + old_flags, + old_offset_handler.height(), + offset_handler.height(), + ) + .ok_or_else(|| Error::UnableToGrowMerkleTree)? + } else { + shrink_merkle_cache( + old_bytes, + old_flags, + old_offset_handler.height(), + offset_handler.height(), + offset_handler.total_chunks(), + ) + .ok_or_else(|| Error::UnableToShrinkMerkleTree)? + }; + + // Create a `TreeHashCache` from the raw elements. + let modified_cache = TreeHashCache::from_elems(new_bytes, new_flags); + + // Splice the newly created `TreeHashCache` over the existing elements. + cache.splice(old_offset_handler.chunk_range(), modified_cache); + } + + match T::tree_hash_type() { + TreeHashType::Basic => { + let leaves = get_packed_leaves(self)?; + + for (i, chunk) in offset_handler.iter_leaf_nodes().enumerate() { + if let Some(latest) = leaves.get(i * HASHSIZE..(i + 1) * HASHSIZE) { + cache.maybe_update_chunk(*chunk, latest)?; + } + } + let first_leaf_chunk = offset_handler.first_leaf_node()?; + + cache.splice( + first_leaf_chunk..offset_handler.next_node, + TreeHashCache::from_bytes(leaves, true)?, + ); + } + TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => { + let mut i = offset_handler.num_leaf_nodes; + for &start_chunk in offset_handler.iter_leaf_nodes().rev() { + i -= 1; + match (other.get(i), self.get(i)) { + // The item existed in the previous list and exsits in the current list. + (Some(old), Some(new)) => { + new.update_tree_hash_cache(old, cache, start_chunk)?; + } + // The item existed in the previous list but does not exist in this list. + // + // I.e., the list has been shortened. + (Some(old), None) => { + // Splice out the entire tree of the removed node, replacing it with a + // single padding node. + let end_chunk = BTreeOverlay::new(old, start_chunk)?.next_node; + + cache.splice( + start_chunk..end_chunk, + TreeHashCache::from_bytes(vec![0; HASHSIZE], true)?, + ); + } + // The item existed in the previous list but does exist in this list. + // + // I.e., the list has been lengthened. + (None, Some(new)) => { + let bytes: Vec = TreeHashCache::new(new)?.into(); + + cache.splice( + start_chunk..start_chunk + 1, + TreeHashCache::from_bytes(bytes, true)?, + ); + } + // The item didn't exist in the old list and doesn't exist in the new list, + // nothing to do. + (None, None) => {} + }; + } + } + } + + for (&parent, children) in offset_handler.iter_internal_nodes().rev() { + if cache.either_modified(children)? { + cache.modify_chunk(parent, &cache.hash_children(children)?)?; + } + } + + // If the root node or the length has changed, mix in the length of the list. + let root_node = offset_handler.root(); + if cache.changed(root_node)? | (self.len() != other.len()) { + cache.modify_chunk(root_node, &cache.mix_in_length(root_node, self.len())?)?; + } + + Ok(offset_handler.next_node) + } +} + +fn get_packed_leaves(vec: &Vec) -> Result, Error> +where + T: CachedTreeHashSubTree, +{ + let num_packed_bytes = (BYTES_PER_CHUNK / T::tree_hash_packing_factor()) * vec.len(); + let num_leaves = num_sanitized_leaves(num_packed_bytes); + + let mut packed = Vec::with_capacity(num_leaves * HASHSIZE); + + for item in vec { + packed.append(&mut item.tree_hash_packed_encoding()); + } + + Ok(sanitise_bytes(packed)) +} diff --git a/eth2/utils/tree_hash/src/cached_tree_hash/resize.rs b/eth2/utils/tree_hash/src/cached_tree_hash/resize.rs new file mode 100644 index 000000000..44b3f0ea5 --- /dev/null +++ b/eth2/utils/tree_hash/src/cached_tree_hash/resize.rs @@ -0,0 +1,276 @@ +use super::*; + +/// New vec is bigger than old vec. +pub fn grow_merkle_cache( + old_bytes: &[u8], + old_flags: &[bool], + from_height: usize, + to_height: usize, +) -> Option<(Vec, Vec)> { + // Determine the size of our new tree. It is not just a simple `1 << to_height` as there can be + // an arbitrary number of nodes in `old_bytes` leaves if those leaves are subtrees. + let to_nodes = { + let old_nodes = old_bytes.len() / HASHSIZE; + let additional_nodes = old_nodes - nodes_in_tree_of_height(from_height); + nodes_in_tree_of_height(to_height) + additional_nodes + }; + + let mut bytes = vec![0; to_nodes * HASHSIZE]; + let mut flags = vec![true; to_nodes]; + + let leaf_level = from_height; + + for i in 0..=from_height as usize { + // If we're on the leaf slice, grab the first byte and all the of the bytes after that. + // This is required because we can have an arbitrary number of bytes at the leaf level + // (e.g., the case where there are subtrees as leaves). + // + // If we're not on a leaf level, the number of nodes is fixed and known. + let (old_byte_slice, old_flag_slice) = if i == leaf_level { + ( + old_bytes.get(first_byte_at_height(i)..)?, + old_flags.get(first_node_at_height(i)..)?, + ) + } else { + ( + old_bytes.get(byte_range_at_height(i))?, + old_flags.get(node_range_at_height(i))?, + ) + }; + + let new_i = i + to_height - from_height; + let (new_byte_slice, new_flag_slice) = if i == leaf_level { + ( + bytes.get_mut(first_byte_at_height(new_i)..)?, + flags.get_mut(first_node_at_height(new_i)..)?, + ) + } else { + ( + bytes.get_mut(byte_range_at_height(new_i))?, + flags.get_mut(node_range_at_height(new_i))?, + ) + }; + + new_byte_slice + .get_mut(0..old_byte_slice.len())? + .copy_from_slice(old_byte_slice); + new_flag_slice + .get_mut(0..old_flag_slice.len())? + .copy_from_slice(old_flag_slice); + } + + Some((bytes, flags)) +} + +/// New vec is smaller than old vec. +pub fn shrink_merkle_cache( + from_bytes: &[u8], + from_flags: &[bool], + from_height: usize, + to_height: usize, + to_nodes: usize, +) -> Option<(Vec, Vec)> { + let mut bytes = vec![0; to_nodes * HASHSIZE]; + let mut flags = vec![true; to_nodes]; + + for i in 0..=to_height as usize { + let from_i = i + from_height - to_height; + + let (from_byte_slice, from_flag_slice) = if from_i == from_height { + ( + from_bytes.get(first_byte_at_height(from_i)..)?, + from_flags.get(first_node_at_height(from_i)..)?, + ) + } else { + ( + from_bytes.get(byte_range_at_height(from_i))?, + from_flags.get(node_range_at_height(from_i))?, + ) + }; + + let (to_byte_slice, to_flag_slice) = if i == to_height { + ( + bytes.get_mut(first_byte_at_height(i)..)?, + flags.get_mut(first_node_at_height(i)..)?, + ) + } else { + ( + bytes.get_mut(byte_range_at_height(i))?, + flags.get_mut(node_range_at_height(i))?, + ) + }; + + to_byte_slice.copy_from_slice(from_byte_slice.get(0..to_byte_slice.len())?); + to_flag_slice.copy_from_slice(from_flag_slice.get(0..to_flag_slice.len())?); + } + + Some((bytes, flags)) +} + +fn nodes_in_tree_of_height(h: usize) -> usize { + 2 * (1 << h) - 1 +} + +fn byte_range_at_height(h: usize) -> Range { + let node_range = node_range_at_height(h); + node_range.start * HASHSIZE..node_range.end * HASHSIZE +} + +fn node_range_at_height(h: usize) -> Range { + first_node_at_height(h)..last_node_at_height(h) + 1 +} + +fn first_byte_at_height(h: usize) -> usize { + first_node_at_height(h) * HASHSIZE +} + +fn first_node_at_height(h: usize) -> usize { + (1 << h) - 1 +} + +fn last_node_at_height(h: usize) -> usize { + (1 << (h + 1)) - 2 +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn can_grow_and_shrink_three_levels() { + let small: usize = 1; + let big: usize = 15; + + let original_bytes = vec![42; small * HASHSIZE]; + let original_flags = vec![false; small]; + + let (grown_bytes, grown_flags) = grow_merkle_cache( + &original_bytes, + &original_flags, + (small + 1).trailing_zeros() as usize - 1, + (big + 1).trailing_zeros() as usize - 1, + ) + .unwrap(); + + let mut expected_bytes = vec![]; + let mut expected_flags = vec![]; + // First level + expected_bytes.append(&mut vec![0; 32]); + expected_flags.push(true); + // Second level + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_flags.push(true); + expected_flags.push(true); + // Third level + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_flags.push(true); + expected_flags.push(true); + expected_flags.push(true); + expected_flags.push(true); + // Fourth level + expected_bytes.append(&mut vec![42; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_flags.push(false); + expected_flags.push(true); + expected_flags.push(true); + expected_flags.push(true); + expected_flags.push(true); + expected_flags.push(true); + expected_flags.push(true); + expected_flags.push(true); + + assert_eq!(expected_bytes, grown_bytes); + assert_eq!(expected_flags, grown_flags); + + let (shrunk_bytes, shrunk_flags) = shrink_merkle_cache( + &grown_bytes, + &grown_flags, + (big + 1).trailing_zeros() as usize - 1, + (small + 1).trailing_zeros() as usize - 1, + small, + ) + .unwrap(); + + assert_eq!(original_bytes, shrunk_bytes); + assert_eq!(original_flags, shrunk_flags); + } + + #[test] + fn can_grow_and_shrink_one_level() { + let small: usize = 7; + let big: usize = 15; + + let original_bytes = vec![42; small * HASHSIZE]; + let original_flags = vec![false; small]; + + let (grown_bytes, grown_flags) = grow_merkle_cache( + &original_bytes, + &original_flags, + (small + 1).trailing_zeros() as usize - 1, + (big + 1).trailing_zeros() as usize - 1, + ) + .unwrap(); + + let mut expected_bytes = vec![]; + let mut expected_flags = vec![]; + // First level + expected_bytes.append(&mut vec![0; 32]); + expected_flags.push(true); + // Second level + expected_bytes.append(&mut vec![42; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_flags.push(false); + expected_flags.push(true); + // Third level + expected_bytes.append(&mut vec![42; 32]); + expected_bytes.append(&mut vec![42; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_flags.push(false); + expected_flags.push(false); + expected_flags.push(true); + expected_flags.push(true); + // Fourth level + expected_bytes.append(&mut vec![42; 32]); + expected_bytes.append(&mut vec![42; 32]); + expected_bytes.append(&mut vec![42; 32]); + expected_bytes.append(&mut vec![42; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_bytes.append(&mut vec![0; 32]); + expected_flags.push(false); + expected_flags.push(false); + expected_flags.push(false); + expected_flags.push(false); + expected_flags.push(true); + expected_flags.push(true); + expected_flags.push(true); + expected_flags.push(true); + + assert_eq!(expected_bytes, grown_bytes); + assert_eq!(expected_flags, grown_flags); + + let (shrunk_bytes, shrunk_flags) = shrink_merkle_cache( + &grown_bytes, + &grown_flags, + (big + 1).trailing_zeros() as usize - 1, + (small + 1).trailing_zeros() as usize - 1, + small, + ) + .unwrap(); + + assert_eq!(original_bytes, shrunk_bytes); + assert_eq!(original_flags, shrunk_flags); + } +} diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs new file mode 100644 index 000000000..ed60079c8 --- /dev/null +++ b/eth2/utils/tree_hash/src/lib.rs @@ -0,0 +1,73 @@ +pub mod cached_tree_hash; +pub mod signed_root; +pub mod standard_tree_hash; + +pub const BYTES_PER_CHUNK: usize = 32; +pub const HASHSIZE: usize = 32; +pub const MERKLE_HASH_CHUNCK: usize = 2 * BYTES_PER_CHUNK; + +pub use cached_tree_hash::{BTreeOverlay, CachedTreeHashSubTree, Error, TreeHashCache}; +pub use signed_root::SignedRoot; +pub use standard_tree_hash::{merkle_root, TreeHash}; + +#[derive(Debug, PartialEq, Clone)] +pub enum TreeHashType { + Basic, + Vector, + List, + Container, +} + +fn num_sanitized_leaves(num_bytes: usize) -> usize { + let leaves = (num_bytes + HASHSIZE - 1) / HASHSIZE; + leaves.next_power_of_two() +} + +fn num_nodes(num_leaves: usize) -> usize { + 2 * num_leaves - 1 +} + +#[macro_export] +macro_rules! tree_hash_ssz_encoding_as_vector { + ($type: ident) => { + impl tree_hash::TreeHash for $type { + fn tree_hash_type() -> tree_hash::TreeHashType { + tree_hash::TreeHashType::Vector + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("Vector should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("Vector should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + tree_hash::merkle_root(&ssz::ssz_encode(self)) + } + } + }; +} +#[macro_export] +macro_rules! tree_hash_ssz_encoding_as_list { + ($type: ident) => { + impl tree_hash::TreeHash for $type { + fn tree_hash_type() -> tree_hash::TreeHashType { + tree_hash::TreeHashType::List + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("List should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("List should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + ssz::ssz_encode(self).tree_hash_root() + } + } + }; +} diff --git a/eth2/utils/ssz/src/signed_root.rs b/eth2/utils/tree_hash/src/signed_root.rs similarity index 100% rename from eth2/utils/ssz/src/signed_root.rs rename to eth2/utils/tree_hash/src/signed_root.rs diff --git a/eth2/utils/tree_hash/src/standard_tree_hash.rs b/eth2/utils/tree_hash/src/standard_tree_hash.rs new file mode 100644 index 000000000..812a2c352 --- /dev/null +++ b/eth2/utils/tree_hash/src/standard_tree_hash.rs @@ -0,0 +1,75 @@ +use super::*; +use hashing::hash; +use int_to_bytes::int_to_bytes32; + +pub use impls::vec_tree_hash_root; + +mod impls; + +pub trait TreeHash { + fn tree_hash_type() -> TreeHashType; + + fn tree_hash_packed_encoding(&self) -> Vec; + + fn tree_hash_packing_factor() -> usize; + + fn tree_hash_root(&self) -> Vec; +} + +pub fn merkle_root(bytes: &[u8]) -> Vec { + // TODO: replace this with a more memory efficient method. + efficient_merkleize(&bytes)[0..32].to_vec() +} + +pub fn efficient_merkleize(bytes: &[u8]) -> Vec { + // If the bytes are just one chunk (or less than one chunk) just return them. + if bytes.len() <= HASHSIZE { + let mut o = bytes.to_vec(); + o.resize(HASHSIZE, 0); + return o; + } + + let leaves = num_sanitized_leaves(bytes.len()); + let nodes = num_nodes(leaves); + let internal_nodes = nodes - leaves; + + let num_bytes = std::cmp::max(internal_nodes, 1) * HASHSIZE + bytes.len(); + + let mut o: Vec = vec![0; internal_nodes * HASHSIZE]; + + o.append(&mut bytes.to_vec()); + + assert_eq!(o.len(), num_bytes); + + let empty_chunk_hash = hash(&[0; MERKLE_HASH_CHUNCK]); + + let mut i = nodes * HASHSIZE; + let mut j = internal_nodes * HASHSIZE; + + while i >= MERKLE_HASH_CHUNCK { + i -= MERKLE_HASH_CHUNCK; + + j -= HASHSIZE; + let hash = match o.get(i..i + MERKLE_HASH_CHUNCK) { + // All bytes are available, hash as ususal. + Some(slice) => hash(slice), + // Unable to get all the bytes. + None => { + match o.get(i..) { + // Able to get some of the bytes, pad them out. + Some(slice) => { + let mut bytes = slice.to_vec(); + bytes.resize(MERKLE_HASH_CHUNCK, 0); + hash(&bytes) + } + // Unable to get any bytes, use the empty-chunk hash. + None => empty_chunk_hash.clone(), + } + } + }; + + o[j..j + HASHSIZE].copy_from_slice(&hash); + } + + o +} diff --git a/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs b/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs new file mode 100644 index 000000000..be6b4ba07 --- /dev/null +++ b/eth2/utils/tree_hash/src/standard_tree_hash/impls.rs @@ -0,0 +1,155 @@ +use super::*; +use ethereum_types::H256; + +macro_rules! impl_for_bitsize { + ($type: ident, $bit_size: expr) => { + impl TreeHash for $type { + fn tree_hash_type() -> TreeHashType { + TreeHashType::Basic + } + + fn tree_hash_packed_encoding(&self) -> Vec { + self.to_le_bytes().to_vec() + } + + fn tree_hash_packing_factor() -> usize { + HASHSIZE / ($bit_size / 8) + } + + fn tree_hash_root(&self) -> Vec { + int_to_bytes32(*self as u64) + } + } + }; +} + +impl_for_bitsize!(u8, 8); +impl_for_bitsize!(u16, 16); +impl_for_bitsize!(u32, 32); +impl_for_bitsize!(u64, 64); +impl_for_bitsize!(usize, 64); + +impl TreeHash for bool { + fn tree_hash_type() -> TreeHashType { + TreeHashType::Basic + } + + fn tree_hash_packed_encoding(&self) -> Vec { + (*self as u8).tree_hash_packed_encoding() + } + + fn tree_hash_packing_factor() -> usize { + u8::tree_hash_packing_factor() + } + + fn tree_hash_root(&self) -> Vec { + int_to_bytes32(*self as u64) + } +} + +impl TreeHash for [u8; 4] { + fn tree_hash_type() -> TreeHashType { + TreeHashType::List + } + + fn tree_hash_packed_encoding(&self) -> Vec { + panic!("bytesN should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + panic!("bytesN should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + merkle_root(&self[..]) + } +} + +impl TreeHash for H256 { + fn tree_hash_type() -> TreeHashType { + TreeHashType::Vector + } + + fn tree_hash_packed_encoding(&self) -> Vec { + self.as_bytes().to_vec() + } + + fn tree_hash_packing_factor() -> usize { + 1 + } + + fn tree_hash_root(&self) -> Vec { + merkle_root(&self.as_bytes().to_vec()) + } +} + +impl TreeHash for Vec +where + T: TreeHash, +{ + fn tree_hash_type() -> TreeHashType { + TreeHashType::List + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("List should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("List should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + let mut root_and_len = Vec::with_capacity(HASHSIZE * 2); + root_and_len.append(&mut vec_tree_hash_root(self)); + root_and_len.append(&mut int_to_bytes32(self.len() as u64)); + + hash(&root_and_len) + } +} + +pub fn vec_tree_hash_root(vec: &[T]) -> Vec +where + T: TreeHash, +{ + let leaves = match T::tree_hash_type() { + TreeHashType::Basic => { + let mut leaves = + Vec::with_capacity((HASHSIZE / T::tree_hash_packing_factor()) * vec.len()); + + for item in vec { + leaves.append(&mut item.tree_hash_packed_encoding()); + } + + leaves + } + TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => { + let mut leaves = Vec::with_capacity(vec.len() * HASHSIZE); + + for item in vec { + leaves.append(&mut item.tree_hash_root()) + } + + leaves + } + }; + + merkle_root(&leaves) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn bool() { + let mut true_bytes: Vec = vec![1]; + true_bytes.append(&mut vec![0; 31]); + + let false_bytes: Vec = vec![0; 32]; + + assert_eq!(true.tree_hash_root(), true_bytes); + assert_eq!(false.tree_hash_root(), false_bytes); + } + +} diff --git a/eth2/utils/tree_hash/tests/tests.rs b/eth2/utils/tree_hash/tests/tests.rs new file mode 100644 index 000000000..4d2c6f282 --- /dev/null +++ b/eth2/utils/tree_hash/tests/tests.rs @@ -0,0 +1,1080 @@ +use hashing::hash; +use int_to_bytes::{int_to_bytes32, int_to_bytes8}; +use tree_hash::cached_tree_hash::*; +use tree_hash::standard_tree_hash::*; +use tree_hash::*; + +#[derive(Clone, Debug)] +pub struct InternalCache { + pub a: u64, + pub b: u64, + pub cache: Option, +} + +impl TreeHash for InternalCache { + fn tree_hash_type() -> TreeHashType { + TreeHashType::Container + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("Struct should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("Struct should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + let mut leaves = Vec::with_capacity(4 * HASHSIZE); + + leaves.append(&mut self.a.tree_hash_root()); + leaves.append(&mut self.b.tree_hash_root()); + + efficient_merkleize(&leaves)[0..32].to_vec() + } +} + +impl CachedTreeHash for InternalCache { + fn update_internal_tree_hash_cache(mut self, mut old: Self) -> Result<(Self, Self), Error> { + let mut local_cache = old.cache; + old.cache = None; + + if let Some(ref mut local_cache) = local_cache { + self.update_tree_hash_cache(&old, local_cache, 0)?; + } else { + local_cache = Some(self.new_tree_hash_cache()?) + } + + self.cache = local_cache; + + Ok((old, self)) + } + + fn cached_tree_hash_root(&self) -> Option> { + match &self.cache { + None => None, + Some(c) => Some(c.root()?.to_vec()), + } + } + + fn clone_without_tree_hash_cache(&self) -> Self { + Self { + a: self.a, + b: self.b, + cache: None, + } + } +} + +#[test] +fn works_when_embedded() { + let old = InternalCache { + a: 99, + b: 99, + cache: None, + }; + + let mut new = old.clone_without_tree_hash_cache(); + new.a = 1; + new.b = 2; + + let (_old, new) = new.update_internal_tree_hash_cache(old).unwrap(); + + let root = new.cached_tree_hash_root().unwrap(); + + let leaves = vec![int_to_bytes32(1), int_to_bytes32(2)]; + let merkle = merkleize(join(leaves)); + + assert_eq!(&merkle[0..32], &root[..]); +} + +impl CachedTreeHashSubTree for InternalCache { + fn new_tree_hash_cache(&self) -> Result { + let tree = TreeHashCache::from_leaves_and_subtrees( + self, + vec![self.a.new_tree_hash_cache()?, self.b.new_tree_hash_cache()?], + )?; + + Ok(tree) + } + + fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result { + let mut lengths = vec![]; + + lengths.push(BTreeOverlay::new(&self.a, 0)?.total_nodes()); + lengths.push(BTreeOverlay::new(&self.b, 0)?.total_nodes()); + + BTreeOverlay::from_lengths(chunk_offset, lengths) + } + + fn update_tree_hash_cache( + &self, + other: &Self, + cache: &mut TreeHashCache, + chunk: usize, + ) -> Result { + let offset_handler = BTreeOverlay::new(self, chunk)?; + + // Skip past the internal nodes and update any changed leaf nodes. + { + let chunk = offset_handler.first_leaf_node()?; + let chunk = self.a.update_tree_hash_cache(&other.a, cache, chunk)?; + let _chunk = self.b.update_tree_hash_cache(&other.b, cache, chunk)?; + } + + for (&parent, children) in offset_handler.iter_internal_nodes().rev() { + if cache.either_modified(children)? { + cache.modify_chunk(parent, &cache.hash_children(children)?)?; + } + } + + Ok(offset_handler.next_node) + } +} + +fn num_nodes(num_leaves: usize) -> usize { + 2 * num_leaves - 1 +} + +#[derive(Clone, Debug)] +pub struct Inner { + pub a: u64, + pub b: u64, + pub c: u64, + pub d: u64, +} + +impl TreeHash for Inner { + fn tree_hash_type() -> TreeHashType { + TreeHashType::Container + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("Struct should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("Struct should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + let mut leaves = Vec::with_capacity(4 * HASHSIZE); + + leaves.append(&mut self.a.tree_hash_root()); + leaves.append(&mut self.b.tree_hash_root()); + leaves.append(&mut self.c.tree_hash_root()); + leaves.append(&mut self.d.tree_hash_root()); + + efficient_merkleize(&leaves)[0..32].to_vec() + } +} + +impl CachedTreeHashSubTree for Inner { + fn new_tree_hash_cache(&self) -> Result { + let tree = TreeHashCache::from_leaves_and_subtrees( + self, + vec![ + self.a.new_tree_hash_cache()?, + self.b.new_tree_hash_cache()?, + self.c.new_tree_hash_cache()?, + self.d.new_tree_hash_cache()?, + ], + )?; + + Ok(tree) + } + + fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result { + let mut lengths = vec![]; + + lengths.push(BTreeOverlay::new(&self.a, 0)?.total_nodes()); + lengths.push(BTreeOverlay::new(&self.b, 0)?.total_nodes()); + lengths.push(BTreeOverlay::new(&self.c, 0)?.total_nodes()); + lengths.push(BTreeOverlay::new(&self.d, 0)?.total_nodes()); + + BTreeOverlay::from_lengths(chunk_offset, lengths) + } + + fn update_tree_hash_cache( + &self, + other: &Self, + cache: &mut TreeHashCache, + chunk: usize, + ) -> Result { + let offset_handler = BTreeOverlay::new(self, chunk)?; + + // Skip past the internal nodes and update any changed leaf nodes. + { + let chunk = offset_handler.first_leaf_node()?; + let chunk = self.a.update_tree_hash_cache(&other.a, cache, chunk)?; + let chunk = self.b.update_tree_hash_cache(&other.b, cache, chunk)?; + let chunk = self.c.update_tree_hash_cache(&other.c, cache, chunk)?; + let _chunk = self.d.update_tree_hash_cache(&other.d, cache, chunk)?; + } + + for (&parent, children) in offset_handler.iter_internal_nodes().rev() { + if cache.either_modified(children)? { + cache.modify_chunk(parent, &cache.hash_children(children)?)?; + } + } + + Ok(offset_handler.next_node) + } +} + +#[derive(Clone, Debug)] +pub struct Outer { + pub a: u64, + pub b: Inner, + pub c: u64, +} + +impl TreeHash for Outer { + fn tree_hash_type() -> TreeHashType { + TreeHashType::Container + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("Struct should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("Struct should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + let mut leaves = Vec::with_capacity(4 * HASHSIZE); + + leaves.append(&mut self.a.tree_hash_root()); + leaves.append(&mut self.b.tree_hash_root()); + leaves.append(&mut self.c.tree_hash_root()); + + efficient_merkleize(&leaves)[0..32].to_vec() + } +} + +impl CachedTreeHashSubTree for Outer { + fn new_tree_hash_cache(&self) -> Result { + let tree = TreeHashCache::from_leaves_and_subtrees( + self, + vec![ + self.a.new_tree_hash_cache()?, + self.b.new_tree_hash_cache()?, + self.c.new_tree_hash_cache()?, + ], + )?; + + Ok(tree) + } + + fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result { + let mut lengths = vec![]; + + lengths.push(BTreeOverlay::new(&self.a, 0)?.total_nodes()); + lengths.push(BTreeOverlay::new(&self.b, 0)?.total_nodes()); + lengths.push(BTreeOverlay::new(&self.c, 0)?.total_nodes()); + + BTreeOverlay::from_lengths(chunk_offset, lengths) + } + + fn update_tree_hash_cache( + &self, + other: &Self, + cache: &mut TreeHashCache, + chunk: usize, + ) -> Result { + let offset_handler = BTreeOverlay::new(self, chunk)?; + + // Skip past the internal nodes and update any changed leaf nodes. + { + let chunk = offset_handler.first_leaf_node()?; + let chunk = self.a.update_tree_hash_cache(&other.a, cache, chunk)?; + let chunk = self.b.update_tree_hash_cache(&other.b, cache, chunk)?; + let _chunk = self.c.update_tree_hash_cache(&other.c, cache, chunk)?; + } + + for (&parent, children) in offset_handler.iter_internal_nodes().rev() { + if cache.either_modified(children)? { + cache.modify_chunk(parent, &cache.hash_children(children)?)?; + } + } + + Ok(offset_handler.next_node) + } +} + +fn join(many: Vec>) -> Vec { + let mut all = vec![]; + for one in many { + all.extend_from_slice(&mut one.clone()) + } + all +} + +#[test] +fn partial_modification_to_inner_struct() { + let original_inner = Inner { + a: 1, + b: 2, + c: 3, + d: 4, + }; + + let original_outer = Outer { + a: 0, + b: original_inner.clone(), + c: 5, + }; + + let modified_inner = Inner { + a: 42, + ..original_inner.clone() + }; + + // Modify outer + let modified_outer = Outer { + b: modified_inner.clone(), + ..original_outer.clone() + }; + + // Perform a differential hash + let mut cache_struct = TreeHashCache::new(&original_outer).unwrap(); + + modified_outer + .update_tree_hash_cache(&original_outer, &mut cache_struct, 0) + .unwrap(); + + let modified_cache: Vec = cache_struct.into(); + + // Generate reference data. + let mut data = vec![]; + data.append(&mut int_to_bytes32(0)); + let inner_bytes: Vec = TreeHashCache::new(&modified_inner).unwrap().into(); + data.append(&mut int_to_bytes32(5)); + + let leaves = vec![ + int_to_bytes32(0), + inner_bytes[0..32].to_vec(), + int_to_bytes32(5), + vec![0; 32], // padding + ]; + let mut merkle = merkleize(join(leaves)); + merkle.splice(4 * 32..5 * 32, inner_bytes); + + assert_eq!(merkle.len() / HASHSIZE, 13); + assert_eq!(modified_cache.len() / HASHSIZE, 13); + + assert_eq!(merkle, modified_cache); +} + +#[test] +fn partial_modification_to_outer() { + let inner = Inner { + a: 1, + b: 2, + c: 3, + d: 4, + }; + + let original_outer = Outer { + a: 0, + b: inner.clone(), + c: 5, + }; + + // Build the initial cache. + // let original_cache = original_outer.build_cache_bytes(); + + // Modify outer + let modified_outer = Outer { + c: 42, + ..original_outer.clone() + }; + + // Perform a differential hash + let mut cache_struct = TreeHashCache::new(&original_outer).unwrap(); + + modified_outer + .update_tree_hash_cache(&original_outer, &mut cache_struct, 0) + .unwrap(); + + let modified_cache: Vec = cache_struct.into(); + + // Generate reference data. + let mut data = vec![]; + data.append(&mut int_to_bytes32(0)); + let inner_bytes: Vec = TreeHashCache::new(&inner).unwrap().into(); + data.append(&mut int_to_bytes32(5)); + + let leaves = vec![ + int_to_bytes32(0), + inner_bytes[0..32].to_vec(), + int_to_bytes32(42), + vec![0; 32], // padding + ]; + let mut merkle = merkleize(join(leaves)); + merkle.splice(4 * 32..5 * 32, inner_bytes); + + assert_eq!(merkle.len() / HASHSIZE, 13); + assert_eq!(modified_cache.len() / HASHSIZE, 13); + + assert_eq!(merkle, modified_cache); +} + +#[test] +fn outer_builds() { + let inner = Inner { + a: 1, + b: 2, + c: 3, + d: 4, + }; + + let outer = Outer { + a: 0, + b: inner.clone(), + c: 5, + }; + + // Build the function output. + let cache: Vec = TreeHashCache::new(&outer).unwrap().into(); + + // Generate reference data. + let mut data = vec![]; + data.append(&mut int_to_bytes32(0)); + let inner_bytes: Vec = TreeHashCache::new(&inner).unwrap().into(); + data.append(&mut int_to_bytes32(5)); + + let leaves = vec![ + int_to_bytes32(0), + inner_bytes[0..32].to_vec(), + int_to_bytes32(5), + vec![0; 32], // padding + ]; + let mut merkle = merkleize(join(leaves)); + merkle.splice(4 * 32..5 * 32, inner_bytes); + + assert_eq!(merkle.len() / HASHSIZE, 13); + assert_eq!(cache.len() / HASHSIZE, 13); + + assert_eq!(merkle, cache); +} + +fn mix_in_length(root: &mut [u8], len: usize) { + let mut bytes = root.to_vec(); + bytes.append(&mut int_to_bytes32(len as u64)); + + root.copy_from_slice(&hash(&bytes)); +} + +/// Generic test that covers: +/// +/// 1. Produce a new cache from `original`. +/// 2. Do a differential hash between `original` and `modified`. +/// 3. Test that the cache generated matches the one we generate manually. +/// +/// In effect it ensures that we can do a differential hash between two `Vec`. +fn test_u64_vec_modifications(original: Vec, modified: Vec) { + // Generate initial cache. + let original_cache: Vec = TreeHashCache::new(&original).unwrap().into(); + + // Perform a differential hash + let mut cache_struct = TreeHashCache::from_bytes(original_cache.clone(), false).unwrap(); + modified + .update_tree_hash_cache(&original, &mut cache_struct, 0) + .unwrap(); + let modified_cache: Vec = cache_struct.into(); + + // Generate reference data. + let mut data = vec![]; + for i in &modified { + data.append(&mut int_to_bytes8(*i)); + } + let data = sanitise_bytes(data); + let mut expected = merkleize(data); + + mix_in_length(&mut expected[0..HASHSIZE], modified.len()); + + assert_eq!(expected, modified_cache); + assert_eq!(&expected[0..32], &modified.tree_hash_root()[..]); +} + +#[test] +fn partial_modification_u64_vec() { + let n: u64 = 2_u64.pow(5); + + let original_vec: Vec = (0..n).collect(); + + let mut modified_vec = original_vec.clone(); + modified_vec[n as usize - 1] = 42; + + test_u64_vec_modifications(original_vec, modified_vec); +} + +#[test] +fn shortened_u64_vec_len_within_pow_2_boundary() { + let n: u64 = 2_u64.pow(5) - 1; + + let original_vec: Vec = (0..n).collect(); + + let mut modified_vec = original_vec.clone(); + modified_vec.pop(); + + test_u64_vec_modifications(original_vec, modified_vec); +} + +#[test] +fn shortened_u64_vec_len_outside_pow_2_boundary() { + let original_vec: Vec = (0..2_u64.pow(6)).collect(); + + let modified_vec: Vec = (0..2_u64.pow(5)).collect(); + + test_u64_vec_modifications(original_vec, modified_vec); +} + +#[test] +fn extended_u64_vec_len_within_pow_2_boundary() { + let n: u64 = 2_u64.pow(5) - 2; + + let original_vec: Vec = (0..n).collect(); + + let mut modified_vec = original_vec.clone(); + modified_vec.push(42); + + test_u64_vec_modifications(original_vec, modified_vec); +} + +#[test] +fn extended_u64_vec_len_outside_pow_2_boundary() { + let original_vec: Vec = (0..2_u64.pow(5)).collect(); + + let modified_vec: Vec = (0..2_u64.pow(6)).collect(); + + test_u64_vec_modifications(original_vec, modified_vec); +} + +#[test] +fn large_vec_of_u64_builds() { + let n: u64 = 50; + + let my_vec: Vec = (0..n).collect(); + + // Generate function output. + let cache: Vec = TreeHashCache::new(&my_vec).unwrap().into(); + + // Generate reference data. + let mut data = vec![]; + for i in &my_vec { + data.append(&mut int_to_bytes8(*i)); + } + let data = sanitise_bytes(data); + let expected = merkleize(data); + + assert_eq!(expected, cache); +} + +/// Generic test that covers: +/// +/// 1. Produce a new cache from `original`. +/// 2. Do a differential hash between `original` and `modified`. +/// 3. Test that the cache generated matches the one we generate manually. +/// +/// The `reference` vec is used to build the tree hash cache manually. `Inner` is just 4x `u64`, so +/// you can represent 2x `Inner` with a `reference` vec of len 8. +/// +/// In effect it ensures that we can do a differential hash between two `Vec`. +fn test_inner_vec_modifications(original: Vec, modified: Vec, reference: Vec) { + let mut cache = TreeHashCache::new(&original).unwrap(); + + modified + .update_tree_hash_cache(&original, &mut cache, 0) + .unwrap(); + let modified_cache: Vec = cache.into(); + + // Build the reference vec. + + let mut leaves = vec![]; + let mut full_bytes = vec![]; + + for n in reference.chunks(4) { + let mut merkle = merkleize(join(vec![ + int_to_bytes32(n[0]), + int_to_bytes32(n[1]), + int_to_bytes32(n[2]), + int_to_bytes32(n[3]), + ])); + leaves.append(&mut merkle[0..HASHSIZE].to_vec()); + full_bytes.append(&mut merkle); + } + + let num_leaves = leaves.len() / HASHSIZE; + let mut expected = merkleize(leaves); + + let num_internal_nodes = num_leaves.next_power_of_two() - 1; + expected.splice(num_internal_nodes * HASHSIZE.., full_bytes); + + for _ in num_leaves..num_leaves.next_power_of_two() { + expected.append(&mut vec![0; HASHSIZE]); + } + + mix_in_length(&mut expected[0..HASHSIZE], modified.len()); + + // Compare the cached tree to the reference tree. + assert_trees_eq(&expected, &modified_cache); + assert_eq!(&expected[0..32], &modified.tree_hash_root()[..]); +} + +#[test] +fn partial_modification_of_vec_of_inner() { + let original = vec![ + Inner { + a: 0, + b: 1, + c: 2, + d: 3, + }, + Inner { + a: 4, + b: 5, + c: 6, + d: 7, + }, + Inner { + a: 8, + b: 9, + c: 10, + d: 11, + }, + ]; + + let mut modified = original.clone(); + modified[1].a = 42; + + let mut reference_vec: Vec = (0..12).collect(); + reference_vec[4] = 42; + + test_inner_vec_modifications(original, modified, reference_vec); +} + +#[test] +fn shortened_vec_of_inner_within_power_of_two_boundary() { + let original = vec![ + Inner { + a: 0, + b: 1, + c: 2, + d: 3, + }, + Inner { + a: 4, + b: 5, + c: 6, + d: 7, + }, + Inner { + a: 8, + b: 9, + c: 10, + d: 11, + }, + Inner { + a: 12, + b: 13, + c: 14, + d: 15, + }, + ]; + + let mut modified = original.clone(); + modified.pop(); // remove the last element from the list. + + let reference_vec: Vec = (0..12).collect(); + + test_inner_vec_modifications(original, modified, reference_vec); +} + +#[test] +fn shortened_vec_of_inner_outside_power_of_two_boundary() { + let original = vec![ + Inner { + a: 0, + b: 1, + c: 2, + d: 3, + }, + Inner { + a: 4, + b: 5, + c: 6, + d: 7, + }, + Inner { + a: 8, + b: 9, + c: 10, + d: 11, + }, + Inner { + a: 12, + b: 13, + c: 14, + d: 15, + }, + Inner { + a: 16, + b: 17, + c: 18, + d: 19, + }, + ]; + + let mut modified = original.clone(); + modified.pop(); // remove the last element from the list. + + let reference_vec: Vec = (0..16).collect(); + + test_inner_vec_modifications(original, modified, reference_vec); +} + +#[test] +fn lengthened_vec_of_inner_within_power_of_two_boundary() { + let original = vec![ + Inner { + a: 0, + b: 1, + c: 2, + d: 3, + }, + Inner { + a: 4, + b: 5, + c: 6, + d: 7, + }, + Inner { + a: 8, + b: 9, + c: 10, + d: 11, + }, + ]; + + let mut modified = original.clone(); + modified.push(Inner { + a: 12, + b: 13, + c: 14, + d: 15, + }); + + let reference_vec: Vec = (0..16).collect(); + + test_inner_vec_modifications(original, modified, reference_vec); +} + +#[test] +fn lengthened_vec_of_inner_outside_power_of_two_boundary() { + let original = vec![ + Inner { + a: 0, + b: 1, + c: 2, + d: 3, + }, + Inner { + a: 4, + b: 5, + c: 6, + d: 7, + }, + Inner { + a: 8, + b: 9, + c: 10, + d: 11, + }, + Inner { + a: 12, + b: 13, + c: 14, + d: 15, + }, + ]; + + let mut modified = original.clone(); + modified.push(Inner { + a: 16, + b: 17, + c: 18, + d: 19, + }); + + let reference_vec: Vec = (0..20).collect(); + + test_inner_vec_modifications(original, modified, reference_vec); +} + +#[test] +fn vec_of_inner_builds() { + let numbers: Vec = (0..12).collect(); + + let mut leaves = vec![]; + let mut full_bytes = vec![]; + + for n in numbers.chunks(4) { + let mut merkle = merkleize(join(vec![ + int_to_bytes32(n[0]), + int_to_bytes32(n[1]), + int_to_bytes32(n[2]), + int_to_bytes32(n[3]), + ])); + leaves.append(&mut merkle[0..HASHSIZE].to_vec()); + full_bytes.append(&mut merkle); + } + + let mut expected = merkleize(leaves); + expected.splice(3 * HASHSIZE.., full_bytes); + expected.append(&mut vec![0; HASHSIZE]); + + let my_vec = vec![ + Inner { + a: 0, + b: 1, + c: 2, + d: 3, + }, + Inner { + a: 4, + b: 5, + c: 6, + d: 7, + }, + Inner { + a: 8, + b: 9, + c: 10, + d: 11, + }, + ]; + + let cache: Vec = TreeHashCache::new(&my_vec).unwrap().into(); + + assert_trees_eq(&expected, &cache); +} + +/// Provides detailed assertions when comparing merkle trees. +fn assert_trees_eq(a: &[u8], b: &[u8]) { + assert_eq!(a.len(), b.len(), "Byte lens different"); + for i in (0..a.len() / HASHSIZE).rev() { + let range = i * HASHSIZE..(i + 1) * HASHSIZE; + assert_eq!( + a[range.clone()], + b[range], + "Chunk {}/{} different \n\n a: {:?} \n\n b: {:?}", + i, + a.len() / HASHSIZE, + a, + b, + ); + } +} + +#[test] +fn vec_of_u64_builds() { + let data = join(vec![ + int_to_bytes8(1), + int_to_bytes8(2), + int_to_bytes8(3), + int_to_bytes8(4), + int_to_bytes8(5), + vec![0; 32 - 8], // padding + ]); + + let expected = merkleize(data); + + let my_vec = vec![1, 2, 3, 4, 5]; + + // + // Note: the length is not mixed-in in this example. The user must ensure the length is + // mixed-in. + // + + let cache: Vec = TreeHashCache::new(&my_vec).unwrap().into(); + + assert_eq!(expected, cache); +} + +#[test] +fn vec_does_mix_in_len() { + let data = join(vec![ + int_to_bytes8(1), + int_to_bytes8(2), + int_to_bytes8(3), + int_to_bytes8(4), + int_to_bytes8(5), + vec![0; 32 - 8], // padding + ]); + + let tree = merkleize(data); + + let my_vec: Vec = vec![1, 2, 3, 4, 5]; + + let mut expected = vec![0; 32]; + expected.copy_from_slice(&tree[0..HASHSIZE]); + expected.append(&mut int_to_bytes32(my_vec.len() as u64)); + let expected = hash(&expected); + + assert_eq!(&expected[0..HASHSIZE], &my_vec.tree_hash_root()[..]); +} + +#[test] +fn merkleize_odd() { + let data = join(vec![ + int_to_bytes32(1), + int_to_bytes32(2), + int_to_bytes32(3), + int_to_bytes32(4), + int_to_bytes32(5), + ]); + + let merkle = merkleize(sanitise_bytes(data)); + + let expected_len = num_nodes(8) * BYTES_PER_CHUNK; + + assert_eq!(merkle.len(), expected_len); +} + +fn generic_test(index: usize) { + let inner = Inner { + a: 1, + b: 2, + c: 3, + d: 4, + }; + + let cache: Vec = TreeHashCache::new(&inner).unwrap().into(); + + let changed_inner = match index { + 0 => Inner { + a: 42, + ..inner.clone() + }, + 1 => Inner { + b: 42, + ..inner.clone() + }, + 2 => Inner { + c: 42, + ..inner.clone() + }, + 3 => Inner { + d: 42, + ..inner.clone() + }, + _ => panic!("bad index"), + }; + + let mut cache_struct = TreeHashCache::from_bytes(cache.clone(), false).unwrap(); + + changed_inner + .update_tree_hash_cache(&inner, &mut cache_struct, 0) + .unwrap(); + + // assert_eq!(*cache_struct.hash_count, 3); + + let new_tree_hash_cache: Vec = cache_struct.into(); + + let data1 = int_to_bytes32(1); + let data2 = int_to_bytes32(2); + let data3 = int_to_bytes32(3); + let data4 = int_to_bytes32(4); + + let mut data = vec![data1, data2, data3, data4]; + + data[index] = int_to_bytes32(42); + + let expected = merkleize(join(data)); + + assert_eq!(expected, new_tree_hash_cache); +} + +#[test] +fn cached_hash_on_inner() { + generic_test(0); + generic_test(1); + generic_test(2); + generic_test(3); +} + +#[test] +fn inner_builds() { + let data1 = int_to_bytes32(1); + let data2 = int_to_bytes32(2); + let data3 = int_to_bytes32(3); + let data4 = int_to_bytes32(4); + + let data = join(vec![data1, data2, data3, data4]); + let expected = merkleize(data); + + let inner = Inner { + a: 1, + b: 2, + c: 3, + d: 4, + }; + + let cache: Vec = TreeHashCache::new(&inner).unwrap().into(); + + assert_eq!(expected, cache); +} + +#[test] +fn merkleize_4_leaves() { + let data1 = hash(&int_to_bytes32(1)); + let data2 = hash(&int_to_bytes32(2)); + let data3 = hash(&int_to_bytes32(3)); + let data4 = hash(&int_to_bytes32(4)); + + let data = join(vec![ + data1.clone(), + data2.clone(), + data3.clone(), + data4.clone(), + ]); + + let cache = merkleize(data); + + let hash_12 = { + let mut joined = vec![]; + joined.append(&mut data1.clone()); + joined.append(&mut data2.clone()); + hash(&joined) + }; + let hash_34 = { + let mut joined = vec![]; + joined.append(&mut data3.clone()); + joined.append(&mut data4.clone()); + hash(&joined) + }; + let hash_hash12_hash_34 = { + let mut joined = vec![]; + joined.append(&mut hash_12.clone()); + joined.append(&mut hash_34.clone()); + hash(&joined) + }; + + for (i, chunk) in cache.chunks(HASHSIZE).enumerate().rev() { + let expected = match i { + 0 => hash_hash12_hash_34.clone(), + 1 => hash_12.clone(), + 2 => hash_34.clone(), + 3 => data1.clone(), + 4 => data2.clone(), + 5 => data3.clone(), + 6 => data4.clone(), + _ => vec![], + }; + + assert_eq!(chunk, &expected[..], "failed at {}", i); + } +} diff --git a/eth2/utils/tree_hash_derive/Cargo.toml b/eth2/utils/tree_hash_derive/Cargo.toml new file mode 100644 index 000000000..f227d7954 --- /dev/null +++ b/eth2/utils/tree_hash_derive/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "tree_hash_derive" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" +description = "Procedural derive macros for SSZ tree hashing." + +[lib] +proc-macro = true + +[dev-dependencies] +tree_hash = { path = "../tree_hash" } + +[dependencies] +syn = "0.15" +quote = "0.6" diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs new file mode 100644 index 000000000..343287313 --- /dev/null +++ b/eth2/utils/tree_hash_derive/src/lib.rs @@ -0,0 +1,209 @@ +#![recursion_limit = "256"] +extern crate proc_macro; + +use proc_macro::TokenStream; +use quote::{quote, ToTokens}; +use syn::{parse_macro_input, DeriveInput}; + +/// Returns a Vec of `syn::Ident` for each named field in the struct, whilst filtering out fields +/// that should not be hashed. +/// +/// # Panics +/// Any unnamed struct field (like in a tuple struct) will raise a panic at compile time. +fn get_hashable_named_field_idents<'a>(struct_data: &'a syn::DataStruct) -> Vec<&'a syn::Ident> { + struct_data + .fields + .iter() + .filter_map(|f| { + if should_skip_hashing(&f) { + None + } else { + Some(match &f.ident { + Some(ref ident) => ident, + _ => panic!("tree_hash_derive only supports named struct fields."), + }) + } + }) + .collect() +} + +/// Returns true if some field has an attribute declaring it should not be hashedd. +/// +/// The field attribute is: `#[tree_hash(skip_hashing)]` +fn should_skip_hashing(field: &syn::Field) -> bool { + field + .attrs + .iter() + .any(|attr| attr.into_token_stream().to_string() == "# [ tree_hash ( skip_hashing ) ]") +} + +/// Implements `tree_hash::CachedTreeHashSubTree` for some `struct`. +/// +/// Fields are hashed in the order they are defined. +#[proc_macro_derive(CachedTreeHashSubTree, attributes(tree_hash))] +pub fn subtree_derive(input: TokenStream) -> TokenStream { + let item = parse_macro_input!(input as DeriveInput); + + let name = &item.ident; + + let struct_data = match &item.data { + syn::Data::Struct(s) => s, + _ => panic!("tree_hash_derive only supports structs."), + }; + + let idents_a = get_hashable_named_field_idents(&struct_data); + let idents_b = idents_a.clone(); + let idents_c = idents_a.clone(); + let idents_d = idents_a.clone(); + + let output = quote! { + impl tree_hash::CachedTreeHashSubTree<#name> for #name { + fn new_tree_hash_cache(&self) -> Result { + let tree = tree_hash::TreeHashCache::from_leaves_and_subtrees( + self, + vec![ + #( + self.#idents_a.new_tree_hash_cache()?, + )* + ], + )?; + + Ok(tree) + } + + fn tree_hash_cache_overlay(&self, chunk_offset: usize) -> Result { + let mut lengths = vec![]; + + #( + lengths.push(tree_hash::BTreeOverlay::new(&self.#idents_b, 0)?.total_nodes()); + )* + + tree_hash::BTreeOverlay::from_lengths(chunk_offset, lengths) + } + + fn update_tree_hash_cache( + &self, + other: &Self, + cache: &mut tree_hash::TreeHashCache, + chunk: usize, + ) -> Result { + let offset_handler = tree_hash::BTreeOverlay::new(self, chunk)?; + + // Skip past the internal nodes and update any changed leaf nodes. + { + let chunk = offset_handler.first_leaf_node()?; + #( + let chunk = self.#idents_c.update_tree_hash_cache(&other.#idents_d, cache, chunk)?; + )* + } + + for (&parent, children) in offset_handler.iter_internal_nodes().rev() { + if cache.either_modified(children)? { + cache.modify_chunk(parent, &cache.hash_children(children)?)?; + } + } + + Ok(offset_handler.next_node) + } + } + }; + output.into() +} + +/// Implements `tree_hash::TreeHash` for some `struct`. +/// +/// Fields are hashed in the order they are defined. +#[proc_macro_derive(TreeHash, attributes(tree_hash))] +pub fn tree_hash_derive(input: TokenStream) -> TokenStream { + let item = parse_macro_input!(input as DeriveInput); + + let name = &item.ident; + + let struct_data = match &item.data { + syn::Data::Struct(s) => s, + _ => panic!("tree_hash_derive only supports structs."), + }; + + let idents = get_hashable_named_field_idents(&struct_data); + + let output = quote! { + impl tree_hash::TreeHash for #name { + fn tree_hash_type() -> tree_hash::TreeHashType { + tree_hash::TreeHashType::Container + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("Struct should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("Struct should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + let mut leaves = Vec::with_capacity(4 * tree_hash::HASHSIZE); + + #( + leaves.append(&mut self.#idents.tree_hash_root()); + )* + + tree_hash::merkle_root(&leaves) + } + } + }; + output.into() +} + +#[proc_macro_derive(SignedRoot, attributes(signed_root))] +pub fn tree_hash_signed_root_derive(input: TokenStream) -> TokenStream { + let item = parse_macro_input!(input as DeriveInput); + + let name = &item.ident; + + let struct_data = match &item.data { + syn::Data::Struct(s) => s, + _ => panic!("tree_hash_derive only supports structs."), + }; + + let idents = get_signed_root_named_field_idents(&struct_data); + let num_elems = idents.len(); + + let output = quote! { + impl tree_hash::SignedRoot for #name { + fn signed_root(&self) -> Vec { + let mut leaves = Vec::with_capacity(#num_elems * tree_hash::HASHSIZE); + + #( + leaves.append(&mut self.#idents.tree_hash_root()); + )* + + tree_hash::merkle_root(&leaves) + } + } + }; + output.into() +} + +fn get_signed_root_named_field_idents(struct_data: &syn::DataStruct) -> Vec<&syn::Ident> { + struct_data + .fields + .iter() + .filter_map(|f| { + if should_skip_signed_root(&f) { + None + } else { + Some(match &f.ident { + Some(ref ident) => ident, + _ => panic!("tree_hash_derive only supports named struct fields"), + }) + } + }) + .collect() +} + +fn should_skip_signed_root(field: &syn::Field) -> bool { + field + .attrs + .iter() + .any(|attr| attr.into_token_stream().to_string() == "# [ signed_root ( skip_hashing ) ]") +} diff --git a/eth2/utils/tree_hash_derive/tests/tests.rs b/eth2/utils/tree_hash_derive/tests/tests.rs new file mode 100644 index 000000000..a7c74b23e --- /dev/null +++ b/eth2/utils/tree_hash_derive/tests/tests.rs @@ -0,0 +1,182 @@ +use tree_hash::{CachedTreeHashSubTree, SignedRoot, TreeHash}; +use tree_hash_derive::{CachedTreeHashSubTree, SignedRoot, TreeHash}; + +#[derive(Clone, Debug, TreeHash, CachedTreeHashSubTree)] +pub struct Inner { + pub a: u64, + pub b: u64, + pub c: u64, + pub d: u64, +} + +fn test_standard_and_cached(original: &T, modified: &T) +where + T: CachedTreeHashSubTree, +{ + let mut cache = original.new_tree_hash_cache().unwrap(); + + let standard_root = original.tree_hash_root(); + let cached_root = cache.root().unwrap().to_vec(); + assert_eq!(standard_root, cached_root); + + // Test after a modification + modified + .update_tree_hash_cache(&original, &mut cache, 0) + .unwrap(); + let standard_root = modified.tree_hash_root(); + let cached_root = cache.root().unwrap().to_vec(); + assert_eq!(standard_root, cached_root); +} + +#[test] +fn inner_standard_vs_cached() { + let original = Inner { + a: 1, + b: 2, + c: 3, + d: 4, + }; + let modified = Inner { + b: 42, + ..original.clone() + }; + + test_standard_and_cached(&original, &modified); +} + +#[derive(Clone, Debug, TreeHash, CachedTreeHashSubTree)] +pub struct Uneven { + pub a: u64, + pub b: u64, + pub c: u64, + pub d: u64, + pub e: u64, +} + +#[test] +fn uneven_standard_vs_cached() { + let original = Uneven { + a: 1, + b: 2, + c: 3, + d: 4, + e: 5, + }; + let modified = Uneven { + e: 42, + ..original.clone() + }; + + test_standard_and_cached(&original, &modified); +} + +#[derive(Clone, Debug, TreeHash, SignedRoot)] +pub struct SignedInner { + pub a: u64, + pub b: u64, + pub c: u64, + pub d: u64, + #[signed_root(skip_hashing)] + pub e: u64, +} + +#[test] +fn signed_root() { + let unsigned = Inner { + a: 1, + b: 2, + c: 3, + d: 4, + }; + let signed = SignedInner { + a: 1, + b: 2, + c: 3, + d: 4, + e: 5, + }; + + assert_eq!(unsigned.tree_hash_root(), signed.signed_root()); +} + +#[derive(TreeHash, SignedRoot)] +struct CryptoKitties { + best_kitty: u64, + worst_kitty: u8, + kitties: Vec, +} + +impl CryptoKitties { + fn new() -> Self { + CryptoKitties { + best_kitty: 9999, + worst_kitty: 1, + kitties: vec![2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43], + } + } + + fn hash(&self) -> Vec { + let mut leaves = vec![]; + leaves.append(&mut self.best_kitty.tree_hash_root()); + leaves.append(&mut self.worst_kitty.tree_hash_root()); + leaves.append(&mut self.kitties.tree_hash_root()); + tree_hash::merkle_root(&leaves) + } +} + +#[test] +fn test_simple_tree_hash_derive() { + let kitties = CryptoKitties::new(); + assert_eq!(kitties.tree_hash_root(), kitties.hash()); +} + +#[test] +fn test_simple_signed_root_derive() { + let kitties = CryptoKitties::new(); + assert_eq!(kitties.signed_root(), kitties.hash()); +} + +#[derive(TreeHash, SignedRoot)] +struct Casper { + friendly: bool, + #[tree_hash(skip_hashing)] + friends: Vec, + #[signed_root(skip_hashing)] + dead: bool, +} + +impl Casper { + fn new() -> Self { + Casper { + friendly: true, + friends: vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + dead: true, + } + } + + fn expected_signed_hash(&self) -> Vec { + let mut list = Vec::new(); + list.append(&mut self.friendly.tree_hash_root()); + list.append(&mut self.friends.tree_hash_root()); + tree_hash::merkle_root(&list) + } + + fn expected_tree_hash(&self) -> Vec { + let mut list = Vec::new(); + list.append(&mut self.friendly.tree_hash_root()); + list.append(&mut self.dead.tree_hash_root()); + tree_hash::merkle_root(&list) + } +} + +#[test] +fn test_annotated_tree_hash_derive() { + let casper = Casper::new(); + assert_eq!(casper.tree_hash_root(), casper.expected_tree_hash()); +} + +#[test] +fn test_annotated_signed_root_derive() { + let casper = Casper::new(); + assert_eq!(casper.signed_root(), casper.expected_signed_hash()); +} diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index 80477c8ea..7f6b0cee9 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -17,6 +17,7 @@ block_proposer = { path = "../eth2/block_proposer" } attester = { path = "../eth2/attester" } bls = { path = "../eth2/utils/bls" } ssz = { path = "../eth2/utils/ssz" } +tree_hash = { path = "../eth2/utils/tree_hash" } clap = "2.32.0" dirs = "1.0.3" grpcio = { version = "0.4", default-features = false, features = ["protobuf-codec"] } diff --git a/validator_client/src/attestation_producer/mod.rs b/validator_client/src/attestation_producer/mod.rs index 0fbc7bcba..d2dbdf2e2 100644 --- a/validator_client/src/attestation_producer/mod.rs +++ b/validator_client/src/attestation_producer/mod.rs @@ -8,7 +8,7 @@ use super::block_producer::{BeaconNodeError, PublishOutcome, ValidatorEvent}; use crate::signer::Signer; use beacon_node_attestation::BeaconNodeAttestation; use slog::{error, info, warn}; -use ssz::TreeHash; +use tree_hash::TreeHash; use types::{ AggregateSignature, Attestation, AttestationData, AttestationDataAndCustodyBit, AttestationDuty, Bitfield, @@ -123,7 +123,7 @@ impl<'a, B: BeaconNodeAttestation, S: Signer> AttestationProducer<'a, B, S> { data: attestation.clone(), custody_bit: false, } - .hash_tree_root(); + .tree_hash_root(); let sig = self.signer.sign_message(&message, domain)?; diff --git a/validator_client/src/block_producer/mod.rs b/validator_client/src/block_producer/mod.rs index 8b4f5abda..2689b302d 100644 --- a/validator_client/src/block_producer/mod.rs +++ b/validator_client/src/block_producer/mod.rs @@ -6,8 +6,8 @@ pub use self::beacon_node_block::{BeaconNodeError, PublishOutcome}; pub use self::grpc::BeaconBlockGrpcClient; use crate::signer::Signer; use slog::{error, info, warn}; -use ssz::{SignedRoot, TreeHash}; use std::sync::Arc; +use tree_hash::{SignedRoot, TreeHash}; use types::{BeaconBlock, ChainSpec, Domain, Fork, Slot}; #[derive(Debug, PartialEq)] @@ -86,7 +86,7 @@ impl<'a, B: BeaconNodeBlock, S: Signer> BlockProducer<'a, B, S> { pub fn produce_block(&mut self) -> Result { let epoch = self.slot.epoch(self.spec.slots_per_epoch); - let message = epoch.hash_tree_root(); + let message = epoch.tree_hash_root(); let randao_reveal = match self.signer.sign_message( &message, self.spec.get_domain(epoch, Domain::Randao, &self.fork),