From 7434ff47badd69746b0ac9d6b2c3f5788fee29eb Mon Sep 17 00:00:00 2001 From: thojest Date: Thu, 14 Feb 2019 13:28:42 +0100 Subject: [PATCH 01/39] added is_double_vote and is_surround_vote (lighthouse-150) --- eth2/types/src/slashable_vote_data.rs | 91 +++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/eth2/types/src/slashable_vote_data.rs b/eth2/types/src/slashable_vote_data.rs index acffca26d..31433a0b1 100644 --- a/eth2/types/src/slashable_vote_data.rs +++ b/eth2/types/src/slashable_vote_data.rs @@ -1,4 +1,5 @@ use super::AttestationData; +use crate::spec::ChainSpec; use crate::test_utils::TestRandom; use bls::AggregateSignature; use rand::RngCore; @@ -13,6 +14,21 @@ pub struct SlashableVoteData { pub aggregate_signature: AggregateSignature, } +impl SlashableVoteData { + pub fn is_double_vote(&self, other: &SlashableVoteData, spec: &ChainSpec) -> bool { + self.data.slot.epoch(spec.epoch_length) == other.data.slot.epoch(spec.epoch_length) + } + + pub fn is_surround_vote(&self, other: &SlashableVoteData, spec: &ChainSpec) -> bool { + let source_epoch_1 = self.data.justified_epoch; + let source_epoch_2 = other.data.justified_epoch; + let target_epoch_1 = self.data.slot.epoch(spec.epoch_length); + let target_epoch_2 = other.data.slot.epoch(spec.epoch_length); + + (source_epoch_1 < source_epoch_2) && (target_epoch_2 < target_epoch_1) + } +} + impl Encodable for SlashableVoteData { fn ssz_append(&self, s: &mut SszStream) { s.append_vec(&self.custody_bit_0_indices); @@ -66,9 +82,71 @@ impl TestRandom for SlashableVoteData { #[cfg(test)] mod tests { use super::*; + use crate::slot_epoch_height::{Epoch, Slot}; + use crate::spec::ChainSpec; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; use ssz::ssz_encode; + #[test] + pub fn test_is_double_vote_true() { + let spec = ChainSpec::foundation(); + let slashable_vote_first = create_slashable_vote_data(1, 1, &spec); + let slashable_vote_second = create_slashable_vote_data(1, 1, &spec); + + assert_eq!( + slashable_vote_first.is_double_vote(&slashable_vote_second, &spec), + true + ) + } + + #[test] + pub fn test_is_double_vote_false() { + let spec = ChainSpec::foundation(); + let slashable_vote_first = create_slashable_vote_data(1, 1, &spec); + let slashable_vote_second = create_slashable_vote_data(2, 1, &spec); + + assert_eq!( + slashable_vote_first.is_double_vote(&slashable_vote_second, &spec), + false + ); + } + + #[test] + pub fn test_is_surround_vote_true() { + let spec = ChainSpec::foundation(); + let slashable_vote_first = create_slashable_vote_data(2, 1, &spec); + let slashable_vote_second = create_slashable_vote_data(1, 2, &spec); + + assert_eq!( + slashable_vote_first.is_surround_vote(&slashable_vote_second, &spec), + true + ); + } + + #[test] + pub fn test_is_surround_vote_false_source_epoch_fails() { + let spec = ChainSpec::foundation(); + let slashable_vote_first = create_slashable_vote_data(2, 2, &spec); + let slashable_vote_second = create_slashable_vote_data(1, 1, &spec); + + assert_eq!( + slashable_vote_first.is_surround_vote(&slashable_vote_second, &spec), + false + ); + } + + #[test] + pub fn test_is_surround_vote_false_target_epoch_fails() { + let spec = ChainSpec::foundation(); + let slashable_vote_first = create_slashable_vote_data(1, 1, &spec); + let slashable_vote_second = create_slashable_vote_data(2, 2, &spec); + + assert_eq!( + slashable_vote_first.is_surround_vote(&slashable_vote_second, &spec), + false + ); + } + #[test] pub fn test_ssz_round_trip() { let mut rng = XorShiftRng::from_seed([42; 16]); @@ -91,4 +169,17 @@ mod tests { // TODO: Add further tests // https://github.com/sigp/lighthouse/issues/170 } + + fn create_slashable_vote_data( + slot_factor: u64, + justified_epoch: u64, + spec: &ChainSpec, + ) -> SlashableVoteData { + let mut rng = XorShiftRng::from_seed([42; 16]); + let mut slashable_vote = SlashableVoteData::random_for_test(&mut rng); + + slashable_vote.data.slot = Slot::new(slot_factor * spec.epoch_length); + slashable_vote.data.justified_epoch = Epoch::new(justified_epoch); + slashable_vote + } } From 054c1f90476c99671ec1c1217d47b0eb5606d096 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 15 Feb 2019 16:55:52 +1100 Subject: [PATCH 02/39] Swap over to v0.2.0 shuffling algo --- eth2/types/Cargo.toml | 2 +- eth2/types/src/beacon_state.rs | 40 ++++++++++++++++++++++++---------- eth2/types/src/spec/mod.rs | 2 +- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/eth2/types/Cargo.toml b/eth2/types/Cargo.toml index fefd431a3..4c5be65b5 100644 --- a/eth2/types/Cargo.toml +++ b/eth2/types/Cargo.toml @@ -18,7 +18,7 @@ serde_derive = "1.0" serde_json = "1.0" slog = "^2.2.3" ssz = { path = "../utils/ssz" } -fisher_yates_shuffle = { path = "../utils/fisher_yates_shuffle" } +swap_or_not_shuffle = { path = "../utils/swap_or_not_shuffle" } [dev-dependencies] env_logger = "0.6.0" diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index d826344de..af815cbe2 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -5,15 +5,16 @@ use crate::{ PendingAttestation, PublicKey, Signature, Slot, Validator, }; use bls::verify_proof_of_possession; -use fisher_yates_shuffle::shuffle; use honey_badger_split::SplitExt; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use swap_or_not_shuffle::get_permutated_index; #[derive(Debug, PartialEq)] pub enum BeaconStateError { EpochOutOfBounds, + UnableToShuffle, InsufficientRandaoMixes, InsufficientValidators, InsufficientBlockRoots, @@ -249,23 +250,36 @@ impl BeaconState { /// committee is itself a list of validator indices. /// /// Spec v0.1 - pub fn get_shuffling(&self, seed: Hash256, epoch: Epoch, spec: &ChainSpec) -> Vec> { + pub fn get_shuffling( + &self, + seed: Hash256, + epoch: Epoch, + spec: &ChainSpec, + ) -> Option>> { let active_validator_indices = get_active_validator_indices(&self.validator_registry, epoch); let committees_per_epoch = self.get_epoch_committee_count(active_validator_indices.len(), spec); - // TODO: check that Hash256::from(u64) matches 'int_to_bytes32'. - let seed = seed ^ Hash256::from(epoch.as_u64()); - // TODO: fix `expect` assert. - let shuffled_active_validator_indices = - shuffle(&seed, active_validator_indices).expect("Max validator count exceed!"); + let mut shuffled_active_validator_indices = + Vec::with_capacity(active_validator_indices.len()); + for &i in &active_validator_indices { + let shuffled_i = get_permutated_index( + i, + active_validator_indices.len(), + &seed[..], + spec.shuffle_round_count, + )?; + shuffled_active_validator_indices[i] = active_validator_indices[shuffled_i] + } - shuffled_active_validator_indices - .honey_badger_split(committees_per_epoch as usize) - .map(|slice: &[usize]| slice.to_vec()) - .collect() + Some( + shuffled_active_validator_indices + .honey_badger_split(committees_per_epoch as usize) + .map(|slice: &[usize]| slice.to_vec()) + .collect(), + ) } /// Return the number of committees in the previous epoch. @@ -401,7 +415,9 @@ impl BeaconState { return Err(BeaconStateError::EpochOutOfBounds); }; - let shuffling = self.get_shuffling(seed, shuffling_epoch, spec); + let shuffling = self + .get_shuffling(seed, shuffling_epoch, spec) + .ok_or_else(|| BeaconStateError::UnableToShuffle)?; let offset = slot.as_u64() % spec.epoch_length; let committees_per_slot = committees_per_epoch / spec.epoch_length; let slot_start_shard = diff --git a/eth2/types/src/spec/mod.rs b/eth2/types/src/spec/mod.rs index 53c78a2c2..3632108ca 100644 --- a/eth2/types/src/spec/mod.rs +++ b/eth2/types/src/spec/mod.rs @@ -17,7 +17,7 @@ pub struct ChainSpec { pub beacon_chain_shard_number: u64, pub max_indices_per_slashable_vote: u64, pub max_withdrawals_per_epoch: u64, - pub shuffle_round_count: u64, + pub shuffle_round_count: u8, /* * Deposit contract From 7c920cfb96136105771bac06b518c8c9efc03f01 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 15 Feb 2019 19:23:22 +1100 Subject: [PATCH 03/39] Add incomplete progress on fixing test harness --- beacon_node/beacon_chain/src/beacon_chain.rs | 6 +++ .../beacon_chain/src/cached_beacon_state.rs | 49 +++++++++++++++++++ beacon_node/beacon_chain/src/lib.rs | 1 + .../test_harness/src/beacon_chain_harness.rs | 1 + .../beacon_chain/test_harness/tests/chain.rs | 17 +++---- eth2/types/src/beacon_state.rs | 45 ++++++++++++++++- eth2/types/src/spec/few_validators.rs | 21 ++++++++ eth2/types/src/spec/mod.rs | 1 + 8 files changed, 128 insertions(+), 13 deletions(-) create mode 100644 beacon_node/beacon_chain/src/cached_beacon_state.rs create mode 100644 eth2/types/src/spec/few_validators.rs diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index c16337fd4..6c7a8c751 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -253,6 +253,7 @@ where /// Information is read from the present `beacon_state` shuffling, so only information from the /// present and prior epoch is available. pub fn block_proposer(&self, slot: Slot) -> Result { + trace!("BeaconChain::block_proposer: slot: {}", slot); let index = self .state .read() @@ -274,6 +275,10 @@ where &self, validator_index: usize, ) -> Result, BeaconStateError> { + trace!( + "BeaconChain::validator_attestion_slot_and_shard: validator_index: {}", + validator_index + ); if let Some((slot, shard, _committee)) = self .state .read() @@ -287,6 +292,7 @@ where /// Produce an `AttestationData` that is valid for the present `slot` and given `shard`. pub fn produce_attestation_data(&self, shard: u64) -> Result { + trace!("BeaconChain::produce_attestation_data: shard: {}", shard); let justified_epoch = self.justified_epoch(); let justified_block_root = *self .state diff --git a/beacon_node/beacon_chain/src/cached_beacon_state.rs b/beacon_node/beacon_chain/src/cached_beacon_state.rs new file mode 100644 index 000000000..4717d1744 --- /dev/null +++ b/beacon_node/beacon_chain/src/cached_beacon_state.rs @@ -0,0 +1,49 @@ +use types::{beacon_state::BeaconStateError, BeaconState, ChainSpec, Epoch, Slot}; + +pub const CACHED_EPOCHS: usize = 3; // previous, current, next. + +pub type CrosslinkCommittees = Vec<(Vec, u64)>; + +pub struct CachedBeaconState<'a> { + state: BeaconState, + crosslinks: Vec>, + spec: &'a ChainSpec, +} + +impl<'a> CachedBeaconState<'a> { + pub fn from_beacon_state( + state: BeaconState, + spec: &'a ChainSpec, + ) -> Result { + let current_epoch = state.current_epoch(spec); + let previous_epoch = if current_epoch == spec.genesis_epoch { + current_epoch + } else { + current_epoch.saturating_sub(1_u64) + }; + let next_epoch = state.next_epoch(spec); + + let mut crosslinks: Vec> = Vec::with_capacity(3); + crosslinks.push(committees_for_all_slots(&state, previous_epoch, spec)?); + crosslinks.push(committees_for_all_slots(&state, current_epoch, spec)?); + crosslinks.push(committees_for_all_slots(&state, next_epoch, spec)?); + + Ok(Self { + state, + crosslinks, + spec, + }) + } +} + +fn committees_for_all_slots( + state: &BeaconState, + epoch: Epoch, + spec: &ChainSpec, +) -> Result, BeaconStateError> { + let mut crosslinks: Vec = Vec::with_capacity(spec.epoch_length as usize); + for slot in epoch.slot_iter(spec.epoch_length) { + crosslinks.push(state.get_crosslink_committees_at_slot(slot, false, spec)?) + } + Ok(crosslinks) +} diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 4dac0b672..bc9085fbe 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -1,5 +1,6 @@ mod attestation_aggregator; mod beacon_chain; +mod cached_beacon_state; mod checkpoint; pub use self::beacon_chain::{BeaconChain, Error}; diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index acba2e015..5ea681ca3 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -209,6 +209,7 @@ impl BeaconChainHarness { self.increment_beacon_chain_slot(); // Produce a new block. + debug!("Producing block..."); let block = self.produce_block(); debug!("Submitting block for processing..."); self.beacon_chain.process_block(block).unwrap(); diff --git a/beacon_node/beacon_chain/test_harness/tests/chain.rs b/beacon_node/beacon_chain/test_harness/tests/chain.rs index 8be6f2a26..13e276abb 100644 --- a/beacon_node/beacon_chain/test_harness/tests/chain.rs +++ b/beacon_node/beacon_chain/test_harness/tests/chain.rs @@ -1,19 +1,14 @@ use env_logger::{Builder, Env}; use log::debug; use test_harness::BeaconChainHarness; -use types::{ChainSpec, Slot}; +use types::ChainSpec; #[test] -#[ignore] fn it_can_build_on_genesis_block() { - let mut spec = ChainSpec::foundation(); - spec.genesis_slot = Slot::new(spec.epoch_length * 8); + Builder::from_env(Env::default().default_filter_or("trace")).init(); + let spec = ChainSpec::few_validators(); - /* - spec.shard_count = spec.shard_count / 8; - spec.target_committee_size = spec.target_committee_size / 8; - */ - let validator_count = 1000; + let validator_count = 8; let mut harness = BeaconChainHarness::new(spec, validator_count as usize); @@ -23,7 +18,7 @@ fn it_can_build_on_genesis_block() { #[test] #[ignore] fn it_can_produce_past_first_epoch_boundary() { - Builder::from_env(Env::default().default_filter_or("debug")).init(); + Builder::from_env(Env::default().default_filter_or("trace")).init(); let validator_count = 100; @@ -33,7 +28,7 @@ fn it_can_produce_past_first_epoch_boundary() { debug!("Harness built, tests starting.."); - let blocks = harness.spec.epoch_length * 3 + 1; + let blocks = harness.spec.epoch_length * 2 + 1; for i in 0..blocks { harness.advance_chain_with_block(); diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index af815cbe2..aa6690705 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -6,6 +6,7 @@ use crate::{ }; use bls::verify_proof_of_possession; use honey_badger_split::SplitExt; +use log::trace; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; @@ -259,11 +260,24 @@ impl BeaconState { let active_validator_indices = get_active_validator_indices(&self.validator_registry, epoch); + if active_validator_indices.is_empty() { + return None; + } + + trace!( + "BeaconState::get_shuffling: active_validator_indices.len() == {}", + active_validator_indices.len() + ); + let committees_per_epoch = self.get_epoch_committee_count(active_validator_indices.len(), spec); - let mut shuffled_active_validator_indices = - Vec::with_capacity(active_validator_indices.len()); + trace!( + "BeaconState::get_shuffling: active_validator_indices.len() == {}, committees_per_epoch: {}", + active_validator_indices.len(), committees_per_epoch + ); + + let mut shuffled_active_validator_indices = vec![0; active_validator_indices.len()]; for &i in &active_validator_indices { let shuffled_i = get_permutated_index( i, @@ -317,9 +331,17 @@ impl BeaconState { + 1; let latest_index_root = current_epoch + spec.entry_exit_delay; + trace!( + "BeaconState::get_active_index_root: epoch: {}, earliest: {}, latest: {}", + epoch, + earliest_index_root, + latest_index_root + ); + if (epoch >= earliest_index_root) & (epoch <= latest_index_root) { Some(self.latest_index_roots[epoch.as_usize() % spec.latest_index_roots_length]) } else { + trace!("BeaconState::get_active_index_root: epoch out of range."); None } } @@ -371,8 +393,14 @@ impl BeaconState { }; let next_epoch = self.next_epoch(spec); + trace!( + "BeaconState::get_crosslink_committees_at_slot: epoch: {}", + epoch + ); + let (committees_per_epoch, seed, shuffling_epoch, shuffling_start_shard) = if epoch == previous_epoch { + trace!("BeaconState::get_crosslink_committees_at_slot: epoch == previous_epoch"); ( self.get_previous_epoch_committee_count(spec), self.previous_epoch_seed, @@ -380,6 +408,7 @@ impl BeaconState { self.previous_epoch_start_shard, ) } else if epoch == current_epoch { + trace!("BeaconState::get_crosslink_committees_at_slot: epoch == current_epoch"); ( self.get_current_epoch_committee_count(spec), self.current_epoch_seed, @@ -387,6 +416,7 @@ impl BeaconState { self.current_epoch_start_shard, ) } else if epoch == next_epoch { + trace!("BeaconState::get_crosslink_committees_at_slot: epoch == next_epoch"); let current_committees_per_epoch = self.get_current_epoch_committee_count(spec); let epochs_since_last_registry_update = current_epoch - self.validator_registry_update_epoch; @@ -423,6 +453,12 @@ impl BeaconState { let slot_start_shard = (shuffling_start_shard + committees_per_slot * offset) % spec.shard_count; + trace!( + "BeaconState::get_crosslink_committees_at_slot: committees_per_slot: {}, slot_start_shard: {}", + committees_per_slot, + slot_start_shard + ); + let mut crosslinks_at_slot = vec![]; for i in 0..committees_per_slot { let tuple = ( @@ -474,6 +510,11 @@ impl BeaconState { spec: &ChainSpec, ) -> Result { let committees = self.get_crosslink_committees_at_slot(slot, false, spec)?; + trace!( + "get_beacon_proposer_index: slot: {}, committees_count: {}", + slot, + committees.len() + ); committees .first() .ok_or(BeaconStateError::InsufficientValidators) diff --git a/eth2/types/src/spec/few_validators.rs b/eth2/types/src/spec/few_validators.rs new file mode 100644 index 000000000..05fe4c6cd --- /dev/null +++ b/eth2/types/src/spec/few_validators.rs @@ -0,0 +1,21 @@ +use crate::{ChainSpec, Slot}; + +impl ChainSpec { + /// Returns a `ChainSpec` compatible with the specification suitable for 8 validators. + /// + /// Spec v0.2.0 + pub fn few_validators() -> Self { + let genesis_slot = Slot::new(2_u64.pow(19)); + let epoch_length = 8; + let genesis_epoch = genesis_slot.epoch(epoch_length); + + Self { + shard_count: 1, + target_committee_size: 1, + genesis_slot, + genesis_epoch, + epoch_length, + ..ChainSpec::foundation() + } + } +} diff --git a/eth2/types/src/spec/mod.rs b/eth2/types/src/spec/mod.rs index 3632108ca..3e983a148 100644 --- a/eth2/types/src/spec/mod.rs +++ b/eth2/types/src/spec/mod.rs @@ -1,3 +1,4 @@ +mod few_validators; mod foundation; use crate::{Address, Epoch, Hash256, Slot}; From 203f3b37f2b9672d4746a771f93ac6f2c3d04288 Mon Sep 17 00:00:00 2001 From: thojest Date: Fri, 15 Feb 2019 12:25:59 +0100 Subject: [PATCH 04/39] adapted import due to renaming of crate slot_epoch_height -> slot_epoch (lighthouse-150) --- eth2/types/src/slashable_vote_data.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth2/types/src/slashable_vote_data.rs b/eth2/types/src/slashable_vote_data.rs index 31433a0b1..170cd635e 100644 --- a/eth2/types/src/slashable_vote_data.rs +++ b/eth2/types/src/slashable_vote_data.rs @@ -82,7 +82,7 @@ impl TestRandom for SlashableVoteData { #[cfg(test)] mod tests { use super::*; - use crate::slot_epoch_height::{Epoch, Slot}; + use crate::slot_epoch::{Epoch, Slot}; use crate::spec::ChainSpec; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; use ssz::ssz_encode; From 3b92b69028021537ba98340c9552bf183957c822 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 16 Feb 2019 11:04:12 +1100 Subject: [PATCH 05/39] Apply CachedBeaconState to some functions - Attestation aggregation - Getting attestation duties --- .../src/attestation_aggregator.rs | 8 +- beacon_node/beacon_chain/src/beacon_chain.rs | 29 +++- .../beacon_chain/src/cached_beacon_state.rs | 138 +++++++++++++++--- .../beacon_chain/test_harness/tests/chain.rs | 9 +- 4 files changed, 151 insertions(+), 33 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_aggregator.rs b/beacon_node/beacon_chain/src/attestation_aggregator.rs index 6fbc11612..fa2ec87ab 100644 --- a/beacon_node/beacon_chain/src/attestation_aggregator.rs +++ b/beacon_node/beacon_chain/src/attestation_aggregator.rs @@ -1,3 +1,4 @@ +use crate::cached_beacon_state::CachedBeaconState; use state_processing::validate_attestation_without_signature; use std::collections::{HashMap, HashSet}; use types::{ @@ -76,12 +77,12 @@ impl AttestationAggregator { /// - The signature is verified against that of the validator at `validator_index`. pub fn process_free_attestation( &mut self, - state: &BeaconState, + cached_state: &CachedBeaconState, free_attestation: &FreeAttestation, spec: &ChainSpec, ) -> Result { let (slot, shard, committee_index) = some_or_invalid!( - state.attestation_slot_and_shard_for_validator( + cached_state.attestation_slot_and_shard_for_validator( free_attestation.validator_index as usize, spec, )?, @@ -104,7 +105,8 @@ impl AttestationAggregator { let signable_message = free_attestation.data.signable_message(PHASE_0_CUSTODY_BIT); let validator_record = some_or_invalid!( - state + cached_state + .state .validator_registry .get(free_attestation.validator_index as usize), Message::BadValidatorIndex diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 6c7a8c751..b2d041654 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1,4 +1,5 @@ use crate::attestation_aggregator::{AttestationAggregator, Outcome as AggregationOutcome}; +use crate::cached_beacon_state::CachedBeaconState; use crate::checkpoint::CheckPoint; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, @@ -69,6 +70,7 @@ pub struct BeaconChain { canonical_head: RwLock, finalized_head: RwLock, pub state: RwLock, + pub cached_state: RwLock, pub spec: ChainSpec, pub fork_choice: RwLock, } @@ -107,6 +109,11 @@ where let block_root = genesis_block.canonical_root(); block_store.put(&block_root, &ssz_encode(&genesis_block)[..])?; + let cached_state = RwLock::new(CachedBeaconState::from_beacon_state( + genesis_state.clone(), + spec.clone(), + )?); + let finalized_head = RwLock::new(CheckPoint::new( genesis_block.clone(), block_root, @@ -127,6 +134,7 @@ where slot_clock, attestation_aggregator, state: RwLock::new(genesis_state.clone()), + cached_state, finalized_head, canonical_head, spec, @@ -280,7 +288,7 @@ where validator_index ); if let Some((slot, shard, _committee)) = self - .state + .cached_state .read() .attestation_slot_and_shard_for_validator(validator_index, &self.spec)? { @@ -338,9 +346,7 @@ where let aggregation_outcome = self .attestation_aggregator .write() - .process_free_attestation(&self.state.read(), &free_attestation, &self.spec)?; - // TODO: Check this comment - //.map_err(|e| e.into())?; + .process_free_attestation(&self.cached_state.read(), &free_attestation, &self.spec)?; // return if the attestation is invalid if !aggregation_outcome.valid { @@ -495,6 +501,9 @@ where ); // Update the local state variable. *self.state.write() = state.clone(); + // Update the cached state variable. + *self.cached_state.write() = + CachedBeaconState::from_beacon_state(state.clone(), self.spec.clone())?; } Ok(BlockProcessingOutcome::ValidBlock(ValidBlock::Processed)) @@ -543,9 +552,15 @@ where }, }; - state - .per_block_processing_without_verifying_block_signature(&block, &self.spec) - .ok()?; + trace!("BeaconChain::produce_block: updating state for new block.",); + + let result = + state.per_block_processing_without_verifying_block_signature(&block, &self.spec); + trace!( + "BeaconNode::produce_block: state processing result: {:?}", + result + ); + result.ok()?; let state_root = state.canonical_root(); diff --git a/beacon_node/beacon_chain/src/cached_beacon_state.rs b/beacon_node/beacon_chain/src/cached_beacon_state.rs index 4717d1744..fec1d7c06 100644 --- a/beacon_node/beacon_chain/src/cached_beacon_state.rs +++ b/beacon_node/beacon_chain/src/cached_beacon_state.rs @@ -1,49 +1,149 @@ +use log::debug; +use std::collections::HashMap; use types::{beacon_state::BeaconStateError, BeaconState, ChainSpec, Epoch, Slot}; -pub const CACHED_EPOCHS: usize = 3; // previous, current, next. +pub const CACHE_PREVIOUS: bool = false; +pub const CACHE_CURRENT: bool = true; +pub const CACHE_NEXT: bool = false; pub type CrosslinkCommittees = Vec<(Vec, u64)>; +pub type Shard = u64; +pub type CommitteeIndex = u64; +pub type AttestationDuty = (Slot, Shard, CommitteeIndex); +pub type AttestationDutyMap = HashMap; -pub struct CachedBeaconState<'a> { - state: BeaconState, - crosslinks: Vec>, - spec: &'a ChainSpec, +// TODO: CachedBeaconState is presently duplicating `BeaconState` and `ChainSpec`. This is a +// massive memory waste, switch them to references. + +pub struct CachedBeaconState { + pub state: BeaconState, + committees: Vec>, + attestation_duties: Vec, + next_epoch: Epoch, + current_epoch: Epoch, + previous_epoch: Epoch, + spec: ChainSpec, } -impl<'a> CachedBeaconState<'a> { +impl CachedBeaconState { pub fn from_beacon_state( state: BeaconState, - spec: &'a ChainSpec, + spec: ChainSpec, ) -> Result { - let current_epoch = state.current_epoch(spec); + let current_epoch = state.current_epoch(&spec); let previous_epoch = if current_epoch == spec.genesis_epoch { current_epoch } else { current_epoch.saturating_sub(1_u64) }; - let next_epoch = state.next_epoch(spec); + let next_epoch = state.next_epoch(&spec); - let mut crosslinks: Vec> = Vec::with_capacity(3); - crosslinks.push(committees_for_all_slots(&state, previous_epoch, spec)?); - crosslinks.push(committees_for_all_slots(&state, current_epoch, spec)?); - crosslinks.push(committees_for_all_slots(&state, next_epoch, spec)?); + let mut committees: Vec> = Vec::with_capacity(3); + let mut attestation_duties: Vec = Vec::with_capacity(3); + + if CACHE_PREVIOUS { + debug!("CachedBeaconState::from_beacon_state: building previous epoch cache."); + let cache = build_epoch_cache(&state, previous_epoch, &spec)?; + committees.push(cache.committees); + attestation_duties.push(cache.attestation_duty_map); + } else { + committees.push(vec![]); + attestation_duties.push(HashMap::new()); + } + if CACHE_CURRENT { + debug!("CachedBeaconState::from_beacon_state: building current epoch cache."); + let cache = build_epoch_cache(&state, current_epoch, &spec)?; + committees.push(cache.committees); + attestation_duties.push(cache.attestation_duty_map); + } else { + committees.push(vec![]); + attestation_duties.push(HashMap::new()); + } + if CACHE_NEXT { + debug!("CachedBeaconState::from_beacon_state: building next epoch cache."); + let cache = build_epoch_cache(&state, next_epoch, &spec)?; + committees.push(cache.committees); + attestation_duties.push(cache.attestation_duty_map); + } else { + committees.push(vec![]); + attestation_duties.push(HashMap::new()); + } Ok(Self { state, - crosslinks, + committees, + attestation_duties, + next_epoch, + current_epoch, + previous_epoch, spec, }) } + + fn slot_to_cache_index(&self, slot: Slot) -> Option { + match slot.epoch(self.spec.epoch_length) { + epoch if (epoch == self.previous_epoch) & CACHE_PREVIOUS => Some(0), + epoch if (epoch == self.current_epoch) & CACHE_CURRENT => Some(1), + epoch if (epoch == self.next_epoch) & CACHE_NEXT => Some(2), + _ => None, + } + } + + /// Returns the `slot`, `shard` and `committee_index` for which a validator must produce an + /// attestation. + /// + /// Cached method. + /// + /// Spec v0.2.0 + pub fn attestation_slot_and_shard_for_validator( + &self, + validator_index: usize, + _spec: &ChainSpec, + ) -> Result, BeaconStateError> { + // Get the result for this epoch. + let cache_index = self + .slot_to_cache_index(self.state.slot) + .expect("Current epoch should always have a cache index."); + + let duties = self.attestation_duties[cache_index] + .get(&(validator_index as u64)) + .and_then(|tuple| Some(*tuple)); + + Ok(duties) + } } -fn committees_for_all_slots( +struct EpochCacheResult { + committees: Vec, + attestation_duty_map: AttestationDutyMap, +} + +fn build_epoch_cache( state: &BeaconState, epoch: Epoch, spec: &ChainSpec, -) -> Result, BeaconStateError> { - let mut crosslinks: Vec = Vec::with_capacity(spec.epoch_length as usize); +) -> Result { + let mut epoch_committees: Vec = + Vec::with_capacity(spec.epoch_length as usize); + let mut attestation_duty_map: AttestationDutyMap = HashMap::new(); + for slot in epoch.slot_iter(spec.epoch_length) { - crosslinks.push(state.get_crosslink_committees_at_slot(slot, false, spec)?) + let slot_committees = state.get_crosslink_committees_at_slot(slot, false, spec)?; + + for (committee, shard) in slot_committees { + for (committee_index, validator_index) in committee.iter().enumerate() { + attestation_duty_map.insert( + *validator_index as u64, + (slot, shard, committee_index as u64), + ); + } + } + + epoch_committees.push(state.get_crosslink_committees_at_slot(slot, false, spec)?) } - Ok(crosslinks) + + Ok(EpochCacheResult { + committees: epoch_committees, + attestation_duty_map, + }) } diff --git a/beacon_node/beacon_chain/test_harness/tests/chain.rs b/beacon_node/beacon_chain/test_harness/tests/chain.rs index 13e276abb..a57a0161f 100644 --- a/beacon_node/beacon_chain/test_harness/tests/chain.rs +++ b/beacon_node/beacon_chain/test_harness/tests/chain.rs @@ -6,8 +6,8 @@ use types::ChainSpec; #[test] fn it_can_build_on_genesis_block() { Builder::from_env(Env::default().default_filter_or("trace")).init(); - let spec = ChainSpec::few_validators(); + let spec = ChainSpec::few_validators(); let validator_count = 8; let mut harness = BeaconChainHarness::new(spec, validator_count as usize); @@ -18,13 +18,14 @@ fn it_can_build_on_genesis_block() { #[test] #[ignore] fn it_can_produce_past_first_epoch_boundary() { - Builder::from_env(Env::default().default_filter_or("trace")).init(); + Builder::from_env(Env::default().default_filter_or("debug")).init(); - let validator_count = 100; + let spec = ChainSpec::few_validators(); + let validator_count = 8; debug!("Starting harness build..."); - let mut harness = BeaconChainHarness::new(ChainSpec::foundation(), validator_count); + let mut harness = BeaconChainHarness::new(spec, validator_count); debug!("Harness built, tests starting.."); From 5e6acb8f39bf72e2754cb8a0f75f5d8e1799cb75 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 16 Feb 2019 11:19:47 +1100 Subject: [PATCH 06/39] Break `BeaconState` tests into separate file --- eth2/types/src/beacon_state.rs | 32 +-------- eth2/types/src/beacon_state/tests.rs | 97 ++++++++++++++++++++++++++++ eth2/types/src/beacon_state_tests.rs | 72 --------------------- eth2/types/src/lib.rs | 1 - 4 files changed, 99 insertions(+), 103 deletions(-) create mode 100644 eth2/types/src/beacon_state/tests.rs delete mode 100644 eth2/types/src/beacon_state_tests.rs diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index af815cbe2..22885adfe 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -11,6 +11,8 @@ use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; use swap_or_not_shuffle::get_permutated_index; +mod tests; + #[derive(Debug, PartialEq)] pub enum BeaconStateError { EpochOutOfBounds, @@ -1080,33 +1082,3 @@ impl TestRandom for BeaconState { } } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; - - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = BeaconState::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = BeaconState::random_for_test(&mut rng); - - let result = original.hash_tree_root(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } -} diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs new file mode 100644 index 000000000..e069e462e --- /dev/null +++ b/eth2/types/src/beacon_state/tests.rs @@ -0,0 +1,97 @@ +#![cfg(test)] + +use super::*; +use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; +use crate::{ + beacon_state::BeaconStateError, BeaconState, ChainSpec, Deposit, DepositData, DepositInput, + Eth1Data, Hash256, Keypair, +}; +use bls::create_proof_of_possession; +use ssz::ssz_encode; + +struct BeaconStateTestBuilder { + pub genesis_time: u64, + pub initial_validator_deposits: Vec, + pub latest_eth1_data: Eth1Data, + pub spec: ChainSpec, + pub keypairs: Vec, +} + +impl BeaconStateTestBuilder { + pub fn with_random_validators(validator_count: usize) -> Self { + let genesis_time = 10_000_000; + let keypairs: Vec = (0..validator_count) + .collect::>() + .iter() + .map(|_| Keypair::random()) + .collect(); + let initial_validator_deposits = keypairs + .iter() + .map(|keypair| Deposit { + branch: vec![], // branch verification is not specified. + index: 0, // index verification is not specified. + deposit_data: DepositData { + amount: 32_000_000_000, // 32 ETH (in Gwei) + timestamp: genesis_time - 1, + deposit_input: DepositInput { + pubkey: keypair.pk.clone(), + withdrawal_credentials: Hash256::zero(), // Withdrawal not possible. + proof_of_possession: create_proof_of_possession(&keypair), + }, + }, + }) + .collect(); + let latest_eth1_data = Eth1Data { + deposit_root: Hash256::zero(), + block_hash: Hash256::zero(), + }; + let spec = ChainSpec::foundation(); + + Self { + genesis_time, + initial_validator_deposits, + latest_eth1_data, + spec, + keypairs, + } + } + + pub fn build(&self) -> Result { + BeaconState::genesis( + self.genesis_time, + self.initial_validator_deposits.clone(), + self.latest_eth1_data.clone(), + &self.spec, + ) + } +} + +#[test] +pub fn can_produce_genesis_block() { + let builder = BeaconStateTestBuilder::with_random_validators(2); + + builder.build().unwrap(); +} + +#[test] +pub fn test_ssz_round_trip() { + let mut rng = XorShiftRng::from_seed([42; 16]); + let original = BeaconState::random_for_test(&mut rng); + + let bytes = ssz_encode(&original); + let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); + + assert_eq!(original, decoded); +} + +#[test] +pub fn test_hash_tree_root() { + let mut rng = XorShiftRng::from_seed([42; 16]); + let original = BeaconState::random_for_test(&mut rng); + + let result = original.hash_tree_root(); + + assert_eq!(result.len(), 32); + // TODO: Add further tests + // https://github.com/sigp/lighthouse/issues/170 +} diff --git a/eth2/types/src/beacon_state_tests.rs b/eth2/types/src/beacon_state_tests.rs deleted file mode 100644 index cae815f33..000000000 --- a/eth2/types/src/beacon_state_tests.rs +++ /dev/null @@ -1,72 +0,0 @@ -#[cfg(test)] -mod tests { - use crate::{ - beacon_state::BeaconStateError, BeaconState, ChainSpec, Deposit, DepositData, DepositInput, - Eth1Data, Hash256, Keypair, - }; - use bls::create_proof_of_possession; - - struct BeaconStateTestBuilder { - pub genesis_time: u64, - pub initial_validator_deposits: Vec, - pub latest_eth1_data: Eth1Data, - pub spec: ChainSpec, - pub keypairs: Vec, - } - - impl BeaconStateTestBuilder { - pub fn with_random_validators(validator_count: usize) -> Self { - let genesis_time = 10_000_000; - let keypairs: Vec = (0..validator_count) - .collect::>() - .iter() - .map(|_| Keypair::random()) - .collect(); - let initial_validator_deposits = keypairs - .iter() - .map(|keypair| Deposit { - branch: vec![], // branch verification is not specified. - index: 0, // index verification is not specified. - deposit_data: DepositData { - amount: 32_000_000_000, // 32 ETH (in Gwei) - timestamp: genesis_time - 1, - deposit_input: DepositInput { - pubkey: keypair.pk.clone(), - withdrawal_credentials: Hash256::zero(), // Withdrawal not possible. - proof_of_possession: create_proof_of_possession(&keypair), - }, - }, - }) - .collect(); - let latest_eth1_data = Eth1Data { - deposit_root: Hash256::zero(), - block_hash: Hash256::zero(), - }; - let spec = ChainSpec::foundation(); - - Self { - genesis_time, - initial_validator_deposits, - latest_eth1_data, - spec, - keypairs, - } - } - - pub fn build(&self) -> Result { - BeaconState::genesis( - self.genesis_time, - self.initial_validator_deposits.clone(), - self.latest_eth1_data.clone(), - &self.spec, - ) - } - } - - #[test] - pub fn can_produce_genesis_block() { - let builder = BeaconStateTestBuilder::with_random_validators(2); - - builder.build().unwrap(); - } -} diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index d33c48532..233d1cc3e 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -7,7 +7,6 @@ pub mod attester_slashing; pub mod beacon_block; pub mod beacon_block_body; pub mod beacon_state; -pub mod beacon_state_tests; pub mod casper_slashing; pub mod crosslink; pub mod deposit; From b0513b1ec1839826ad64704b5457d47adcc8d257 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 16 Feb 2019 15:08:33 +1100 Subject: [PATCH 07/39] Add and update logs --- .../beacon_chain/src/cached_beacon_state.rs | 9 ++-- .../test_harness/src/beacon_chain_harness.rs | 13 ++++- .../beacon_chain/test_harness/tests/chain.rs | 2 +- .../state_processing/src/block_processable.rs | 13 ++--- .../state_processing/src/epoch_processable.rs | 13 +++++ eth2/types/src/beacon_state.rs | 49 ++++++++----------- 6 files changed, 59 insertions(+), 40 deletions(-) diff --git a/beacon_node/beacon_chain/src/cached_beacon_state.rs b/beacon_node/beacon_chain/src/cached_beacon_state.rs index fec1d7c06..e14e9fe99 100644 --- a/beacon_node/beacon_chain/src/cached_beacon_state.rs +++ b/beacon_node/beacon_chain/src/cached_beacon_state.rs @@ -1,4 +1,4 @@ -use log::debug; +use log::{debug, trace}; use std::collections::HashMap; use types::{beacon_state::BeaconStateError, BeaconState, ChainSpec, Epoch, Slot}; @@ -42,7 +42,7 @@ impl CachedBeaconState { let mut attestation_duties: Vec = Vec::with_capacity(3); if CACHE_PREVIOUS { - debug!("CachedBeaconState::from_beacon_state: building previous epoch cache."); + debug!("from_beacon_state: building previous epoch cache."); let cache = build_epoch_cache(&state, previous_epoch, &spec)?; committees.push(cache.committees); attestation_duties.push(cache.attestation_duty_map); @@ -51,7 +51,7 @@ impl CachedBeaconState { attestation_duties.push(HashMap::new()); } if CACHE_CURRENT { - debug!("CachedBeaconState::from_beacon_state: building current epoch cache."); + debug!("from_beacon_state: building current epoch cache."); let cache = build_epoch_cache(&state, current_epoch, &spec)?; committees.push(cache.committees); attestation_duties.push(cache.attestation_duty_map); @@ -60,7 +60,7 @@ impl CachedBeaconState { attestation_duties.push(HashMap::new()); } if CACHE_NEXT { - debug!("CachedBeaconState::from_beacon_state: building next epoch cache."); + debug!("from_beacon_state: building next epoch cache."); let cache = build_epoch_cache(&state, next_epoch, &spec)?; committees.push(cache.committees); attestation_duties.push(cache.attestation_duty_map); @@ -81,6 +81,7 @@ impl CachedBeaconState { } fn slot_to_cache_index(&self, slot: Slot) -> Option { + trace!("slot_to_cache_index: cache lookup"); match slot.epoch(self.spec.epoch_length) { epoch if (epoch == self.previous_epoch) & CACHE_PREVIOUS => Some(0), epoch if (epoch == self.current_epoch) & CACHE_CURRENT => Some(1), diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index 5ea681ca3..9d61952f0 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -128,7 +128,18 @@ impl BeaconChainHarness { pub fn increment_beacon_chain_slot(&mut self) -> Slot { let slot = self.beacon_chain.present_slot() + 1; - debug!("Incrementing BeaconChain slot to {}.", slot); + let nth_slot = slot + - slot + .epoch(self.spec.epoch_length) + .start_slot(self.spec.epoch_length); + let nth_epoch = slot.epoch(self.spec.epoch_length) - self.spec.genesis_epoch; + debug!( + "Advancing BeaconChain to slot {}, epoch {} (epoch height: {}, slot {} in epoch.).", + slot, + slot.epoch(self.spec.epoch_length), + nth_epoch, + nth_slot + ); self.beacon_chain.slot_clock.set_slot(slot.as_u64()); self.beacon_chain.advance_state(slot).unwrap(); diff --git a/beacon_node/beacon_chain/test_harness/tests/chain.rs b/beacon_node/beacon_chain/test_harness/tests/chain.rs index a57a0161f..b3354784f 100644 --- a/beacon_node/beacon_chain/test_harness/tests/chain.rs +++ b/beacon_node/beacon_chain/test_harness/tests/chain.rs @@ -33,7 +33,7 @@ fn it_can_produce_past_first_epoch_boundary() { for i in 0..blocks { harness.advance_chain_with_block(); - debug!("Produced block {}/{}.", i, blocks); + debug!("Produced block {}/{}.", i + 1, blocks); } let dump = harness.chain_dump().expect("Chain dump failed."); diff --git a/eth2/state_processing/src/block_processable.rs b/eth2/state_processing/src/block_processable.rs index 904d2fac5..539711c69 100644 --- a/eth2/state_processing/src/block_processable.rs +++ b/eth2/state_processing/src/block_processable.rs @@ -1,7 +1,7 @@ use crate::SlotProcessingError; use hashing::hash; use int_to_bytes::int_to_bytes32; -use log::debug; +use log::{debug, trace}; use ssz::{ssz_encode, TreeHash}; use types::{ beacon_state::{AttestationParticipantsError, BeaconStateError}, @@ -219,6 +219,8 @@ fn per_block_processing_signature_optional( Error::MaxAttestationsExceeded ); + debug!("Verifying {} attestations.", block.body.attestations.len()); + for attestation in &block.body.attestations { validate_attestation(&state, attestation, spec)?; @@ -231,11 +233,6 @@ fn per_block_processing_signature_optional( state.latest_attestations.push(pending_attestation); } - debug!( - "{} attestations verified & processed.", - block.body.attestations.len() - ); - /* * Deposits */ @@ -312,6 +309,10 @@ fn validate_attestation_signature_optional( spec: &ChainSpec, verify_signature: bool, ) -> Result<(), AttestationValidationError> { + trace!( + "validate_attestation_signature_optional: attestation epoch: {}", + attestation.data.slot.epoch(spec.epoch_length) + ); ensure!( attestation.data.slot + spec.min_attestation_inclusion_delay <= state.slot, AttestationValidationError::IncludedTooEarly diff --git a/eth2/state_processing/src/epoch_processable.rs b/eth2/state_processing/src/epoch_processable.rs index 658449d6f..80318914c 100644 --- a/eth2/state_processing/src/epoch_processable.rs +++ b/eth2/state_processing/src/epoch_processable.rs @@ -315,6 +315,11 @@ impl EpochProcessable for BeaconState { // for slot in self.slot.saturating_sub(2 * spec.epoch_length)..self.slot { for slot in self.previous_epoch(spec).slot_iter(spec.epoch_length) { + trace!( + "Finding winning root for slot: {} (epoch: {})", + slot, + slot.epoch(spec.epoch_length) + ); let crosslink_committees_at_slot = self.get_crosslink_committees_at_slot(slot, false, spec)?; @@ -539,6 +544,12 @@ impl EpochProcessable for BeaconState { */ self.previous_calculation_epoch = self.current_calculation_epoch; self.previous_epoch_start_shard = self.current_epoch_start_shard; + + debug!( + "setting previous_epoch_seed to : {}", + self.current_epoch_seed + ); + self.previous_epoch_seed = self.current_epoch_seed; let should_update_validator_registy = if self.finalized_epoch @@ -553,6 +564,7 @@ impl EpochProcessable for BeaconState { }; if should_update_validator_registy { + trace!("updating validator registry."); self.update_validator_registry(spec); self.current_calculation_epoch = next_epoch; @@ -561,6 +573,7 @@ impl EpochProcessable for BeaconState { % spec.shard_count; self.current_epoch_seed = self.generate_seed(self.current_calculation_epoch, spec)? } else { + trace!("not updating validator registry."); let epochs_since_last_registry_update = current_epoch - self.validator_registry_update_epoch; if (epochs_since_last_registry_update > 1) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index aa6690705..6a4033d5c 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -265,7 +265,7 @@ impl BeaconState { } trace!( - "BeaconState::get_shuffling: active_validator_indices.len() == {}", + "get_shuffling: active_validator_indices.len() == {}", active_validator_indices.len() ); @@ -273,8 +273,9 @@ impl BeaconState { self.get_epoch_committee_count(active_validator_indices.len(), spec); trace!( - "BeaconState::get_shuffling: active_validator_indices.len() == {}, committees_per_epoch: {}", - active_validator_indices.len(), committees_per_epoch + "get_shuffling: active_validator_indices.len() == {}, committees_per_epoch: {}", + active_validator_indices.len(), + committees_per_epoch ); let mut shuffled_active_validator_indices = vec![0; active_validator_indices.len()]; @@ -332,7 +333,7 @@ impl BeaconState { let latest_index_root = current_epoch + spec.entry_exit_delay; trace!( - "BeaconState::get_active_index_root: epoch: {}, earliest: {}, latest: {}", + "get_active_index_root: epoch: {}, earliest: {}, latest: {}", epoch, earliest_index_root, latest_index_root @@ -341,7 +342,7 @@ impl BeaconState { if (epoch >= earliest_index_root) & (epoch <= latest_index_root) { Some(self.latest_index_roots[epoch.as_usize() % spec.latest_index_roots_length]) } else { - trace!("BeaconState::get_active_index_root: epoch out of range."); + trace!("get_active_index_root: epoch out of range."); None } } @@ -386,37 +387,28 @@ impl BeaconState { ) -> Result, u64)>, BeaconStateError> { let epoch = slot.epoch(spec.epoch_length); let current_epoch = self.current_epoch(spec); - let previous_epoch = if current_epoch == spec.genesis_epoch { - current_epoch - } else { - current_epoch.saturating_sub(1_u64) - }; + let previous_epoch = self.previous_epoch(spec); let next_epoch = self.next_epoch(spec); - trace!( - "BeaconState::get_crosslink_committees_at_slot: epoch: {}", - epoch - ); - let (committees_per_epoch, seed, shuffling_epoch, shuffling_start_shard) = - if epoch == previous_epoch { - trace!("BeaconState::get_crosslink_committees_at_slot: epoch == previous_epoch"); - ( - self.get_previous_epoch_committee_count(spec), - self.previous_epoch_seed, - self.previous_calculation_epoch, - self.previous_epoch_start_shard, - ) - } else if epoch == current_epoch { - trace!("BeaconState::get_crosslink_committees_at_slot: epoch == current_epoch"); + if epoch == current_epoch { + trace!("get_crosslink_committees_at_slot: current_epoch"); ( self.get_current_epoch_committee_count(spec), self.current_epoch_seed, self.current_calculation_epoch, self.current_epoch_start_shard, ) + } else if epoch == previous_epoch { + trace!("get_crosslink_committees_at_slot: previous_epoch"); + ( + self.get_previous_epoch_committee_count(spec), + self.previous_epoch_seed, + self.previous_calculation_epoch, + self.previous_epoch_start_shard, + ) } else if epoch == next_epoch { - trace!("BeaconState::get_crosslink_committees_at_slot: epoch == next_epoch"); + trace!("get_crosslink_committees_at_slot: next_epoch"); let current_committees_per_epoch = self.get_current_epoch_committee_count(spec); let epochs_since_last_registry_update = current_epoch - self.validator_registry_update_epoch; @@ -454,9 +446,10 @@ impl BeaconState { (shuffling_start_shard + committees_per_slot * offset) % spec.shard_count; trace!( - "BeaconState::get_crosslink_committees_at_slot: committees_per_slot: {}, slot_start_shard: {}", + "get_crosslink_committees_at_slot: committees_per_slot: {}, slot_start_shard: {}, seed: {}", committees_per_slot, - slot_start_shard + slot_start_shard, + seed ); let mut crosslinks_at_slot = vec![]; From c5158e297494dcae1c992fe592121698bd1c2176 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 16 Feb 2019 15:08:57 +1100 Subject: [PATCH 08/39] Fix bug with total_balance in epoch processing --- eth2/state_processing/src/epoch_processable.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eth2/state_processing/src/epoch_processable.rs b/eth2/state_processing/src/epoch_processable.rs index 80318914c..a930cd750 100644 --- a/eth2/state_processing/src/epoch_processable.rs +++ b/eth2/state_processing/src/epoch_processable.rs @@ -144,8 +144,10 @@ impl EpochProcessable for BeaconState { let previous_epoch_attester_indices = self.get_attestation_participants_union(&previous_epoch_attestations[..], spec)?; - let previous_total_balance = - self.get_total_balance(&previous_epoch_attester_indices[..], spec); + let previous_total_balance = self.get_total_balance( + &get_active_validator_indices(&self.validator_registry, previous_epoch), + spec, + ); /* * Validators targetting the previous justified slot From b79f0cdf68a26666bbd2bac6e578bf59627ff76c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 16 Feb 2019 15:09:14 +1100 Subject: [PATCH 09/39] Fix bug with reward quotient in epoch processing --- eth2/state_processing/src/epoch_processable.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eth2/state_processing/src/epoch_processable.rs b/eth2/state_processing/src/epoch_processable.rs index a930cd750..11b2b224d 100644 --- a/eth2/state_processing/src/epoch_processable.rs +++ b/eth2/state_processing/src/epoch_processable.rs @@ -359,7 +359,8 @@ impl EpochProcessable for BeaconState { /* * Rewards and Penalities */ - let base_reward_quotient = previous_total_balance.integer_sqrt(); + let base_reward_quotient = + previous_total_balance.integer_sqrt() / spec.base_reward_quotient; if base_reward_quotient == 0 { return Err(Error::BaseRewardQuotientIsZero); } From f83d02b394d7cc40e5ce5647c286d061bb2f9599 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 16 Feb 2019 15:09:43 +1100 Subject: [PATCH 10/39] Update previous epoch function --- eth2/types/src/beacon_state.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 6a4033d5c..2e1df45f2 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -203,7 +203,12 @@ impl BeaconState { /// /// Spec v0.2.0 pub fn previous_epoch(&self, spec: &ChainSpec) -> Epoch { - self.current_epoch(spec).saturating_sub(1_u64) + let current_epoch = self.current_epoch(&spec); + if current_epoch == spec.genesis_epoch { + current_epoch + } else { + current_epoch - 1 + } } /// The epoch following `self.current_epoch()`. From c4bedc03a8f704cb10e6c8da689e3f9b06362c1a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 17 Feb 2019 20:21:13 +1100 Subject: [PATCH 11/39] Fix file org. inconsistency in `types` --- .../src/{spec/foundation.rs => chain_spec.rs} | 111 +++++++++++++++++- eth2/types/src/lib.rs | 4 +- eth2/types/src/spec/few_validators.rs | 21 ---- eth2/types/src/spec/mod.rs | 93 --------------- 4 files changed, 112 insertions(+), 117 deletions(-) rename eth2/types/src/{spec/foundation.rs => chain_spec.rs} (54%) delete mode 100644 eth2/types/src/spec/few_validators.rs delete mode 100644 eth2/types/src/spec/mod.rs diff --git a/eth2/types/src/spec/foundation.rs b/eth2/types/src/chain_spec.rs similarity index 54% rename from eth2/types/src/spec/foundation.rs rename to eth2/types/src/chain_spec.rs index 79abe4061..b5d5689e3 100644 --- a/eth2/types/src/spec/foundation.rs +++ b/eth2/types/src/chain_spec.rs @@ -1,7 +1,96 @@ -use crate::{Address, ChainSpec, Epoch, Hash256, Signature, Slot}; +use crate::{Address, Epoch, Hash256, Slot}; +use bls::Signature; const GWEI: u64 = 1_000_000_000; +/// Holds all the "constants" for a BeaconChain. +/// +/// Spec v0.2.0 +#[derive(PartialEq, Debug, Clone)] +pub struct ChainSpec { + /* + * Misc + */ + pub shard_count: u64, + pub target_committee_size: u64, + pub max_balance_churn_quotient: u64, + pub beacon_chain_shard_number: u64, + pub max_indices_per_slashable_vote: u64, + pub max_withdrawals_per_epoch: u64, + pub shuffle_round_count: u8, + + /* + * Deposit contract + */ + pub deposit_contract_address: Address, + pub deposit_contract_tree_depth: u64, + + /* + * Gwei values + */ + pub min_deposit_amount: u64, + pub max_deposit_amount: u64, + pub fork_choice_balance_increment: u64, + pub ejection_balance: u64, + + /* + * Initial Values + */ + pub genesis_fork_version: u64, + pub genesis_slot: Slot, + pub genesis_epoch: Epoch, + pub genesis_start_shard: u64, + pub far_future_epoch: Epoch, + pub zero_hash: Hash256, + pub empty_signature: Signature, + pub bls_withdrawal_prefix_byte: u8, + + /* + * Time parameters + */ + pub slot_duration: u64, + pub min_attestation_inclusion_delay: u64, + pub epoch_length: u64, + pub seed_lookahead: Epoch, + pub entry_exit_delay: u64, + pub eth1_data_voting_period: u64, + pub min_validator_withdrawal_epochs: Epoch, + + /* + * State list lengths + */ + pub latest_block_roots_length: usize, + pub latest_randao_mixes_length: usize, + pub latest_index_roots_length: usize, + pub latest_penalized_exit_length: usize, + + /* + * Reward and penalty quotients + */ + pub base_reward_quotient: u64, + pub whistleblower_reward_quotient: u64, + pub includer_reward_quotient: u64, + pub inactivity_penalty_quotient: u64, + + /* + * Max operations per block + */ + pub max_proposer_slashings: u64, + pub max_attester_slashings: u64, + pub max_attestations: u64, + pub max_deposits: u64, + pub max_exits: u64, + + /* + * Signature domains + */ + pub domain_deposit: u64, + pub domain_attestation: u64, + pub domain_proposal: u64, + pub domain_exit: u64, + pub domain_randao: u64, +} + impl ChainSpec { /// Returns a `ChainSpec` compatible with the specification from Ethereum Foundation. /// @@ -100,6 +189,26 @@ impl ChainSpec { } } +impl ChainSpec { + /// Returns a `ChainSpec` compatible with the specification suitable for 8 validators. + /// + /// Spec v0.2.0 + pub fn few_validators() -> Self { + let genesis_slot = Slot::new(2_u64.pow(19)); + let epoch_length = 8; + let genesis_epoch = genesis_slot.epoch(epoch_length); + + Self { + shard_count: 1, + target_committee_size: 1, + genesis_slot, + genesis_epoch, + epoch_length, + ..ChainSpec::foundation() + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index 233d1cc3e..f2c128440 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -8,6 +8,7 @@ pub mod beacon_block; pub mod beacon_block_body; pub mod beacon_state; pub mod casper_slashing; +pub mod chain_spec; pub mod crosslink; pub mod deposit; pub mod deposit_data; @@ -28,7 +29,6 @@ pub mod slashable_vote_data; pub mod slot_epoch_macros; pub mod slot_epoch; pub mod slot_height; -pub mod spec; pub mod validator; pub mod validator_registry; pub mod validator_registry_delta_block; @@ -44,6 +44,7 @@ pub use crate::beacon_block::BeaconBlock; pub use crate::beacon_block_body::BeaconBlockBody; pub use crate::beacon_state::BeaconState; pub use crate::casper_slashing::CasperSlashing; +pub use crate::chain_spec::ChainSpec; pub use crate::crosslink::Crosslink; pub use crate::deposit::Deposit; pub use crate::deposit_data::DepositData; @@ -60,7 +61,6 @@ pub use crate::slashable_attestation::SlashableAttestation; pub use crate::slashable_vote_data::SlashableVoteData; pub use crate::slot_epoch::{Epoch, Slot}; pub use crate::slot_height::SlotHeight; -pub use crate::spec::ChainSpec; pub use crate::validator::{StatusFlags as ValidatorStatusFlags, Validator}; pub use crate::validator_registry_delta_block::ValidatorRegistryDeltaBlock; diff --git a/eth2/types/src/spec/few_validators.rs b/eth2/types/src/spec/few_validators.rs deleted file mode 100644 index 05fe4c6cd..000000000 --- a/eth2/types/src/spec/few_validators.rs +++ /dev/null @@ -1,21 +0,0 @@ -use crate::{ChainSpec, Slot}; - -impl ChainSpec { - /// Returns a `ChainSpec` compatible with the specification suitable for 8 validators. - /// - /// Spec v0.2.0 - pub fn few_validators() -> Self { - let genesis_slot = Slot::new(2_u64.pow(19)); - let epoch_length = 8; - let genesis_epoch = genesis_slot.epoch(epoch_length); - - Self { - shard_count: 1, - target_committee_size: 1, - genesis_slot, - genesis_epoch, - epoch_length, - ..ChainSpec::foundation() - } - } -} diff --git a/eth2/types/src/spec/mod.rs b/eth2/types/src/spec/mod.rs deleted file mode 100644 index 3e983a148..000000000 --- a/eth2/types/src/spec/mod.rs +++ /dev/null @@ -1,93 +0,0 @@ -mod few_validators; -mod foundation; - -use crate::{Address, Epoch, Hash256, Slot}; -use bls::Signature; - -/// Holds all the "constants" for a BeaconChain. -/// -/// Spec v0.2.0 -#[derive(PartialEq, Debug, Clone)] -pub struct ChainSpec { - /* - * Misc - */ - pub shard_count: u64, - pub target_committee_size: u64, - pub max_balance_churn_quotient: u64, - pub beacon_chain_shard_number: u64, - pub max_indices_per_slashable_vote: u64, - pub max_withdrawals_per_epoch: u64, - pub shuffle_round_count: u8, - - /* - * Deposit contract - */ - pub deposit_contract_address: Address, - pub deposit_contract_tree_depth: u64, - - /* - * Gwei values - */ - pub min_deposit_amount: u64, - pub max_deposit_amount: u64, - pub fork_choice_balance_increment: u64, - pub ejection_balance: u64, - - /* - * Initial Values - */ - pub genesis_fork_version: u64, - pub genesis_slot: Slot, - pub genesis_epoch: Epoch, - pub genesis_start_shard: u64, - pub far_future_epoch: Epoch, - pub zero_hash: Hash256, - pub empty_signature: Signature, - pub bls_withdrawal_prefix_byte: u8, - - /* - * Time parameters - */ - pub slot_duration: u64, - pub min_attestation_inclusion_delay: u64, - pub epoch_length: u64, - pub seed_lookahead: Epoch, - pub entry_exit_delay: u64, - pub eth1_data_voting_period: u64, - pub min_validator_withdrawal_epochs: Epoch, - - /* - * State list lengths - */ - pub latest_block_roots_length: usize, - pub latest_randao_mixes_length: usize, - pub latest_index_roots_length: usize, - pub latest_penalized_exit_length: usize, - - /* - * Reward and penalty quotients - */ - pub base_reward_quotient: u64, - pub whistleblower_reward_quotient: u64, - pub includer_reward_quotient: u64, - pub inactivity_penalty_quotient: u64, - - /* - * Max operations per block - */ - pub max_proposer_slashings: u64, - pub max_attester_slashings: u64, - pub max_attestations: u64, - pub max_deposits: u64, - pub max_exits: u64, - - /* - * Signature domains - */ - pub domain_deposit: u64, - pub domain_attestation: u64, - pub domain_proposal: u64, - pub domain_exit: u64, - pub domain_randao: u64, -} From dc0696754be06da921b2536af1771f9bdbc9d200 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 17 Feb 2019 20:23:31 +1100 Subject: [PATCH 12/39] Raise log level on test_harness tests --- beacon_node/beacon_chain/test_harness/tests/chain.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/tests/chain.rs b/beacon_node/beacon_chain/test_harness/tests/chain.rs index b3354784f..1a08ffcf1 100644 --- a/beacon_node/beacon_chain/test_harness/tests/chain.rs +++ b/beacon_node/beacon_chain/test_harness/tests/chain.rs @@ -5,7 +5,7 @@ use types::ChainSpec; #[test] fn it_can_build_on_genesis_block() { - Builder::from_env(Env::default().default_filter_or("trace")).init(); + Builder::from_env(Env::default().default_filter_or("info")).init(); let spec = ChainSpec::few_validators(); let validator_count = 8; @@ -18,7 +18,7 @@ fn it_can_build_on_genesis_block() { #[test] #[ignore] fn it_can_produce_past_first_epoch_boundary() { - Builder::from_env(Env::default().default_filter_or("debug")).init(); + Builder::from_env(Env::default().default_filter_or("info")).init(); let spec = ChainSpec::few_validators(); let validator_count = 8; From 6fa141181b997746cfa7b675b2519040eb27d5f7 Mon Sep 17 00:00:00 2001 From: mjkeating Date: Sun, 17 Feb 2019 09:30:18 -0800 Subject: [PATCH 13/39] Updated TreeHash to spec - added padding --- eth2/types/src/attestation.rs | 14 ++-- eth2/types/src/attestation_data.rs | 22 +++---- .../src/attestation_data_and_custody_bit.rs | 10 +-- eth2/types/src/attester_slashing.rs | 10 +-- eth2/types/src/beacon_block.rs | 20 +++--- eth2/types/src/beacon_block_body.rs | 16 ++--- eth2/types/src/beacon_state.rs | 56 ++++++++-------- eth2/types/src/beacon_state/tests.rs | 4 +- eth2/types/src/casper_slashing.rs | 10 +-- eth2/types/src/crosslink.rs | 10 +-- eth2/types/src/deposit.rs | 12 ++-- eth2/types/src/deposit_data.rs | 12 ++-- eth2/types/src/deposit_input.rs | 12 ++-- eth2/types/src/eth1_data.rs | 10 +-- eth2/types/src/eth1_data_vote.rs | 10 +-- eth2/types/src/exit.rs | 12 ++-- eth2/types/src/fork.rs | 12 ++-- eth2/types/src/pending_attestation.rs | 14 ++-- eth2/types/src/proposal_signed_data.rs | 12 ++-- eth2/types/src/proposer_slashing.rs | 16 ++--- eth2/types/src/shard_reassignment_record.rs | 12 ++-- eth2/types/src/slashable_attestation.rs | 14 ++-- eth2/types/src/slashable_vote_data.rs | 14 ++-- eth2/types/src/slot_epoch_macros.rs | 8 +-- eth2/types/src/validator.rs | 22 ++++--- .../src/validator_registry_delta_block.rs | 16 ++--- eth2/utils/bls/src/aggregate_signature.rs | 2 +- eth2/utils/bls/src/public_key.rs | 2 +- eth2/utils/bls/src/secret_key.rs | 2 +- eth2/utils/bls/src/signature.rs | 2 +- eth2/utils/boolean-bitfield/src/lib.rs | 4 +- eth2/utils/ssz/src/impl_tree_hash.rs | 24 +++---- eth2/utils/ssz/src/tree_hash.rs | 64 +++++++++++++++---- 33 files changed, 262 insertions(+), 218 deletions(-) diff --git a/eth2/types/src/attestation.rs b/eth2/types/src/attestation.rs index eb375d490..50e0949e1 100644 --- a/eth2/types/src/attestation.rs +++ b/eth2/types/src/attestation.rs @@ -60,12 +60,12 @@ impl Decodable for Attestation { } impl TreeHash for Attestation { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.aggregation_bitfield.hash_tree_root()); - result.append(&mut self.data.hash_tree_root()); - result.append(&mut self.custody_bitfield.hash_tree_root()); - result.append(&mut self.aggregate_signature.hash_tree_root()); + result.append(&mut self.aggregation_bitfield.hash_tree_root_internal()); + result.append(&mut self.data.hash_tree_root_internal()); + result.append(&mut self.custody_bitfield.hash_tree_root_internal()); + result.append(&mut self.aggregate_signature.hash_tree_root_internal()); hash(&result) } } @@ -99,11 +99,11 @@ mod tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = Attestation::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data.rs index 702bba416..6f98c557c 100644 --- a/eth2/types/src/attestation_data.rs +++ b/eth2/types/src/attestation_data.rs @@ -82,16 +82,16 @@ impl Decodable for AttestationData { } impl TreeHash for AttestationData { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.slot.hash_tree_root()); - result.append(&mut self.shard.hash_tree_root()); - result.append(&mut self.beacon_block_root.hash_tree_root()); - result.append(&mut self.epoch_boundary_root.hash_tree_root()); - result.append(&mut self.shard_block_root.hash_tree_root()); - result.append(&mut self.latest_crosslink.hash_tree_root()); - result.append(&mut self.justified_epoch.hash_tree_root()); - result.append(&mut self.justified_block_root.hash_tree_root()); + result.append(&mut self.slot.hash_tree_root_internal()); + result.append(&mut self.shard.hash_tree_root_internal()); + result.append(&mut self.beacon_block_root.hash_tree_root_internal()); + result.append(&mut self.epoch_boundary_root.hash_tree_root_internal()); + result.append(&mut self.shard_block_root.hash_tree_root_internal()); + result.append(&mut self.latest_crosslink.hash_tree_root_internal()); + result.append(&mut self.justified_epoch.hash_tree_root_internal()); + result.append(&mut self.justified_block_root.hash_tree_root_internal()); hash(&result) } } @@ -129,11 +129,11 @@ mod tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = AttestationData::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/attestation_data_and_custody_bit.rs b/eth2/types/src/attestation_data_and_custody_bit.rs index 4e93dd893..48add3d21 100644 --- a/eth2/types/src/attestation_data_and_custody_bit.rs +++ b/eth2/types/src/attestation_data_and_custody_bit.rs @@ -29,11 +29,11 @@ impl Decodable for AttestationDataAndCustodyBit { } impl TreeHash for AttestationDataAndCustodyBit { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.data.hash_tree_root()); + result.append(&mut self.data.hash_tree_root_internal()); // TODO: add bool ssz - // result.append(custody_bit.hash_tree_root()); + // result.append(custody_bit.hash_tree_root_internal()); ssz::hash(&result) } } @@ -68,11 +68,11 @@ mod test { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = AttestationDataAndCustodyBit::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/attester_slashing.rs b/eth2/types/src/attester_slashing.rs index 0b27d2030..80fa673a7 100644 --- a/eth2/types/src/attester_slashing.rs +++ b/eth2/types/src/attester_slashing.rs @@ -32,10 +32,10 @@ impl Decodable for AttesterSlashing { } impl TreeHash for AttesterSlashing { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.slashable_attestation_1.hash_tree_root()); - result.append(&mut self.slashable_attestation_2.hash_tree_root()); + result.append(&mut self.slashable_attestation_1.hash_tree_root_internal()); + result.append(&mut self.slashable_attestation_2.hash_tree_root_internal()); hash(&result) } } @@ -67,11 +67,11 @@ mod tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = AttesterSlashing::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/beacon_block.rs b/eth2/types/src/beacon_block.rs index f6977595a..f0661a1c9 100644 --- a/eth2/types/src/beacon_block.rs +++ b/eth2/types/src/beacon_block.rs @@ -97,15 +97,15 @@ impl Decodable for BeaconBlock { } impl TreeHash for BeaconBlock { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.slot.hash_tree_root()); - result.append(&mut self.parent_root.hash_tree_root()); - result.append(&mut self.state_root.hash_tree_root()); - result.append(&mut self.randao_reveal.hash_tree_root()); - result.append(&mut self.eth1_data.hash_tree_root()); - result.append(&mut self.signature.hash_tree_root()); - result.append(&mut self.body.hash_tree_root()); + result.append(&mut self.slot.hash_tree_root_internal()); + result.append(&mut self.parent_root.hash_tree_root_internal()); + result.append(&mut self.state_root.hash_tree_root_internal()); + result.append(&mut self.randao_reveal.hash_tree_root_internal()); + result.append(&mut self.eth1_data.hash_tree_root_internal()); + result.append(&mut self.signature.hash_tree_root_internal()); + result.append(&mut self.body.hash_tree_root_internal()); hash(&result) } } @@ -142,11 +142,11 @@ mod tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = BeaconBlock::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/beacon_block_body.rs b/eth2/types/src/beacon_block_body.rs index d3a61f7ba..9b25c919a 100644 --- a/eth2/types/src/beacon_block_body.rs +++ b/eth2/types/src/beacon_block_body.rs @@ -45,13 +45,13 @@ impl Decodable for BeaconBlockBody { } impl TreeHash for BeaconBlockBody { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.proposer_slashings.hash_tree_root()); - result.append(&mut self.attester_slashings.hash_tree_root()); - result.append(&mut self.attestations.hash_tree_root()); - result.append(&mut self.deposits.hash_tree_root()); - result.append(&mut self.exits.hash_tree_root()); + result.append(&mut self.proposer_slashings.hash_tree_root_internal()); + result.append(&mut self.attester_slashings.hash_tree_root_internal()); + result.append(&mut self.attestations.hash_tree_root_internal()); + result.append(&mut self.deposits.hash_tree_root_internal()); + result.append(&mut self.exits.hash_tree_root_internal()); hash(&result) } } @@ -86,11 +86,11 @@ mod tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = BeaconBlockBody::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 22885adfe..24023d298 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -1020,33 +1020,37 @@ impl Decodable for BeaconState { } impl TreeHash for BeaconState { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.slot.hash_tree_root()); - result.append(&mut self.genesis_time.hash_tree_root()); - result.append(&mut self.fork.hash_tree_root()); - result.append(&mut self.validator_registry.hash_tree_root()); - result.append(&mut self.validator_balances.hash_tree_root()); - result.append(&mut self.validator_registry_update_epoch.hash_tree_root()); - result.append(&mut self.latest_randao_mixes.hash_tree_root()); - result.append(&mut self.previous_epoch_start_shard.hash_tree_root()); - result.append(&mut self.current_epoch_start_shard.hash_tree_root()); - result.append(&mut self.previous_calculation_epoch.hash_tree_root()); - result.append(&mut self.current_calculation_epoch.hash_tree_root()); - result.append(&mut self.previous_epoch_seed.hash_tree_root()); - result.append(&mut self.current_epoch_seed.hash_tree_root()); - result.append(&mut self.previous_justified_epoch.hash_tree_root()); - result.append(&mut self.justified_epoch.hash_tree_root()); - result.append(&mut self.justification_bitfield.hash_tree_root()); - result.append(&mut self.finalized_epoch.hash_tree_root()); - result.append(&mut self.latest_crosslinks.hash_tree_root()); - result.append(&mut self.latest_block_roots.hash_tree_root()); - result.append(&mut self.latest_index_roots.hash_tree_root()); - result.append(&mut self.latest_penalized_balances.hash_tree_root()); - result.append(&mut self.latest_attestations.hash_tree_root()); - result.append(&mut self.batched_block_roots.hash_tree_root()); - result.append(&mut self.latest_eth1_data.hash_tree_root()); - result.append(&mut self.eth1_data_votes.hash_tree_root()); + result.append(&mut self.slot.hash_tree_root_internal()); + result.append(&mut self.genesis_time.hash_tree_root_internal()); + result.append(&mut self.fork.hash_tree_root_internal()); + result.append(&mut self.validator_registry.hash_tree_root_internal()); + result.append(&mut self.validator_balances.hash_tree_root_internal()); + result.append( + &mut self + .validator_registry_update_epoch + .hash_tree_root_internal(), + ); + result.append(&mut self.latest_randao_mixes.hash_tree_root_internal()); + result.append(&mut self.previous_epoch_start_shard.hash_tree_root_internal()); + result.append(&mut self.current_epoch_start_shard.hash_tree_root_internal()); + result.append(&mut self.previous_calculation_epoch.hash_tree_root_internal()); + result.append(&mut self.current_calculation_epoch.hash_tree_root_internal()); + result.append(&mut self.previous_epoch_seed.hash_tree_root_internal()); + result.append(&mut self.current_epoch_seed.hash_tree_root_internal()); + result.append(&mut self.previous_justified_epoch.hash_tree_root_internal()); + result.append(&mut self.justified_epoch.hash_tree_root_internal()); + result.append(&mut self.justification_bitfield.hash_tree_root_internal()); + result.append(&mut self.finalized_epoch.hash_tree_root_internal()); + result.append(&mut self.latest_crosslinks.hash_tree_root_internal()); + result.append(&mut self.latest_block_roots.hash_tree_root_internal()); + result.append(&mut self.latest_index_roots.hash_tree_root_internal()); + result.append(&mut self.latest_penalized_balances.hash_tree_root_internal()); + result.append(&mut self.latest_attestations.hash_tree_root_internal()); + result.append(&mut self.batched_block_roots.hash_tree_root_internal()); + result.append(&mut self.latest_eth1_data.hash_tree_root_internal()); + result.append(&mut self.eth1_data_votes.hash_tree_root_internal()); hash(&result) } } diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs index e069e462e..2b7c5b539 100644 --- a/eth2/types/src/beacon_state/tests.rs +++ b/eth2/types/src/beacon_state/tests.rs @@ -85,11 +85,11 @@ pub fn test_ssz_round_trip() { } #[test] -pub fn test_hash_tree_root() { +pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = BeaconState::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/casper_slashing.rs b/eth2/types/src/casper_slashing.rs index 0eab069b4..f9dc2f178 100644 --- a/eth2/types/src/casper_slashing.rs +++ b/eth2/types/src/casper_slashing.rs @@ -33,10 +33,10 @@ impl Decodable for CasperSlashing { } impl TreeHash for CasperSlashing { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.slashable_vote_data_1.hash_tree_root()); - result.append(&mut self.slashable_vote_data_2.hash_tree_root()); + result.append(&mut self.slashable_vote_data_1.hash_tree_root_internal()); + result.append(&mut self.slashable_vote_data_2.hash_tree_root_internal()); hash(&result) } } @@ -68,11 +68,11 @@ mod tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = CasperSlashing::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/crosslink.rs b/eth2/types/src/crosslink.rs index 3cb857ef4..90ee5dc9b 100644 --- a/eth2/types/src/crosslink.rs +++ b/eth2/types/src/crosslink.rs @@ -43,10 +43,10 @@ impl Decodable for Crosslink { } impl TreeHash for Crosslink { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.epoch.hash_tree_root()); - result.append(&mut self.shard_block_root.hash_tree_root()); + result.append(&mut self.epoch.hash_tree_root_internal()); + result.append(&mut self.shard_block_root.hash_tree_root_internal()); hash(&result) } } @@ -78,11 +78,11 @@ mod tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = Crosslink::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/deposit.rs b/eth2/types/src/deposit.rs index 62349cbc1..eebcdb5d0 100644 --- a/eth2/types/src/deposit.rs +++ b/eth2/types/src/deposit.rs @@ -37,11 +37,11 @@ impl Decodable for Deposit { } impl TreeHash for Deposit { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.branch.hash_tree_root()); - result.append(&mut self.index.hash_tree_root()); - result.append(&mut self.deposit_data.hash_tree_root()); + result.append(&mut self.branch.hash_tree_root_internal()); + result.append(&mut self.index.hash_tree_root_internal()); + result.append(&mut self.deposit_data.hash_tree_root_internal()); hash(&result) } } @@ -74,11 +74,11 @@ mod tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = Deposit::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/deposit_data.rs b/eth2/types/src/deposit_data.rs index 5c8c302f4..443c371e3 100644 --- a/eth2/types/src/deposit_data.rs +++ b/eth2/types/src/deposit_data.rs @@ -37,11 +37,11 @@ impl Decodable for DepositData { } impl TreeHash for DepositData { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.amount.hash_tree_root()); - result.append(&mut self.timestamp.hash_tree_root()); - result.append(&mut self.deposit_input.hash_tree_root()); + result.append(&mut self.amount.hash_tree_root_internal()); + result.append(&mut self.timestamp.hash_tree_root_internal()); + result.append(&mut self.deposit_input.hash_tree_root_internal()); hash(&result) } } @@ -74,11 +74,11 @@ mod tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = DepositData::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/deposit_input.rs b/eth2/types/src/deposit_input.rs index fc53baae9..440fcba1c 100644 --- a/eth2/types/src/deposit_input.rs +++ b/eth2/types/src/deposit_input.rs @@ -38,11 +38,11 @@ impl Decodable for DepositInput { } impl TreeHash for DepositInput { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.pubkey.hash_tree_root()); - result.append(&mut self.withdrawal_credentials.hash_tree_root()); - result.append(&mut self.proof_of_possession.hash_tree_root()); + result.append(&mut self.pubkey.hash_tree_root_internal()); + result.append(&mut self.withdrawal_credentials.hash_tree_root_internal()); + result.append(&mut self.proof_of_possession.hash_tree_root_internal()); hash(&result) } } @@ -75,11 +75,11 @@ mod tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = DepositInput::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/eth1_data.rs b/eth2/types/src/eth1_data.rs index 6e9bb7d26..d68e611dd 100644 --- a/eth2/types/src/eth1_data.rs +++ b/eth2/types/src/eth1_data.rs @@ -34,10 +34,10 @@ impl Decodable for Eth1Data { } impl TreeHash for Eth1Data { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.deposit_root.hash_tree_root()); - result.append(&mut self.block_hash.hash_tree_root()); + result.append(&mut self.deposit_root.hash_tree_root_internal()); + result.append(&mut self.block_hash.hash_tree_root_internal()); hash(&result) } } @@ -69,11 +69,11 @@ mod tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = Eth1Data::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/eth1_data_vote.rs b/eth2/types/src/eth1_data_vote.rs index 2bfee4d02..a09b0888d 100644 --- a/eth2/types/src/eth1_data_vote.rs +++ b/eth2/types/src/eth1_data_vote.rs @@ -34,10 +34,10 @@ impl Decodable for Eth1DataVote { } impl TreeHash for Eth1DataVote { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.eth1_data.hash_tree_root()); - result.append(&mut self.vote_count.hash_tree_root()); + result.append(&mut self.eth1_data.hash_tree_root_internal()); + result.append(&mut self.vote_count.hash_tree_root_internal()); hash(&result) } } @@ -69,11 +69,11 @@ mod tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = Eth1DataVote::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/exit.rs b/eth2/types/src/exit.rs index cd7746919..a1dcd6122 100644 --- a/eth2/types/src/exit.rs +++ b/eth2/types/src/exit.rs @@ -37,11 +37,11 @@ impl Decodable for Exit { } impl TreeHash for Exit { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.epoch.hash_tree_root()); - result.append(&mut self.validator_index.hash_tree_root()); - result.append(&mut self.signature.hash_tree_root()); + result.append(&mut self.epoch.hash_tree_root_internal()); + result.append(&mut self.validator_index.hash_tree_root_internal()); + result.append(&mut self.signature.hash_tree_root_internal()); hash(&result) } } @@ -74,11 +74,11 @@ mod tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = Exit::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs index 1c96a34ac..829d7d4a9 100644 --- a/eth2/types/src/fork.rs +++ b/eth2/types/src/fork.rs @@ -36,11 +36,11 @@ impl Decodable for Fork { } impl TreeHash for Fork { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.previous_version.hash_tree_root()); - result.append(&mut self.current_version.hash_tree_root()); - result.append(&mut self.epoch.hash_tree_root()); + result.append(&mut self.previous_version.hash_tree_root_internal()); + result.append(&mut self.current_version.hash_tree_root_internal()); + result.append(&mut self.epoch.hash_tree_root_internal()); hash(&result) } } @@ -73,11 +73,11 @@ mod tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = Fork::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/pending_attestation.rs b/eth2/types/src/pending_attestation.rs index 25ec109d7..b84a63297 100644 --- a/eth2/types/src/pending_attestation.rs +++ b/eth2/types/src/pending_attestation.rs @@ -41,12 +41,12 @@ impl Decodable for PendingAttestation { } impl TreeHash for PendingAttestation { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.aggregation_bitfield.hash_tree_root()); - result.append(&mut self.data.hash_tree_root()); - result.append(&mut self.custody_bitfield.hash_tree_root()); - result.append(&mut self.inclusion_slot.hash_tree_root()); + result.append(&mut self.aggregation_bitfield.hash_tree_root_internal()); + result.append(&mut self.data.hash_tree_root_internal()); + result.append(&mut self.custody_bitfield.hash_tree_root_internal()); + result.append(&mut self.inclusion_slot.hash_tree_root_internal()); hash(&result) } } @@ -80,11 +80,11 @@ mod tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = PendingAttestation::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/proposal_signed_data.rs b/eth2/types/src/proposal_signed_data.rs index c57eb1e2a..3069a7431 100644 --- a/eth2/types/src/proposal_signed_data.rs +++ b/eth2/types/src/proposal_signed_data.rs @@ -37,11 +37,11 @@ impl Decodable for ProposalSignedData { } impl TreeHash for ProposalSignedData { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.slot.hash_tree_root()); - result.append(&mut self.shard.hash_tree_root()); - result.append(&mut self.block_root.hash_tree_root()); + result.append(&mut self.slot.hash_tree_root_internal()); + result.append(&mut self.shard.hash_tree_root_internal()); + result.append(&mut self.block_root.hash_tree_root_internal()); hash(&result) } } @@ -74,11 +74,11 @@ mod tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = ProposalSignedData::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/proposer_slashing.rs b/eth2/types/src/proposer_slashing.rs index 417d23dbc..c76609335 100644 --- a/eth2/types/src/proposer_slashing.rs +++ b/eth2/types/src/proposer_slashing.rs @@ -46,13 +46,13 @@ impl Decodable for ProposerSlashing { } impl TreeHash for ProposerSlashing { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.proposer_index.hash_tree_root()); - result.append(&mut self.proposal_data_1.hash_tree_root()); - result.append(&mut self.proposal_signature_1.hash_tree_root()); - result.append(&mut self.proposal_data_2.hash_tree_root()); - result.append(&mut self.proposal_signature_2.hash_tree_root()); + result.append(&mut self.proposer_index.hash_tree_root_internal()); + result.append(&mut self.proposal_data_1.hash_tree_root_internal()); + result.append(&mut self.proposal_signature_1.hash_tree_root_internal()); + result.append(&mut self.proposal_data_2.hash_tree_root_internal()); + result.append(&mut self.proposal_signature_2.hash_tree_root_internal()); hash(&result) } } @@ -87,11 +87,11 @@ mod tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = ProposerSlashing::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/shard_reassignment_record.rs b/eth2/types/src/shard_reassignment_record.rs index 61f68ac05..374499a13 100644 --- a/eth2/types/src/shard_reassignment_record.rs +++ b/eth2/types/src/shard_reassignment_record.rs @@ -36,11 +36,11 @@ impl Decodable for ShardReassignmentRecord { } impl TreeHash for ShardReassignmentRecord { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.validator_index.hash_tree_root()); - result.append(&mut self.shard.hash_tree_root()); - result.append(&mut self.slot.hash_tree_root()); + result.append(&mut self.validator_index.hash_tree_root_internal()); + result.append(&mut self.shard.hash_tree_root_internal()); + result.append(&mut self.slot.hash_tree_root_internal()); hash(&result) } } @@ -73,11 +73,11 @@ mod tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = ShardReassignmentRecord::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/slashable_attestation.rs b/eth2/types/src/slashable_attestation.rs index 6d83ad147..862d93c60 100644 --- a/eth2/types/src/slashable_attestation.rs +++ b/eth2/types/src/slashable_attestation.rs @@ -40,12 +40,12 @@ impl Decodable for SlashableAttestation { } impl TreeHash for SlashableAttestation { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.validator_indices.hash_tree_root()); - result.append(&mut self.data.hash_tree_root()); - result.append(&mut self.custody_bitfield.hash_tree_root()); - result.append(&mut self.aggregate_signature.hash_tree_root()); + result.append(&mut self.validator_indices.hash_tree_root_internal()); + result.append(&mut self.data.hash_tree_root_internal()); + result.append(&mut self.custody_bitfield.hash_tree_root_internal()); + result.append(&mut self.aggregate_signature.hash_tree_root_internal()); hash(&result) } } @@ -79,11 +79,11 @@ mod tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = SlashableAttestation::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/slashable_vote_data.rs b/eth2/types/src/slashable_vote_data.rs index acffca26d..3eb0e0c6f 100644 --- a/eth2/types/src/slashable_vote_data.rs +++ b/eth2/types/src/slashable_vote_data.rs @@ -42,12 +42,12 @@ impl Decodable for SlashableVoteData { } impl TreeHash for SlashableVoteData { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.custody_bit_0_indices.hash_tree_root()); - result.append(&mut self.custody_bit_1_indices.hash_tree_root()); - result.append(&mut self.data.hash_tree_root()); - result.append(&mut self.aggregate_signature.hash_tree_root()); + result.append(&mut self.custody_bit_0_indices.hash_tree_root_internal()); + result.append(&mut self.custody_bit_1_indices.hash_tree_root_internal()); + result.append(&mut self.data.hash_tree_root_internal()); + result.append(&mut self.aggregate_signature.hash_tree_root_internal()); hash(&result) } } @@ -81,11 +81,11 @@ mod tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = SlashableVoteData::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/slot_epoch_macros.rs b/eth2/types/src/slot_epoch_macros.rs index 48bc219da..54a8f2ce9 100644 --- a/eth2/types/src/slot_epoch_macros.rs +++ b/eth2/types/src/slot_epoch_macros.rs @@ -224,9 +224,9 @@ macro_rules! impl_ssz { } impl TreeHash for $type { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.0.hash_tree_root()); + result.append(&mut self.0.hash_tree_root_internal()); hash(&result) } } @@ -560,11 +560,11 @@ macro_rules! ssz_tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = $type::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/validator.rs b/eth2/types/src/validator.rs index 047817a86..b832283a0 100644 --- a/eth2/types/src/validator.rs +++ b/eth2/types/src/validator.rs @@ -122,15 +122,17 @@ impl Decodable for Validator { } impl TreeHash for Validator { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.pubkey.hash_tree_root()); - result.append(&mut self.withdrawal_credentials.hash_tree_root()); - result.append(&mut self.activation_epoch.hash_tree_root()); - result.append(&mut self.exit_epoch.hash_tree_root()); - result.append(&mut self.withdrawal_epoch.hash_tree_root()); - result.append(&mut self.penalized_epoch.hash_tree_root()); - result.append(&mut u64::from(status_flag_to_byte(self.status_flags)).hash_tree_root()); + result.append(&mut self.pubkey.hash_tree_root_internal()); + result.append(&mut self.withdrawal_credentials.hash_tree_root_internal()); + result.append(&mut self.activation_epoch.hash_tree_root_internal()); + result.append(&mut self.exit_epoch.hash_tree_root_internal()); + result.append(&mut self.withdrawal_epoch.hash_tree_root_internal()); + result.append(&mut self.penalized_epoch.hash_tree_root_internal()); + result.append( + &mut u64::from(status_flag_to_byte(self.status_flags)).hash_tree_root_internal(), + ); hash(&result) } } @@ -190,11 +192,11 @@ mod tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = Validator::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/validator_registry_delta_block.rs b/eth2/types/src/validator_registry_delta_block.rs index 3142e0263..13d2e0059 100644 --- a/eth2/types/src/validator_registry_delta_block.rs +++ b/eth2/types/src/validator_registry_delta_block.rs @@ -59,13 +59,13 @@ impl Decodable for ValidatorRegistryDeltaBlock { } impl TreeHash for ValidatorRegistryDeltaBlock { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.latest_registry_delta_root.hash_tree_root()); - result.append(&mut self.validator_index.hash_tree_root()); - result.append(&mut self.pubkey.hash_tree_root()); - result.append(&mut self.slot.hash_tree_root()); - result.append(&mut self.flag.hash_tree_root()); + result.append(&mut self.latest_registry_delta_root.hash_tree_root_internal()); + result.append(&mut self.validator_index.hash_tree_root_internal()); + result.append(&mut self.pubkey.hash_tree_root_internal()); + result.append(&mut self.slot.hash_tree_root_internal()); + result.append(&mut self.flag.hash_tree_root_internal()); hash(&result) } } @@ -100,11 +100,11 @@ mod tests { } #[test] - pub fn test_hash_tree_root() { + pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); let original = ValidatorRegistryDeltaBlock::random_for_test(&mut rng); - let result = original.hash_tree_root(); + let result = original.hash_tree_root_internal(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/utils/bls/src/aggregate_signature.rs b/eth2/utils/bls/src/aggregate_signature.rs index 6fed183f0..b606a5ebd 100644 --- a/eth2/utils/bls/src/aggregate_signature.rs +++ b/eth2/utils/bls/src/aggregate_signature.rs @@ -57,7 +57,7 @@ impl Serialize for AggregateSignature { } impl TreeHash for AggregateSignature { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { hash(&self.0.as_bytes()) } } diff --git a/eth2/utils/bls/src/public_key.rs b/eth2/utils/bls/src/public_key.rs index 0c2ad81bb..c7fd526a0 100644 --- a/eth2/utils/bls/src/public_key.rs +++ b/eth2/utils/bls/src/public_key.rs @@ -66,7 +66,7 @@ impl Serialize for PublicKey { } impl TreeHash for PublicKey { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { hash(&self.0.as_bytes()) } } diff --git a/eth2/utils/bls/src/secret_key.rs b/eth2/utils/bls/src/secret_key.rs index 4ff9f8684..f2d54f4ac 100644 --- a/eth2/utils/bls/src/secret_key.rs +++ b/eth2/utils/bls/src/secret_key.rs @@ -41,7 +41,7 @@ impl Decodable for SecretKey { } impl TreeHash for SecretKey { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { self.0.as_bytes().clone() } } diff --git a/eth2/utils/bls/src/signature.rs b/eth2/utils/bls/src/signature.rs index 396e4eab7..511681957 100644 --- a/eth2/utils/bls/src/signature.rs +++ b/eth2/utils/bls/src/signature.rs @@ -61,7 +61,7 @@ impl Decodable for Signature { } impl TreeHash for Signature { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { hash(&self.0.as_bytes()) } } diff --git a/eth2/utils/boolean-bitfield/src/lib.rs b/eth2/utils/boolean-bitfield/src/lib.rs index a0fce1f0a..fb3a78e7a 100644 --- a/eth2/utils/boolean-bitfield/src/lib.rs +++ b/eth2/utils/boolean-bitfield/src/lib.rs @@ -187,8 +187,8 @@ impl Serialize for BooleanBitfield { } impl ssz::TreeHash for BooleanBitfield { - fn hash_tree_root(&self) -> Vec { - self.to_bytes().hash_tree_root() + fn hash_tree_root_internal(&self) -> Vec { + self.to_bytes().hash_tree_root_internal() } } diff --git a/eth2/utils/ssz/src/impl_tree_hash.rs b/eth2/utils/ssz/src/impl_tree_hash.rs index 578977eec..7c3dae596 100644 --- a/eth2/utils/ssz/src/impl_tree_hash.rs +++ b/eth2/utils/ssz/src/impl_tree_hash.rs @@ -3,49 +3,49 @@ use super::{merkle_hash, ssz_encode, TreeHash}; use hashing::hash; impl TreeHash for u8 { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { ssz_encode(self) } } impl TreeHash for u16 { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { ssz_encode(self) } } impl TreeHash for u32 { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { ssz_encode(self) } } impl TreeHash for u64 { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { ssz_encode(self) } } impl TreeHash for usize { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { ssz_encode(self) } } impl TreeHash for Address { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { ssz_encode(self) } } impl TreeHash for H256 { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { ssz_encode(self) } } impl TreeHash for [u8] { - fn hash_tree_root(&self) -> Vec { + fn hash_tree_root_internal(&self) -> Vec { if self.len() > 32 { return hash(&self); } @@ -57,12 +57,12 @@ impl TreeHash for Vec where T: TreeHash, { - /// Returns the merkle_hash of a list of hash_tree_root values created + /// Returns the merkle_hash of a list of hash_tree_root_internal values created /// from the given list. /// Note: A byte vector, Vec, must be converted to a slice (as_slice()) /// to be handled properly (i.e. hashed) as byte array. - fn hash_tree_root(&self) -> Vec { - let mut tree_hashes = self.iter().map(|x| x.hash_tree_root()).collect(); + fn hash_tree_root_internal(&self) -> Vec { + let mut tree_hashes = self.iter().map(|x| x.hash_tree_root_internal()).collect(); merkle_hash(&mut tree_hashes) } } @@ -73,7 +73,7 @@ mod tests { #[test] fn test_impl_tree_hash_vec() { - let result = vec![1u32, 2, 3, 4, 5, 6, 7].hash_tree_root(); + let result = vec![1u32, 2, 3, 4, 5, 6, 7].hash_tree_root_internal(); assert_eq!(result.len(), 32); } } diff --git a/eth2/utils/ssz/src/tree_hash.rs b/eth2/utils/ssz/src/tree_hash.rs index a9ab0f467..3f190e32e 100644 --- a/eth2/utils/ssz/src/tree_hash.rs +++ b/eth2/utils/ssz/src/tree_hash.rs @@ -4,7 +4,14 @@ const SSZ_CHUNK_SIZE: usize = 128; const HASHSIZE: usize = 32; pub trait TreeHash { - fn hash_tree_root(&self) -> Vec; + fn hash_tree_root_internal(&self) -> Vec; + fn hash_tree_root(&self) -> Vec { + let mut result = self.hash_tree_root_internal(); + if result.len() < HASHSIZE { + zpad(&mut result, HASHSIZE); + } + result + } } /// Returns a 32 byte hash of 'list' - a vector of byte vectors. @@ -14,7 +21,8 @@ pub fn merkle_hash(list: &mut Vec>) -> Vec { let (mut chunk_size, mut chunkz) = list_to_blob(list); // get data_len as bytes. It will hashed will the merkle root - let datalen = list.len().to_le_bytes(); + let mut datalen = list.len().to_le_bytes().to_vec(); + zpad(&mut datalen, 32); // Tree-hash while chunkz.len() > HASHSIZE { @@ -36,33 +44,63 @@ pub fn merkle_hash(list: &mut Vec>) -> Vec { chunkz = new_chunkz; } - chunkz.append(&mut datalen.to_vec()); + chunkz.append(&mut datalen); hash(&chunkz) } fn list_to_blob(list: &mut Vec>) -> (usize, Vec) { - let chunk_size = if list.is_empty() { + let chunk_size = if list.is_empty() || list[0].len() < SSZ_CHUNK_SIZE { SSZ_CHUNK_SIZE - } else if list[0].len() < SSZ_CHUNK_SIZE { - let items_per_chunk = SSZ_CHUNK_SIZE / list[0].len(); - items_per_chunk * list[0].len() } else { list[0].len() }; - let mut data = Vec::new(); + let items_per_chunk = SSZ_CHUNK_SIZE / list[0].len(); + let chunk_count = list.len() / items_per_chunk; + + let mut chunkz = Vec::new(); if list.is_empty() { // handle and empty list - data.append(&mut vec![0; SSZ_CHUNK_SIZE]); - } else { + chunkz.append(&mut vec![0; SSZ_CHUNK_SIZE]); + } else if list[0].len() <= SSZ_CHUNK_SIZE { // just create a blob here; we'll divide into // chunked slices when we merklize - data.reserve(list[0].len() * list.len()); + let mut chunk = Vec::with_capacity(chunk_size); + let mut item_count_in_chunk = 0; + chunkz.reserve(chunk_count * chunk_size); for item in list.iter_mut() { - data.append(item); + item_count_in_chunk += 1; + chunk.append(item); + + // completed chunk? + if item_count_in_chunk == items_per_chunk { + zpad(&mut chunk, chunk_size); + chunkz.append(&mut chunk); + item_count_in_chunk = 0; + } + } + + // left-over uncompleted chunk? + if item_count_in_chunk != 0 { + zpad(&mut chunk, chunk_size); + chunkz.append(&mut chunk); + } + } else { + // chunks larger than SSZ_CHUNK_SIZE + chunkz.reserve(chunk_count * chunk_size); + for item in list.iter_mut() { + chunkz.append(item); } } - (chunk_size, data) + + (chunk_size, chunkz) +} + +/// right pads with zeros making 'bytes' 'size' in length +fn zpad(bytes: &mut Vec, size: usize) { + if bytes.len() < size { + bytes.resize(size, 0); + } } #[cfg(test)] From 8b34bc490b60e0534a1a10c0f736070e0ff06d03 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 18 Feb 2019 17:32:13 +1100 Subject: [PATCH 14/39] Add fork-choice yaml tests. --- eth2/fork_choice/Cargo.toml | 2 + eth2/fork_choice/src/lib.rs | 2 + eth2/fork_choice/src/optimised_lmd_ghost.rs | 51 ++-- .../tests/fork_choice_test_vectors.yaml | 21 ++ .../tests/optimised_lmd_ghost_test.rs | 248 ++++++++++++++++++ 5 files changed, 302 insertions(+), 22 deletions(-) create mode 100644 eth2/fork_choice/tests/fork_choice_test_vectors.yaml create mode 100644 eth2/fork_choice/tests/optimised_lmd_ghost_test.rs diff --git a/eth2/fork_choice/Cargo.toml b/eth2/fork_choice/Cargo.toml index 72a653032..2765622d1 100644 --- a/eth2/fork_choice/Cargo.toml +++ b/eth2/fork_choice/Cargo.toml @@ -10,9 +10,11 @@ ssz = { path = "../utils/ssz" } types = { path = "../types" } fast-math = "0.1.1" byteorder = "1.3.1" +log = "0.4.6" [dev-dependencies] yaml-rust = "0.4.2" bls = { path = "../utils/bls" } slot_clock = { path = "../utils/slot_clock" } beacon_chain = { path = "../../beacon_node/beacon_chain" } +env_logger = "0.6.0" diff --git a/eth2/fork_choice/src/lib.rs b/eth2/fork_choice/src/lib.rs index c0df820c6..7b97fdc0e 100644 --- a/eth2/fork_choice/src/lib.rs +++ b/eth2/fork_choice/src/lib.rs @@ -41,6 +41,8 @@ extern crate db; extern crate ssz; extern crate types; +#[macro_use] +extern crate log; pub mod longest_chain; pub mod optimised_lmd_ghost; diff --git a/eth2/fork_choice/src/optimised_lmd_ghost.rs b/eth2/fork_choice/src/optimised_lmd_ghost.rs index 6b21e39f8..8f538da0a 100644 --- a/eth2/fork_choice/src/optimised_lmd_ghost.rs +++ b/eth2/fork_choice/src/optimised_lmd_ghost.rs @@ -1,23 +1,3 @@ -// Copyright 2019 Sigma Prime Pty Ltd. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - extern crate byteorder; extern crate fast_math; use crate::{ForkChoice, ForkChoiceError}; @@ -108,6 +88,7 @@ where // Note: Votes are weighted by min(balance, MAX_DEPOSIT_AMOUNT) // // FORK_CHOICE_BALANCE_INCREMENT // build a hashmap of block_hash to weighted votes + trace!("FORKCHOICE: Getting the latest votes"); let mut latest_votes: HashMap = HashMap::new(); // gets the current weighted votes let current_state = self @@ -119,6 +100,10 @@ where ¤t_state.validator_registry[..], block_slot.epoch(EPOCH_LENGTH), ); + trace!( + "FORKCHOICE: Active validator indicies: {:?}", + active_validator_indices + ); for index in active_validator_indices { let balance = @@ -130,7 +115,7 @@ where } } } - + trace!("FORKCHOICE: Latest votes: {:?}", latest_votes); Ok(latest_votes) } @@ -212,6 +197,7 @@ where // Finds the best child, splitting children into a binary tree, based on their hashes fn choose_best_child(&self, votes: &HashMap) -> Option { + println!("Votes: {:?}", votes); let mut bitmask = 0; for bit in (0..=255).rev() { let mut zero_votes = 0; @@ -298,12 +284,21 @@ impl ForkChoice for OptimisedLMDGhost { ) -> Result<(), ForkChoiceError> { // simply add the attestation to the latest_attestation_target if the block_height is // larger + trace!( + "FORKCHOICE: Adding attestation of validator: {:?} for block: {:?}", + validator_index, + target_block_root + ); let attestation_target = self .latest_attestation_targets .entry(validator_index) .or_insert_with(|| *target_block_root); // if we already have a value if attestation_target != target_block_root { + trace!( + "FORKCHOICE: Old attestation found: {:?}", + attestation_target + ); // get the height of the target block let block_height = self .block_store @@ -319,8 +314,11 @@ impl ForkChoice for OptimisedLMDGhost { .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*attestation_target))? .slot() .height(Slot::from(GENESIS_SLOT)); + trace!("FORKCHOICE: Old block height: {:?}", past_block_height); + trace!("FORKCHOICE: New block height: {:?}", block_height); // update the attestation only if the new target is higher if past_block_height < block_height { + trace!("FORKCHOICE: Updating old attestation"); *attestation_target = *target_block_root; } } @@ -329,6 +327,7 @@ impl ForkChoice for OptimisedLMDGhost { /// Perform lmd_ghost on the current chain to find the head. fn find_head(&mut self, justified_block_start: &Hash256) -> Result { + trace!("Starting optimised fork choice"); let block = self .block_store .get_deserialized(&justified_block_start)? @@ -344,6 +343,7 @@ impl ForkChoice for OptimisedLMDGhost { // remove any votes that don't relate to our current head. latest_votes.retain(|hash, _| self.get_ancestor(*hash, block_height) == Some(current_head)); + trace!("FORKCHOICE: Latest votes: {:?}", latest_votes); // begin searching for the head loop { @@ -368,13 +368,19 @@ impl ForkChoice for OptimisedLMDGhost { step /= 2; } if step > 0 { + trace!("FORKCHOICE: Found clear winner in log lookup"); } // if our skip lookup failed and we only have one child, progress to that child else if children.len() == 1 { current_head = children[0]; + trace!( + "FORKCHOICE: Lookup failed, only one child, proceeding to child: {}", + current_head + ); } // we need to find the best child path to progress down. else { + trace!("FORKCHOICE: Searching for best child"); let mut child_votes = HashMap::new(); for (voted_hash, vote) in latest_votes.iter() { // if the latest votes correspond to a child @@ -383,14 +389,15 @@ impl ForkChoice for OptimisedLMDGhost { *child_votes.entry(child).or_insert_with(|| 0) += vote; } } + println!("Child votes: {:?}", child_votes); // given the votes on the children, find the best child current_head = self .choose_best_child(&child_votes) .ok_or(ForkChoiceError::CannotFindBestChild)?; + trace!("FORKCHOICE: Best child found: {}", current_head); } // No head was found, re-iterate - // update the block height for the next iteration let block_height = self .block_store diff --git a/eth2/fork_choice/tests/fork_choice_test_vectors.yaml b/eth2/fork_choice/tests/fork_choice_test_vectors.yaml new file mode 100644 index 000000000..5a8869e8b --- /dev/null +++ b/eth2/fork_choice/tests/fork_choice_test_vectors.yaml @@ -0,0 +1,21 @@ +title: Fork-choice Tests +summary: A collection of abstract fork-choice tests. +test_suite: Fork-Choice + +test_cases: +- blocks: + - id: 'b0' + parent: 'b0' + - id: 'b1' + parent: 'b0' + - id: 'b2' + parent: 'b1' + - id: 'b3' + parent: 'b1' + weights: + - b0: 0 + - b1: 0 + - b2: 5 + - b3: 10 + heads: + - id: 'b3' diff --git a/eth2/fork_choice/tests/optimised_lmd_ghost_test.rs b/eth2/fork_choice/tests/optimised_lmd_ghost_test.rs new file mode 100644 index 000000000..ac0b6888c --- /dev/null +++ b/eth2/fork_choice/tests/optimised_lmd_ghost_test.rs @@ -0,0 +1,248 @@ +// Tests the optimised LMD Ghost Algorithm + +extern crate beacon_chain; +extern crate bls; +extern crate db; +extern crate env_logger; +extern crate fork_choice; +extern crate log; +extern crate slot_clock; +extern crate types; +extern crate yaml_rust; + +pub use beacon_chain::BeaconChain; +use bls::{PublicKey, Signature}; +use db::stores::{BeaconBlockStore, BeaconStateStore}; +use db::MemoryDB; +use env_logger::{Builder, Env}; +use fork_choice::{ForkChoice, ForkChoiceError, OptimisedLMDGhost}; +use ssz::ssz_encode; +use std::collections::HashMap; +use std::sync::Arc; +use std::{fs::File, io::prelude::*, path::PathBuf, str::FromStr}; +use types::test_utils::{SeedableRng, TestRandom, XorShiftRng}; +use types::validator_registry::get_active_validator_indices; +use types::{ + BeaconBlock, BeaconBlockBody, BeaconState, ChainSpec, Deposit, DepositData, DepositInput, + Eth1Data, Hash256, Slot, Validator, +}; +use yaml_rust::yaml; + +// initialise a single validator and state. All blocks will reference this state root. +fn setup_inital_state( + no_validators: usize, +) -> (impl ForkChoice, Arc>, Hash256) { + let zero_hash = Hash256::zero(); + + let db = Arc::new(MemoryDB::open()); + let block_store = Arc::new(BeaconBlockStore::new(db.clone())); + let state_store = Arc::new(BeaconStateStore::new(db.clone())); + + // the fork choice instantiation + let optimised_lmd_ghost = OptimisedLMDGhost::new(block_store.clone(), state_store.clone()); + + // misc vars for setting up the state + let genesis_time = 1_550_381_159; + + let latest_eth1_data = Eth1Data { + deposit_root: zero_hash.clone(), + block_hash: zero_hash.clone(), + }; + + let initial_validator_deposits = vec![]; + + /* + (0..no_validators) + .map(|_| Deposit { + branch: vec![], + index: 0, + deposit_data: DepositData { + amount: 32_000_000_000, // 32 ETH (in Gwei) + timestamp: genesis_time - 1, + deposit_input: DepositInput { + pubkey: PublicKey::default(), + withdrawal_credentials: zero_hash.clone(), + proof_of_possession: Signature::empty_signature(), + }, + }, + }) + .collect(); + */ + + let spec = ChainSpec::foundation(); + + // create the state + let mut state = BeaconState::genesis( + genesis_time, + initial_validator_deposits, + latest_eth1_data, + &spec, + ) + .unwrap(); + + // activate the validators + for _ in 0..no_validators { + let validator = Validator { + pubkey: PublicKey::default(), + withdrawal_credentials: zero_hash, + activation_epoch: spec.far_future_epoch, + exit_epoch: spec.far_future_epoch, + withdrawal_epoch: spec.far_future_epoch, + penalized_epoch: spec.far_future_epoch, + status_flags: None, + }; + state.validator_registry.push(validator); + state.validator_balances.push(32_000_000_000); + } + + let state_root = state.canonical_root(); + state_store + .put(&state_root, &ssz_encode(&state)[..]) + .unwrap(); + + println!( + "Active: {:?}", + get_active_validator_indices( + &state.validator_registry, + Slot::from(5u64).epoch(spec.EPOCH_LENGTH) + ) + ); + // return initialised vars + (optimised_lmd_ghost, block_store, state_root) +} + +// YAML test vectors +#[test] +fn test_optimised_lmd_ghost() { + // set up logging + Builder::from_env(Env::default().default_filter_or("trace")).init(); + + // load the yaml + let mut file = { + let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + file_path_buf.push("tests/fork_choice_test_vectors.yaml"); + + File::open(file_path_buf).unwrap() + }; + + let mut yaml_str = String::new(); + file.read_to_string(&mut yaml_str).unwrap(); + let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap(); + let doc = &docs[0]; + let test_cases = doc["test_cases"].as_vec().unwrap(); + + // set up the test + let total_emulated_validators = 20; // the number of validators used to give weights. + let (mut fork_choice, block_store, state_root) = setup_inital_state(total_emulated_validators); + + // keep a hashmap of block_id's to block_hashes (random hashes to abstract block_id) + let mut block_id_map: HashMap = HashMap::new(); + // keep a list of hash to slot + let mut block_slot: HashMap = HashMap::new(); + + // default vars + let zero_hash = Hash256::zero(); + let eth1_data = Eth1Data { + deposit_root: zero_hash.clone(), + block_hash: zero_hash.clone(), + }; + let randao_reveal = Signature::empty_signature(); + let signature = Signature::empty_signature(); + let body = BeaconBlockBody { + proposer_slashings: vec![], + attester_slashings: vec![], + attestations: vec![], + deposits: vec![], + exits: vec![], + }; + + // process the tests + for test_case in test_cases { + // assume the block tree is given to us in order. + for block in test_case["blocks"].clone().into_vec().unwrap() { + let block_id = block["id"].as_str().unwrap().to_string(); + let parent_id = block["parent"].as_str().unwrap(); + + // default params for genesis + let mut block_hash = zero_hash.clone(); + let mut slot = Slot::from(0u64); + let mut parent_root = zero_hash; + + // set the slot and parent based off the YAML. Start with genesis; + // if not the genesis, update slot and parent + if parent_id != block_id { + // generate a random hash for the block_hash + block_hash = Hash256::random(); + // find the parent hash + parent_root = *block_id_map + .get(parent_id) + .expect(&format!("Parent not found: {}", parent_id)); + slot = *(block_slot + .get(&parent_root) + .expect("Parent should have a slot number")) + + 1; + } + + block_id_map.insert(block_id.clone(), block_hash.clone()); + + // update slot mapping + block_slot.insert(block_hash, slot); + + // build the BeaconBlock + let beacon_block = BeaconBlock { + slot, + parent_root, + state_root: state_root.clone(), + randao_reveal: randao_reveal.clone(), + eth1_data: eth1_data.clone(), + signature: signature.clone(), + body: body.clone(), + }; + + // Store the block and state. + block_store + .put(&block_hash, &ssz_encode(&beacon_block)[..]) + .unwrap(); + + // run add block for fork choice if not genesis + if parent_id != block_id { + fork_choice.add_block(&beacon_block, &block_hash).unwrap(); + } + } + + // add the weights (attestations) + let mut current_validator = 0; + for id_map in test_case["weights"].clone().into_vec().unwrap() { + // get the block id and weights + for (map_id, map_weight) in id_map.as_hash().unwrap().iter() { + let id = map_id.as_str().unwrap(); + let block_root = block_id_map + .get(id) + .expect(&format!("Cannot find block id: {} in weights", id)); + let weight = map_weight.as_i64().unwrap(); + // we assume a validator has a value 1 and add an attestation for to achieve the + // correct weight + for _ in 0..weight { + assert!(current_validator <= total_emulated_validators); + fork_choice + .add_attestation(current_validator as u64, &block_root) + .unwrap(); + current_validator += 1; + } + } + } + + // everything is set up, run the fork choice, using genesis as the head + println!("Running fork choice"); + let head = fork_choice.find_head(&zero_hash).unwrap(); + + let mut found_id = None; + for (id, block_hash) in block_id_map.iter() { + if *block_hash == head { + found_id = Some(id); + } + } + + println!("Head Block ID: {:?}", found_id); + } +} From 6e6d45197894481a3cdc5f7f211c93c60c888503 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 18 Feb 2019 17:42:07 +1100 Subject: [PATCH 15/39] Updates optimised ghost to use chainspec for consts. --- eth2/fork_choice/src/lib.rs | 10 +++- eth2/fork_choice/src/optimised_lmd_ghost.rs | 62 ++++++++++++--------- 2 files changed, 44 insertions(+), 28 deletions(-) diff --git a/eth2/fork_choice/src/lib.rs b/eth2/fork_choice/src/lib.rs index 7b97fdc0e..11da12304 100644 --- a/eth2/fork_choice/src/lib.rs +++ b/eth2/fork_choice/src/lib.rs @@ -50,7 +50,7 @@ pub mod slow_lmd_ghost; use db::stores::BeaconBlockAtSlotError; use db::DBError; -use types::{BeaconBlock, Hash256}; +use types::{BeaconBlock, ChainSpec, Hash256}; pub use longest_chain::LongestChain; pub use optimised_lmd_ghost::OptimisedLMDGhost; @@ -65,6 +65,7 @@ pub trait ForkChoice: Send + Sync { &mut self, block: &BeaconBlock, block_hash: &Hash256, + spec: &ChainSpec, ) -> Result<(), ForkChoiceError>; /// Called when an attestation has been added. Allows generic attestation-level data structures to be built for a given fork choice. // This can be generalised to a full attestation if required later. @@ -72,10 +73,15 @@ pub trait ForkChoice: Send + Sync { &mut self, validator_index: u64, target_block_hash: &Hash256, + spec: &ChainSpec, ) -> Result<(), ForkChoiceError>; /// The fork-choice algorithm to find the current canonical head of the chain. // TODO: Remove the justified_start_block parameter and make it internal - fn find_head(&mut self, justified_start_block: &Hash256) -> Result; + fn find_head( + &mut self, + justified_start_block: &Hash256, + spec: &ChainSpec, + ) -> Result; } /// Possible fork choice errors that can occur. diff --git a/eth2/fork_choice/src/optimised_lmd_ghost.rs b/eth2/fork_choice/src/optimised_lmd_ghost.rs index 8f538da0a..b88aa09bd 100644 --- a/eth2/fork_choice/src/optimised_lmd_ghost.rs +++ b/eth2/fork_choice/src/optimised_lmd_ghost.rs @@ -11,18 +11,12 @@ use std::collections::HashMap; use std::sync::Arc; use types::{ readers::BeaconBlockReader, validator_registry::get_active_validator_indices, BeaconBlock, - Hash256, Slot, SlotHeight, + ChainSpec, Hash256, Slot, SlotHeight, }; //TODO: Pruning - Children //TODO: Handle Syncing -//TODO: Sort out global constants -const GENESIS_SLOT: u64 = 0; -const FORK_CHOICE_BALANCE_INCREMENT: u64 = 1e9 as u64; -const MAX_DEPOSIT_AMOUNT: u64 = 32e9 as u64; -const EPOCH_LENGTH: u64 = 64; - /// The optimised LMD-GHOST fork choice rule. /// NOTE: This uses u32 to represent difference between block heights. Thus this is only /// applicable for block height differences in the range of a u32. @@ -82,7 +76,8 @@ where pub fn get_latest_votes( &self, state_root: &Hash256, - block_slot: Slot, + block_slot: &Slot, + spec: &ChainSpec, ) -> Result, ForkChoiceError> { // get latest votes // Note: Votes are weighted by min(balance, MAX_DEPOSIT_AMOUNT) // @@ -98,7 +93,7 @@ where let active_validator_indices = get_active_validator_indices( ¤t_state.validator_registry[..], - block_slot.epoch(EPOCH_LENGTH), + block_slot.epoch(spec.epoch_length), ); trace!( "FORKCHOICE: Active validator indicies: {:?}", @@ -106,9 +101,10 @@ where ); for index in active_validator_indices { - let balance = - std::cmp::min(current_state.validator_balances[index], MAX_DEPOSIT_AMOUNT) - / FORK_CHOICE_BALANCE_INCREMENT; + let balance = std::cmp::min( + current_state.validator_balances[index], + spec.max_deposit_amount, + ) / spec.fork_choice_balance_increment; if balance > 0 { if let Some(target) = self.latest_attestation_targets.get(&(index as u64)) { *latest_votes.entry(*target).or_insert_with(|| 0) += balance; @@ -120,7 +116,12 @@ where } /// Gets the ancestor at a given height `at_height` of a block specified by `block_hash`. - fn get_ancestor(&mut self, block_hash: Hash256, at_height: SlotHeight) -> Option { + fn get_ancestor( + &mut self, + block_hash: Hash256, + at_height: SlotHeight, + spec: &ChainSpec, + ) -> Option { // return None if we can't get the block from the db. let block_height = { let block_slot = self @@ -130,7 +131,7 @@ where .expect("Should have returned already if None") .slot; - block_slot.height(Slot::from(GENESIS_SLOT)) + block_slot.height(Slot::from(spec.genesis_slot)) }; // verify we haven't exceeded the block height @@ -155,7 +156,7 @@ where .get(&block_hash) //TODO: Panic if we can't lookup and fork choice fails .expect("All blocks should be added to the ancestor log lookup table"); - self.get_ancestor(*ancestor_lookup, at_height) + self.get_ancestor(*ancestor_lookup, at_height, &spec) } { // add the result to the cache self.cache.insert(cache_key, ancestor); @@ -170,6 +171,7 @@ where &mut self, latest_votes: &HashMap, block_height: SlotHeight, + spec: &ChainSpec, ) -> Option { // map of vote counts for every hash at this height let mut current_votes: HashMap = HashMap::new(); @@ -178,7 +180,7 @@ where // loop through the latest votes and count all votes // these have already been weighted by balance for (hash, votes) in latest_votes.iter() { - if let Some(ancestor) = self.get_ancestor(*hash, block_height) { + if let Some(ancestor) = self.get_ancestor(*hash, block_height, spec) { let current_vote_value = current_votes.get(&ancestor).unwrap_or_else(|| &0); current_votes.insert(ancestor, current_vote_value + *votes); total_vote_count += votes; @@ -244,6 +246,7 @@ impl ForkChoice for OptimisedLMDGhost { &mut self, block: &BeaconBlock, block_hash: &Hash256, + spec: &ChainSpec, ) -> Result<(), ForkChoiceError> { // get the height of the parent let parent_height = self @@ -251,7 +254,7 @@ impl ForkChoice for OptimisedLMDGhost { .get_deserialized(&block.parent_root)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(block.parent_root))? .slot() - .height(Slot::from(GENESIS_SLOT)); + .height(Slot::from(spec.genesis_slot)); let parent_hash = &block.parent_root; @@ -281,6 +284,7 @@ impl ForkChoice for OptimisedLMDGhost { &mut self, validator_index: u64, target_block_root: &Hash256, + spec: &ChainSpec, ) -> Result<(), ForkChoiceError> { // simply add the attestation to the latest_attestation_target if the block_height is // larger @@ -305,7 +309,7 @@ impl ForkChoice for OptimisedLMDGhost { .get_deserialized(&target_block_root)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*target_block_root))? .slot() - .height(Slot::from(GENESIS_SLOT)); + .height(Slot::from(spec.genesis_slot)); // get the height of the past target block let past_block_height = self @@ -313,7 +317,7 @@ impl ForkChoice for OptimisedLMDGhost { .get_deserialized(&attestation_target)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*attestation_target))? .slot() - .height(Slot::from(GENESIS_SLOT)); + .height(Slot::from(spec.genesis_slot)); trace!("FORKCHOICE: Old block height: {:?}", past_block_height); trace!("FORKCHOICE: New block height: {:?}", block_height); // update the attestation only if the new target is higher @@ -326,7 +330,11 @@ impl ForkChoice for OptimisedLMDGhost { } /// Perform lmd_ghost on the current chain to find the head. - fn find_head(&mut self, justified_block_start: &Hash256) -> Result { + fn find_head( + &mut self, + justified_block_start: &Hash256, + spec: &ChainSpec, + ) -> Result { trace!("Starting optimised fork choice"); let block = self .block_store @@ -334,7 +342,7 @@ impl ForkChoice for OptimisedLMDGhost { .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*justified_block_start))?; let block_slot = block.slot(); - let block_height = block_slot.height(Slot::from(GENESIS_SLOT)); + let block_height = block_slot.height(Slot::from(spec.genesis_slot)); let state_root = block.state_root(); let mut current_head = *justified_block_start; @@ -342,7 +350,8 @@ impl ForkChoice for OptimisedLMDGhost { let mut latest_votes = self.get_latest_votes(&state_root, block_slot)?; // remove any votes that don't relate to our current head. - latest_votes.retain(|hash, _| self.get_ancestor(*hash, block_height) == Some(current_head)); + latest_votes + .retain(|hash, _| self.get_ancestor(*hash, block_height, spec) == Some(current_head)); trace!("FORKCHOICE: Latest votes: {:?}", latest_votes); // begin searching for the head @@ -384,7 +393,7 @@ impl ForkChoice for OptimisedLMDGhost { let mut child_votes = HashMap::new(); for (voted_hash, vote) in latest_votes.iter() { // if the latest votes correspond to a child - if let Some(child) = self.get_ancestor(*voted_hash, block_height + 1) { + if let Some(child) = self.get_ancestor(*voted_hash, block_height + 1, spec) { // add up the votes for each child *child_votes.entry(child).or_insert_with(|| 0) += vote; } @@ -404,12 +413,13 @@ impl ForkChoice for OptimisedLMDGhost { .get_deserialized(¤t_head)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*justified_block_start))? .slot() - .height(Slot::from(GENESIS_SLOT)); + .height(Slot::from(spec.genesis_slot)); // prune the latest votes for votes that are not part of current chosen chain // more specifically, only keep votes that have head as an ancestor - latest_votes - .retain(|hash, _| self.get_ancestor(*hash, block_height) == Some(current_head)); + latest_votes.retain(|hash, _| { + self.get_ancestor(*hash, block_height, spec) == Some(current_head) + }); } } } From 4eddb47fd059e887de56b7941976eb7721bfc9e7 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 18 Feb 2019 17:49:05 +1100 Subject: [PATCH 16/39] Updates all fork-choices to use ChainSpec for consts. --- beacon_node/beacon_chain/src/beacon_chain.rs | 10 ++++- eth2/fork_choice/src/longest_chain.rs | 12 ++++-- eth2/fork_choice/src/optimised_lmd_ghost.rs | 3 +- eth2/fork_choice/src/slow_lmd_ghost.rs | 43 ++++++++++++-------- 4 files changed, 44 insertions(+), 24 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index b2d041654..40e30b2fb 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -357,6 +357,7 @@ where self.fork_choice.write().add_attestation( free_attestation.validator_index, &free_attestation.data.beacon_block_root, + &self.spec, )?; Ok(aggregation_outcome) } @@ -486,7 +487,9 @@ where self.state_store.put(&state_root, &ssz_encode(&state)[..])?; // run the fork_choice add_block logic - self.fork_choice.write().add_block(&block, &block_root)?; + self.fork_choice + .write() + .add_block(&block, &block_root, &self.spec)?; // If the parent block was the parent_block, automatically update the canonical head. // @@ -575,7 +578,10 @@ where pub fn fork_choice(&self) -> Result<(), Error> { let present_head = self.finalized_head().beacon_block_root; - let new_head = self.fork_choice.write().find_head(&present_head)?; + let new_head = self + .fork_choice + .write() + .find_head(&present_head, &self.spec)?; if new_head != present_head { let block = self diff --git a/eth2/fork_choice/src/longest_chain.rs b/eth2/fork_choice/src/longest_chain.rs index 8056c11f2..ea2cc33bb 100644 --- a/eth2/fork_choice/src/longest_chain.rs +++ b/eth2/fork_choice/src/longest_chain.rs @@ -1,7 +1,7 @@ use crate::{ForkChoice, ForkChoiceError}; use db::{stores::BeaconBlockStore, ClientDB}; use std::sync::Arc; -use types::{BeaconBlock, Hash256, Slot}; +use types::{BeaconBlock, ChainSpec, Hash256, Slot}; pub struct LongestChain where @@ -30,6 +30,7 @@ impl ForkChoice for LongestChain { &mut self, block: &BeaconBlock, block_hash: &Hash256, + spec: &ChainSpec, ) -> Result<(), ForkChoiceError> { // add the block hash to head_block_hashes removing the parent if it exists self.head_block_hashes @@ -38,12 +39,17 @@ impl ForkChoice for LongestChain { Ok(()) } - fn add_attestation(&mut self, _: u64, _: &Hash256) -> Result<(), ForkChoiceError> { + fn add_attestation( + &mut self, + _: u64, + _: &Hash256, + _: &ChainSpec, + ) -> Result<(), ForkChoiceError> { // do nothing Ok(()) } - fn find_head(&mut self, _: &Hash256) -> Result { + fn find_head(&mut self, _: &Hash256, _: &ChainSpec) -> Result { let mut head_blocks: Vec<(usize, BeaconBlock)> = vec![]; /* * Load all the head_block hashes from the DB as SszBeaconBlocks. diff --git a/eth2/fork_choice/src/optimised_lmd_ghost.rs b/eth2/fork_choice/src/optimised_lmd_ghost.rs index b88aa09bd..e60af4e66 100644 --- a/eth2/fork_choice/src/optimised_lmd_ghost.rs +++ b/eth2/fork_choice/src/optimised_lmd_ghost.rs @@ -347,7 +347,7 @@ impl ForkChoice for OptimisedLMDGhost { let mut current_head = *justified_block_start; - let mut latest_votes = self.get_latest_votes(&state_root, block_slot)?; + let mut latest_votes = self.get_latest_votes(&state_root, &block_slot, spec)?; // remove any votes that don't relate to our current head. latest_votes @@ -370,6 +370,7 @@ impl ForkChoice for OptimisedLMDGhost { if let Some(clear_winner) = self.get_clear_winner( &latest_votes, block_height - (block_height % u64::from(step)) + u64::from(step), + spec, ) { current_head = clear_winner; break; diff --git a/eth2/fork_choice/src/slow_lmd_ghost.rs b/eth2/fork_choice/src/slow_lmd_ghost.rs index 3184150fd..81bc5d628 100644 --- a/eth2/fork_choice/src/slow_lmd_ghost.rs +++ b/eth2/fork_choice/src/slow_lmd_ghost.rs @@ -29,17 +29,11 @@ use std::collections::HashMap; use std::sync::Arc; use types::{ readers::BeaconBlockReader, validator_registry::get_active_validator_indices, BeaconBlock, - Hash256, Slot, + ChainSpec, Hash256, Slot, }; //TODO: Pruning and syncing -//TODO: Sort out global constants -const GENESIS_SLOT: u64 = 0; -const FORK_CHOICE_BALANCE_INCREMENT: u64 = 1e9 as u64; -const MAX_DEPOSIT_AMOUNT: u64 = 32e9 as u64; -const EPOCH_LENGTH: u64 = 64; - pub struct SlowLMDGhost { /// The latest attestation targets as a map of validator index to block hash. //TODO: Could this be a fixed size vec @@ -70,12 +64,14 @@ where pub fn get_latest_votes( &self, state_root: &Hash256, - block_slot: Slot, + block_slot: &Slot, + spec: &ChainSpec, ) -> Result, ForkChoiceError> { // get latest votes // Note: Votes are weighted by min(balance, MAX_DEPOSIT_AMOUNT) // // FORK_CHOICE_BALANCE_INCREMENT // build a hashmap of block_hash to weighted votes + trace!("FORKCHOICE: Getting the latest votes"); let mut latest_votes: HashMap = HashMap::new(); // gets the current weighted votes let current_state = self @@ -84,21 +80,26 @@ where .ok_or_else(|| ForkChoiceError::MissingBeaconState(*state_root))?; let active_validator_indices = get_active_validator_indices( - ¤t_state.validator_registry, - block_slot.epoch(EPOCH_LENGTH), + ¤t_state.validator_registry[..], + block_slot.epoch(spec.epoch_length), + ); + trace!( + "FORKCHOICE: Active validator indicies: {:?}", + active_validator_indices ); for index in active_validator_indices { - let balance = - std::cmp::min(current_state.validator_balances[index], MAX_DEPOSIT_AMOUNT) - / FORK_CHOICE_BALANCE_INCREMENT; + let balance = std::cmp::min( + current_state.validator_balances[index], + spec.max_deposit_amount, + ) / spec.fork_choice_balance_increment; if balance > 0 { if let Some(target) = self.latest_attestation_targets.get(&(index as u64)) { *latest_votes.entry(*target).or_insert_with(|| 0) += balance; } } } - + trace!("FORKCHOICE: Latest votes: {:?}", latest_votes); Ok(latest_votes) } @@ -136,6 +137,7 @@ impl ForkChoice for SlowLMDGhost { &mut self, block: &BeaconBlock, block_hash: &Hash256, + _: &ChainSpec, ) -> Result<(), ForkChoiceError> { // build the children hashmap // add the new block to the children of parent @@ -153,6 +155,7 @@ impl ForkChoice for SlowLMDGhost { &mut self, validator_index: u64, target_block_root: &Hash256, + spec: &ChainSpec, ) -> Result<(), ForkChoiceError> { // simply add the attestation to the latest_attestation_target if the block_height is // larger @@ -168,7 +171,7 @@ impl ForkChoice for SlowLMDGhost { .get_deserialized(&target_block_root)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*target_block_root))? .slot() - .height(Slot::from(GENESIS_SLOT)); + .height(Slot::from(spec.genesis_slot)); // get the height of the past target block let past_block_height = self @@ -176,7 +179,7 @@ impl ForkChoice for SlowLMDGhost { .get_deserialized(&attestation_target)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*attestation_target))? .slot() - .height(Slot::from(GENESIS_SLOT)); + .height(Slot::from(spec.genesis_slot)); // update the attestation only if the new target is higher if past_block_height < block_height { *attestation_target = *target_block_root; @@ -186,7 +189,11 @@ impl ForkChoice for SlowLMDGhost { } /// A very inefficient implementation of LMD ghost. - fn find_head(&mut self, justified_block_start: &Hash256) -> Result { + fn find_head( + &mut self, + justified_block_start: &Hash256, + spec: &ChainSpec, + ) -> Result { let start = self .block_store .get_deserialized(&justified_block_start)? @@ -194,7 +201,7 @@ impl ForkChoice for SlowLMDGhost { let start_state_root = start.state_root(); - let latest_votes = self.get_latest_votes(&start_state_root, start.slot())?; + let latest_votes = self.get_latest_votes(&start_state_root, &start.slot(), spec)?; let mut head_hash = Hash256::zero(); From f88155625c709efd6dc42c7faae537e6bff486c6 Mon Sep 17 00:00:00 2001 From: thojest Date: Mon, 18 Feb 2019 12:12:01 +0100 Subject: [PATCH 17/39] added comment to indicate highest spec version of implemented functions; added realistic test scenario for is_surround_vote (lighthouse-150) --- eth2/types/src/slashable_vote_data.rs | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/eth2/types/src/slashable_vote_data.rs b/eth2/types/src/slashable_vote_data.rs index 170cd635e..ff9e8b658 100644 --- a/eth2/types/src/slashable_vote_data.rs +++ b/eth2/types/src/slashable_vote_data.rs @@ -1,5 +1,5 @@ use super::AttestationData; -use crate::spec::ChainSpec; +use crate::chain_spec::ChainSpec; use crate::test_utils::TestRandom; use bls::AggregateSignature; use rand::RngCore; @@ -15,10 +15,16 @@ pub struct SlashableVoteData { } impl SlashableVoteData { + /// Check if ``attestation_data_1`` and ``attestation_data_2`` have the same target. + /// + /// Spec v0.3.0 pub fn is_double_vote(&self, other: &SlashableVoteData, spec: &ChainSpec) -> bool { self.data.slot.epoch(spec.epoch_length) == other.data.slot.epoch(spec.epoch_length) } + /// Check if ``attestation_data_1`` surrounds ``attestation_data_2``. + /// + /// Spec v0.3.0 pub fn is_surround_vote(&self, other: &SlashableVoteData, spec: &ChainSpec) -> bool { let source_epoch_1 = self.data.justified_epoch; let source_epoch_2 = other.data.justified_epoch; @@ -82,8 +88,8 @@ impl TestRandom for SlashableVoteData { #[cfg(test)] mod tests { use super::*; + use crate::chain_spec::ChainSpec; use crate::slot_epoch::{Epoch, Slot}; - use crate::spec::ChainSpec; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; use ssz::ssz_encode; @@ -123,6 +129,18 @@ mod tests { ); } + #[test] + pub fn test_is_surround_vote_true_realistic() { + let spec = ChainSpec::foundation(); + let slashable_vote_first = create_slashable_vote_data(4, 1, &spec); + let slashable_vote_second = create_slashable_vote_data(3, 2, &spec); + + assert_eq!( + slashable_vote_first.is_surround_vote(&slashable_vote_second, &spec), + true + ); + } + #[test] pub fn test_is_surround_vote_false_source_epoch_fails() { let spec = ChainSpec::foundation(); From 1bdce182a94fa45efafd8ad365924613823fc372 Mon Sep 17 00:00:00 2001 From: Feng94 <8751189+Feng94@users.noreply.github.com> Date: Tue, 19 Feb 2019 00:03:35 +1100 Subject: [PATCH 18/39] Rename block_producer crate to block_proposer and change references inside it to block_proposer --- eth2/{block_producer => block_proposer}/Cargo.toml | 2 +- eth2/{block_producer => block_proposer}/src/lib.rs | 12 ++++++------ .../src/test_utils/epoch_map.rs | 0 .../src/test_utils/local_signer.rs | 0 .../src/test_utils/mod.rs | 0 .../src/test_utils/simulated_beacon_node.rs | 0 .../{block_producer => block_proposer}/src/traits.rs | 0 7 files changed, 7 insertions(+), 7 deletions(-) rename eth2/{block_producer => block_proposer}/Cargo.toml (91%) rename eth2/{block_producer => block_proposer}/src/lib.rs (97%) rename eth2/{block_producer => block_proposer}/src/test_utils/epoch_map.rs (100%) rename eth2/{block_producer => block_proposer}/src/test_utils/local_signer.rs (100%) rename eth2/{block_producer => block_proposer}/src/test_utils/mod.rs (100%) rename eth2/{block_producer => block_proposer}/src/test_utils/simulated_beacon_node.rs (100%) rename eth2/{block_producer => block_proposer}/src/traits.rs (100%) diff --git a/eth2/block_producer/Cargo.toml b/eth2/block_proposer/Cargo.toml similarity index 91% rename from eth2/block_producer/Cargo.toml rename to eth2/block_proposer/Cargo.toml index 15d1343cc..81f1ccc28 100644 --- a/eth2/block_producer/Cargo.toml +++ b/eth2/block_proposer/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "block_producer" +name = "block_proposer" version = "0.1.0" authors = ["Paul Hauner "] edition = "2018" diff --git a/eth2/block_producer/src/lib.rs b/eth2/block_proposer/src/lib.rs similarity index 97% rename from eth2/block_producer/src/lib.rs rename to eth2/block_proposer/src/lib.rs index 7b15eb4e9..cf71edd99 100644 --- a/eth2/block_producer/src/lib.rs +++ b/eth2/block_proposer/src/lib.rs @@ -236,7 +236,7 @@ mod tests { epoch_map.map.insert(produce_epoch, produce_slot); let epoch_map = Arc::new(epoch_map); - let mut block_producer = BlockProducer::new( + let mut block_proposer = BlockProducer::new( spec.clone(), epoch_map.clone(), slot_clock.clone(), @@ -251,28 +251,28 @@ mod tests { // One slot before production slot... slot_clock.set_slot(produce_slot.as_u64() - 1); assert_eq!( - block_producer.poll(), + block_proposer.poll(), Ok(PollOutcome::BlockProductionNotRequired(produce_slot - 1)) ); // On the produce slot... slot_clock.set_slot(produce_slot.as_u64()); assert_eq!( - block_producer.poll(), + block_proposer.poll(), Ok(PollOutcome::BlockProduced(produce_slot.into())) ); // Trying the same produce slot again... slot_clock.set_slot(produce_slot.as_u64()); assert_eq!( - block_producer.poll(), + block_proposer.poll(), Ok(PollOutcome::SlotAlreadyProcessed(produce_slot)) ); // One slot after the produce slot... slot_clock.set_slot(produce_slot.as_u64() + 1); assert_eq!( - block_producer.poll(), + block_proposer.poll(), Ok(PollOutcome::BlockProductionNotRequired(produce_slot + 1)) ); @@ -280,7 +280,7 @@ mod tests { let slot = (produce_epoch.as_u64() + 1) * spec.epoch_length; slot_clock.set_slot(slot); assert_eq!( - block_producer.poll(), + block_proposer.poll(), Ok(PollOutcome::ProducerDutiesUnknown(Slot::new(slot))) ); } diff --git a/eth2/block_producer/src/test_utils/epoch_map.rs b/eth2/block_proposer/src/test_utils/epoch_map.rs similarity index 100% rename from eth2/block_producer/src/test_utils/epoch_map.rs rename to eth2/block_proposer/src/test_utils/epoch_map.rs diff --git a/eth2/block_producer/src/test_utils/local_signer.rs b/eth2/block_proposer/src/test_utils/local_signer.rs similarity index 100% rename from eth2/block_producer/src/test_utils/local_signer.rs rename to eth2/block_proposer/src/test_utils/local_signer.rs diff --git a/eth2/block_producer/src/test_utils/mod.rs b/eth2/block_proposer/src/test_utils/mod.rs similarity index 100% rename from eth2/block_producer/src/test_utils/mod.rs rename to eth2/block_proposer/src/test_utils/mod.rs diff --git a/eth2/block_producer/src/test_utils/simulated_beacon_node.rs b/eth2/block_proposer/src/test_utils/simulated_beacon_node.rs similarity index 100% rename from eth2/block_producer/src/test_utils/simulated_beacon_node.rs rename to eth2/block_proposer/src/test_utils/simulated_beacon_node.rs diff --git a/eth2/block_producer/src/traits.rs b/eth2/block_proposer/src/traits.rs similarity index 100% rename from eth2/block_producer/src/traits.rs rename to eth2/block_proposer/src/traits.rs From 7ed606eca126567845eea0c333f1254a8be5a3e0 Mon Sep 17 00:00:00 2001 From: Feng94 <8751189+Feng94@users.noreply.github.com> Date: Tue, 19 Feb 2019 00:21:23 +1100 Subject: [PATCH 19/39] Modify block_producer references in other crates where needed for compilation --- Cargo.toml | 2 +- beacon_node/beacon_chain/Cargo.toml | 2 +- beacon_node/beacon_chain/test_harness/Cargo.toml | 2 +- .../test_harness/src/validator_harness/direct_beacon_node.rs | 2 +- .../test_harness/src/validator_harness/direct_duties.rs | 2 +- .../test_harness/src/validator_harness/local_signer.rs | 2 +- .../beacon_chain/test_harness/src/validator_harness/mod.rs | 4 ++-- validator_client/Cargo.toml | 2 +- .../src/block_producer_service/beacon_block_grpc_client.rs | 2 +- validator_client/src/block_producer_service/mod.rs | 2 +- validator_client/src/duties/epoch_duties.rs | 2 +- validator_client/src/main.rs | 2 +- 12 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5ab0ba847..302c04154 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [workspace] members = [ "eth2/attester", - "eth2/block_producer", + "eth2/block_proposer", "eth2/fork_choice", "eth2/state_processing", "eth2/types", diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index 36d7b3721..4ce894477 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Paul Hauner "] edition = "2018" [dependencies] -block_producer = { path = "../../eth2/block_producer" } +block_proposer = { path = "../../eth2/block_proposer" } bls = { path = "../../eth2/utils/bls" } boolean-bitfield = { path = "../../eth2/utils/boolean-bitfield" } db = { path = "../db" } diff --git a/beacon_node/beacon_chain/test_harness/Cargo.toml b/beacon_node/beacon_chain/test_harness/Cargo.toml index bb335c152..77b52ccf6 100644 --- a/beacon_node/beacon_chain/test_harness/Cargo.toml +++ b/beacon_node/beacon_chain/test_harness/Cargo.toml @@ -14,7 +14,7 @@ criterion = "0.2" [dependencies] attester = { path = "../../../eth2/attester" } beacon_chain = { path = "../../beacon_chain" } -block_producer = { path = "../../../eth2/block_producer" } +block_proposer = { path = "../../../eth2/block_proposer" } bls = { path = "../../../eth2/utils/bls" } boolean-bitfield = { path = "../../../eth2/utils/boolean-bitfield" } db = { path = "../../db" } diff --git a/beacon_node/beacon_chain/test_harness/src/validator_harness/direct_beacon_node.rs b/beacon_node/beacon_chain/test_harness/src/validator_harness/direct_beacon_node.rs index be71b9abd..06d3e7c72 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator_harness/direct_beacon_node.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator_harness/direct_beacon_node.rs @@ -3,7 +3,7 @@ use attester::{ PublishOutcome as AttestationPublishOutcome, }; use beacon_chain::BeaconChain; -use block_producer::{ +use block_proposer::{ BeaconNode as BeaconBlockNode, BeaconNodeError as BeaconBlockNodeError, PublishOutcome as BlockPublishOutcome, }; diff --git a/beacon_node/beacon_chain/test_harness/src/validator_harness/direct_duties.rs b/beacon_node/beacon_chain/test_harness/src/validator_harness/direct_duties.rs index 66b9d650c..5bed59531 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator_harness/direct_duties.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator_harness/direct_duties.rs @@ -2,7 +2,7 @@ use attester::{ DutiesReader as AttesterDutiesReader, DutiesReaderError as AttesterDutiesReaderError, }; use beacon_chain::BeaconChain; -use block_producer::{ +use block_proposer::{ DutiesReader as ProducerDutiesReader, DutiesReaderError as ProducerDutiesReaderError, }; use db::ClientDB; diff --git a/beacon_node/beacon_chain/test_harness/src/validator_harness/local_signer.rs b/beacon_node/beacon_chain/test_harness/src/validator_harness/local_signer.rs index 8e901b057..aa46a1c9a 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator_harness/local_signer.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator_harness/local_signer.rs @@ -1,5 +1,5 @@ use attester::Signer as AttesterSigner; -use block_producer::Signer as BlockProposerSigner; +use block_proposer::Signer as BlockProposerSigner; use std::sync::RwLock; use types::{Keypair, Signature}; diff --git a/beacon_node/beacon_chain/test_harness/src/validator_harness/mod.rs b/beacon_node/beacon_chain/test_harness/src/validator_harness/mod.rs index 3df32fa64..f48309541 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator_harness/mod.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator_harness/mod.rs @@ -5,8 +5,8 @@ mod local_signer; use attester::PollOutcome as AttestationPollOutcome; use attester::{Attester, Error as AttestationPollError}; use beacon_chain::BeaconChain; -use block_producer::PollOutcome as BlockPollOutcome; -use block_producer::{BlockProducer, Error as BlockPollError}; +use block_proposer::PollOutcome as BlockPollOutcome; +use block_proposer::{BlockProducer, Error as BlockPollError}; use db::MemoryDB; use direct_beacon_node::DirectBeaconNode; use direct_duties::DirectDuties; diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index 8ab515e15..f76772f28 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Paul Hauner "] edition = "2018" [dependencies] -block_producer = { path = "../eth2/block_producer" } +block_proposer = { path = "../eth2/block_proposer" } bls = { path = "../eth2/utils/bls" } clap = "2.32.0" dirs = "1.0.3" diff --git a/validator_client/src/block_producer_service/beacon_block_grpc_client.rs b/validator_client/src/block_producer_service/beacon_block_grpc_client.rs index 39ef7fcdc..6bf3005d4 100644 --- a/validator_client/src/block_producer_service/beacon_block_grpc_client.rs +++ b/validator_client/src/block_producer_service/beacon_block_grpc_client.rs @@ -1,4 +1,4 @@ -use block_producer::{BeaconNode, BeaconNodeError, PublishOutcome}; +use block_proposer::{BeaconNode, BeaconNodeError, PublishOutcome}; use protos::services::{ BeaconBlock as GrpcBeaconBlock, ProduceBeaconBlockRequest, PublishBeaconBlockRequest, }; diff --git a/validator_client/src/block_producer_service/mod.rs b/validator_client/src/block_producer_service/mod.rs index 82c3f2537..bd1e691cb 100644 --- a/validator_client/src/block_producer_service/mod.rs +++ b/validator_client/src/block_producer_service/mod.rs @@ -1,7 +1,7 @@ mod beacon_block_grpc_client; // mod block_producer_service; -use block_producer::{ +use block_proposer::{ BeaconNode, BlockProducer, DutiesReader, PollOutcome as BlockProducerPollOutcome, Signer, }; use slog::{error, info, warn, Logger}; diff --git a/validator_client/src/duties/epoch_duties.rs b/validator_client/src/duties/epoch_duties.rs index b555eee28..54a882f8d 100644 --- a/validator_client/src/duties/epoch_duties.rs +++ b/validator_client/src/duties/epoch_duties.rs @@ -1,4 +1,4 @@ -use block_producer::{DutiesReader, DutiesReaderError}; +use block_proposer::{DutiesReader, DutiesReaderError}; use std::collections::HashMap; use std::sync::RwLock; use types::{Epoch, Slot}; diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index 98be9159a..c835300b5 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -1,7 +1,7 @@ use self::block_producer_service::{BeaconBlockGrpcClient, BlockProducerService}; use self::duties::{DutiesManager, DutiesManagerService, EpochDutiesMap}; use crate::config::ClientConfig; -use block_producer::{test_utils::LocalSigner, BlockProducer}; +use block_proposer::{test_utils::LocalSigner, BlockProducer}; use bls::Keypair; use clap::{App, Arg}; use grpcio::{ChannelBuilder, EnvBuilder}; From cd676d5621069050d6ceebef3c562ee46fd18de4 Mon Sep 17 00:00:00 2001 From: mjkeating Date: Mon, 18 Feb 2019 11:55:44 -0800 Subject: [PATCH 20/39] fixed a bug in TreeHash.rs/list_to_blob --- eth2/utils/ssz/src/tree_hash.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/eth2/utils/ssz/src/tree_hash.rs b/eth2/utils/ssz/src/tree_hash.rs index 3f190e32e..bb05f01db 100644 --- a/eth2/utils/ssz/src/tree_hash.rs +++ b/eth2/utils/ssz/src/tree_hash.rs @@ -55,8 +55,13 @@ fn list_to_blob(list: &mut Vec>) -> (usize, Vec) { list[0].len() }; - let items_per_chunk = SSZ_CHUNK_SIZE / list[0].len(); - let chunk_count = list.len() / items_per_chunk; + let (items_per_chunk, chunk_count) = if list.is_empty() { + (1, 1) + } else { + let items_per_chunk = SSZ_CHUNK_SIZE / list[0].len(); + let chunk_count = list.len() / items_per_chunk; + (items_per_chunk, chunk_count) + }; let mut chunkz = Vec::new(); if list.is_empty() { From 8baae0e02ee79bdb79190a4d9a31ea16af27ae64 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 19 Feb 2019 11:58:17 +1100 Subject: [PATCH 21/39] Updates optimised fork choice. - Bug fixes in optimised fork choice. - YAML tests functioning. - Implement Clippy suggestions. - Remove Copywrite notices. --- eth2/fork_choice/Cargo.toml | 2 +- eth2/fork_choice/src/lib.rs | 20 ---- eth2/fork_choice/src/longest_chain.rs | 2 +- eth2/fork_choice/src/optimised_lmd_ghost.rs | 110 ++++++++++-------- eth2/fork_choice/src/slow_lmd_ghost.rs | 30 +---- .../tests/optimised_lmd_ghost_test.rs | 89 ++++++-------- 6 files changed, 104 insertions(+), 149 deletions(-) diff --git a/eth2/fork_choice/Cargo.toml b/eth2/fork_choice/Cargo.toml index 2765622d1..d12f5ae7a 100644 --- a/eth2/fork_choice/Cargo.toml +++ b/eth2/fork_choice/Cargo.toml @@ -9,8 +9,8 @@ db = { path = "../../beacon_node/db" } ssz = { path = "../utils/ssz" } types = { path = "../types" } fast-math = "0.1.1" -byteorder = "1.3.1" log = "0.4.6" +bit-vec = "0.5.0" [dev-dependencies] yaml-rust = "0.4.2" diff --git a/eth2/fork_choice/src/lib.rs b/eth2/fork_choice/src/lib.rs index 11da12304..3923fca9b 100644 --- a/eth2/fork_choice/src/lib.rs +++ b/eth2/fork_choice/src/lib.rs @@ -1,23 +1,3 @@ -// Copyright 2019 Sigma Prime Pty Ltd. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - //! This crate stores the various implementations of fork-choice rules that can be used for the //! beacon blockchain. //! diff --git a/eth2/fork_choice/src/longest_chain.rs b/eth2/fork_choice/src/longest_chain.rs index ea2cc33bb..333553c02 100644 --- a/eth2/fork_choice/src/longest_chain.rs +++ b/eth2/fork_choice/src/longest_chain.rs @@ -30,7 +30,7 @@ impl ForkChoice for LongestChain { &mut self, block: &BeaconBlock, block_hash: &Hash256, - spec: &ChainSpec, + _: &ChainSpec, ) -> Result<(), ForkChoiceError> { // add the block hash to head_block_hashes removing the parent if it exists self.head_block_hashes diff --git a/eth2/fork_choice/src/optimised_lmd_ghost.rs b/eth2/fork_choice/src/optimised_lmd_ghost.rs index e60af4e66..f211359ea 100644 --- a/eth2/fork_choice/src/optimised_lmd_ghost.rs +++ b/eth2/fork_choice/src/optimised_lmd_ghost.rs @@ -1,7 +1,8 @@ -extern crate byteorder; +extern crate bit_vec; extern crate fast_math; + use crate::{ForkChoice, ForkChoiceError}; -use byteorder::{BigEndian, ByteOrder}; +use bit_vec::BitVec; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, ClientDB, @@ -25,6 +26,9 @@ use types::{ // the comparison. Log2_raw takes 2ns according to the documentation. #[inline] fn log2_int(x: u32) -> u32 { + if x == 0 { + return 0; + } log2_raw(x as f32) as u32 } @@ -76,14 +80,13 @@ where pub fn get_latest_votes( &self, state_root: &Hash256, - block_slot: &Slot, + block_slot: Slot, spec: &ChainSpec, ) -> Result, ForkChoiceError> { // get latest votes // Note: Votes are weighted by min(balance, MAX_DEPOSIT_AMOUNT) // // FORK_CHOICE_BALANCE_INCREMENT // build a hashmap of block_hash to weighted votes - trace!("FORKCHOICE: Getting the latest votes"); let mut latest_votes: HashMap = HashMap::new(); // gets the current weighted votes let current_state = self @@ -95,10 +98,6 @@ where ¤t_state.validator_registry[..], block_slot.epoch(spec.epoch_length), ); - trace!( - "FORKCHOICE: Active validator indicies: {:?}", - active_validator_indices - ); for index in active_validator_indices { let balance = std::cmp::min( @@ -119,7 +118,7 @@ where fn get_ancestor( &mut self, block_hash: Hash256, - at_height: SlotHeight, + target_height: SlotHeight, spec: &ChainSpec, ) -> Option { // return None if we can't get the block from the db. @@ -131,32 +130,31 @@ where .expect("Should have returned already if None") .slot; - block_slot.height(Slot::from(spec.genesis_slot)) + block_slot.height(spec.genesis_slot) }; // verify we haven't exceeded the block height - if at_height >= block_height { - if at_height > block_height { + if target_height >= block_height { + if target_height > block_height { return None; } else { return Some(block_hash); } } // check if the result is stored in our cache - let cache_key = CacheKey::new(&block_hash, at_height.as_u32()); + let cache_key = CacheKey::new(&block_hash, target_height.as_u32()); if let Some(ancestor) = self.cache.get(&cache_key) { return Some(*ancestor); } // not in the cache recursively search for ancestors using a log-lookup - if let Some(ancestor) = { let ancestor_lookup = self.ancestors - [log2_int((block_height - at_height - 1u64).as_u32()) as usize] + [log2_int((block_height - target_height - 1u64).as_u32()) as usize] .get(&block_hash) //TODO: Panic if we can't lookup and fork choice fails .expect("All blocks should be added to the ancestor log lookup table"); - self.get_ancestor(*ancestor_lookup, at_height, &spec) + self.get_ancestor(*ancestor_lookup, target_height, &spec) } { // add the result to the cache self.cache.insert(cache_key, ancestor); @@ -177,6 +175,7 @@ where let mut current_votes: HashMap = HashMap::new(); let mut total_vote_count = 0; + trace!("FORKCHOICE: Clear winner at block height: {}", block_height); // loop through the latest votes and count all votes // these have already been weighted by balance for (hash, votes) in latest_votes.iter() { @@ -199,19 +198,30 @@ where // Finds the best child, splitting children into a binary tree, based on their hashes fn choose_best_child(&self, votes: &HashMap) -> Option { - println!("Votes: {:?}", votes); - let mut bitmask = 0; - for bit in (0..=255).rev() { + if votes.is_empty() { + return None; + } + let mut bitmask: BitVec = BitVec::new(); + // loop through bytes then bits + for bit in 0..256 { let mut zero_votes = 0; let mut one_votes = 0; let mut single_candidate = None; for (candidate, votes) in votes.iter() { - let candidate_uint = BigEndian::read_u32(candidate); - if candidate_uint >> (bit + 1) != bitmask { + let candidate_bit: BitVec = BitVec::from_bytes(&candidate); + + // if the bitmasks don't match + if !bitmask.iter().eq(candidate_bit.iter().take(bit)) { + trace!( + "FORKCHOICE: Child: {} was removed in bit: {} with the bitmask: {:?}", + candidate, + bit, + bitmask + ); continue; } - if (candidate_uint >> bit) % 2 == 0 { + if candidate_bit.get(bit) == Some(false) { zero_votes += votes; } else { one_votes += votes; @@ -223,18 +233,10 @@ where single_candidate = None; } } - bitmask = (bitmask * 2) + { - if one_votes > zero_votes { - 1 - } else { - 0 - } - }; + bitmask.push(one_votes > zero_votes); if let Some(candidate) = single_candidate { return Some(*candidate); } - //TODO Remove this during benchmark after testing - assert!(bit >= 1); } // should never reach here None @@ -254,7 +256,7 @@ impl ForkChoice for OptimisedLMDGhost { .get_deserialized(&block.parent_root)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(block.parent_root))? .slot() - .height(Slot::from(spec.genesis_slot)); + .height(spec.genesis_slot); let parent_hash = &block.parent_root; @@ -289,7 +291,7 @@ impl ForkChoice for OptimisedLMDGhost { // simply add the attestation to the latest_attestation_target if the block_height is // larger trace!( - "FORKCHOICE: Adding attestation of validator: {:?} for block: {:?}", + "FORKCHOICE: Adding attestation of validator: {:?} for block: {}", validator_index, target_block_root ); @@ -309,7 +311,7 @@ impl ForkChoice for OptimisedLMDGhost { .get_deserialized(&target_block_root)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*target_block_root))? .slot() - .height(Slot::from(spec.genesis_slot)); + .height(spec.genesis_slot); // get the height of the past target block let past_block_height = self @@ -317,9 +319,7 @@ impl ForkChoice for OptimisedLMDGhost { .get_deserialized(&attestation_target)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*attestation_target))? .slot() - .height(Slot::from(spec.genesis_slot)); - trace!("FORKCHOICE: Old block height: {:?}", past_block_height); - trace!("FORKCHOICE: New block height: {:?}", block_height); + .height(spec.genesis_slot); // update the attestation only if the new target is higher if past_block_height < block_height { trace!("FORKCHOICE: Updating old attestation"); @@ -335,27 +335,34 @@ impl ForkChoice for OptimisedLMDGhost { justified_block_start: &Hash256, spec: &ChainSpec, ) -> Result { - trace!("Starting optimised fork choice"); + debug!( + "Starting optimised fork choice at block: {}", + justified_block_start + ); let block = self .block_store .get_deserialized(&justified_block_start)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*justified_block_start))?; let block_slot = block.slot(); - let block_height = block_slot.height(Slot::from(spec.genesis_slot)); let state_root = block.state_root(); + let mut block_height = block_slot.height(spec.genesis_slot); let mut current_head = *justified_block_start; - let mut latest_votes = self.get_latest_votes(&state_root, &block_slot, spec)?; + let mut latest_votes = self.get_latest_votes(&state_root, block_slot, spec)?; // remove any votes that don't relate to our current head. latest_votes .retain(|hash, _| self.get_ancestor(*hash, block_height, spec) == Some(current_head)); - trace!("FORKCHOICE: Latest votes: {:?}", latest_votes); // begin searching for the head loop { + debug!( + "FORKCHOICE: Iteration for block: {} with vote length: {}", + current_head, + latest_votes.len() + ); // if there are no children, we are done, return the current_head let children = match self.children.get(¤t_head) { Some(children) => children.clone(), @@ -367,6 +374,7 @@ impl ForkChoice for OptimisedLMDGhost { let mut step = power_of_2_below(self.max_known_height.saturating_sub(block_height).as_u32()) / 2; while step > 0 { + trace!("Current Step: {}", step); if let Some(clear_winner) = self.get_clear_winner( &latest_votes, block_height - (block_height % u64::from(step)) + u64::from(step), @@ -399,7 +407,6 @@ impl ForkChoice for OptimisedLMDGhost { *child_votes.entry(child).or_insert_with(|| 0) += vote; } } - println!("Child votes: {:?}", child_votes); // given the votes on the children, find the best child current_head = self .choose_best_child(&child_votes) @@ -407,17 +414,24 @@ impl ForkChoice for OptimisedLMDGhost { trace!("FORKCHOICE: Best child found: {}", current_head); } - // No head was found, re-iterate - // update the block height for the next iteration - let block_height = self + // didn't find head yet, proceed to next iteration + // update block height + block_height = self .block_store .get_deserialized(¤t_head)? - .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*justified_block_start))? + .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(current_head))? .slot() - .height(Slot::from(spec.genesis_slot)); - + .height(spec.genesis_slot); // prune the latest votes for votes that are not part of current chosen chain // more specifically, only keep votes that have head as an ancestor + for hash in latest_votes.keys() { + trace!( + "FORKCHOICE: Ancestor for vote: {} at height: {} is: {:?}", + hash, + block_height, + self.get_ancestor(*hash, block_height, spec) + ); + } latest_votes.retain(|hash, _| { self.get_ancestor(*hash, block_height, spec) == Some(current_head) }); diff --git a/eth2/fork_choice/src/slow_lmd_ghost.rs b/eth2/fork_choice/src/slow_lmd_ghost.rs index 81bc5d628..7041b1b46 100644 --- a/eth2/fork_choice/src/slow_lmd_ghost.rs +++ b/eth2/fork_choice/src/slow_lmd_ghost.rs @@ -1,23 +1,3 @@ -// Copyright 2019 Sigma Prime Pty Ltd. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - extern crate db; use crate::{ForkChoice, ForkChoiceError}; @@ -64,7 +44,7 @@ where pub fn get_latest_votes( &self, state_root: &Hash256, - block_slot: &Slot, + block_slot: Slot, spec: &ChainSpec, ) -> Result, ForkChoiceError> { // get latest votes @@ -122,7 +102,7 @@ where let (root_at_slot, _) = self .block_store .block_at_slot(&block_root, block_slot)? - .ok_or(ForkChoiceError::MissingBeaconBlock(*block_root))?; + .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*block_root))?; if root_at_slot == *target_hash { count += votes; } @@ -171,7 +151,7 @@ impl ForkChoice for SlowLMDGhost { .get_deserialized(&target_block_root)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*target_block_root))? .slot() - .height(Slot::from(spec.genesis_slot)); + .height(spec.genesis_slot); // get the height of the past target block let past_block_height = self @@ -179,7 +159,7 @@ impl ForkChoice for SlowLMDGhost { .get_deserialized(&attestation_target)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*attestation_target))? .slot() - .height(Slot::from(spec.genesis_slot)); + .height(spec.genesis_slot); // update the attestation only if the new target is higher if past_block_height < block_height { *attestation_target = *target_block_root; @@ -201,7 +181,7 @@ impl ForkChoice for SlowLMDGhost { let start_state_root = start.state_root(); - let latest_votes = self.get_latest_votes(&start_state_root, &start.slot(), spec)?; + let latest_votes = self.get_latest_votes(&start_state_root, start.slot(), spec)?; let mut head_hash = Hash256::zero(); diff --git a/eth2/fork_choice/tests/optimised_lmd_ghost_test.rs b/eth2/fork_choice/tests/optimised_lmd_ghost_test.rs index ac0b6888c..3cff5b546 100644 --- a/eth2/fork_choice/tests/optimised_lmd_ghost_test.rs +++ b/eth2/fork_choice/tests/optimised_lmd_ghost_test.rs @@ -15,16 +15,13 @@ use bls::{PublicKey, Signature}; use db::stores::{BeaconBlockStore, BeaconStateStore}; use db::MemoryDB; use env_logger::{Builder, Env}; -use fork_choice::{ForkChoice, ForkChoiceError, OptimisedLMDGhost}; +use fork_choice::{ForkChoice, OptimisedLMDGhost}; use ssz::ssz_encode; use std::collections::HashMap; use std::sync::Arc; -use std::{fs::File, io::prelude::*, path::PathBuf, str::FromStr}; -use types::test_utils::{SeedableRng, TestRandom, XorShiftRng}; -use types::validator_registry::get_active_validator_indices; +use std::{fs::File, io::prelude::*, path::PathBuf}; use types::{ - BeaconBlock, BeaconBlockBody, BeaconState, ChainSpec, Deposit, DepositData, DepositInput, - Eth1Data, Hash256, Slot, Validator, + BeaconBlock, BeaconBlockBody, BeaconState, ChainSpec, Epoch, Eth1Data, Hash256, Slot, Validator, }; use yaml_rust::yaml; @@ -50,25 +47,6 @@ fn setup_inital_state( }; let initial_validator_deposits = vec![]; - - /* - (0..no_validators) - .map(|_| Deposit { - branch: vec![], - index: 0, - deposit_data: DepositData { - amount: 32_000_000_000, // 32 ETH (in Gwei) - timestamp: genesis_time - 1, - deposit_input: DepositInput { - pubkey: PublicKey::default(), - withdrawal_credentials: zero_hash.clone(), - proof_of_possession: Signature::empty_signature(), - }, - }, - }) - .collect(); - */ - let spec = ChainSpec::foundation(); // create the state @@ -80,18 +58,18 @@ fn setup_inital_state( ) .unwrap(); + let default_validator = Validator { + pubkey: PublicKey::default(), + withdrawal_credentials: zero_hash, + activation_epoch: Epoch::from(0u64), + exit_epoch: spec.far_future_epoch, + withdrawal_epoch: spec.far_future_epoch, + penalized_epoch: spec.far_future_epoch, + status_flags: None, + }; // activate the validators for _ in 0..no_validators { - let validator = Validator { - pubkey: PublicKey::default(), - withdrawal_credentials: zero_hash, - activation_epoch: spec.far_future_epoch, - exit_epoch: spec.far_future_epoch, - withdrawal_epoch: spec.far_future_epoch, - penalized_epoch: spec.far_future_epoch, - status_flags: None, - }; - state.validator_registry.push(validator); + state.validator_registry.push(default_validator.clone()); state.validator_balances.push(32_000_000_000); } @@ -100,13 +78,6 @@ fn setup_inital_state( .put(&state_root, &ssz_encode(&state)[..]) .unwrap(); - println!( - "Active: {:?}", - get_active_validator_indices( - &state.validator_registry, - Slot::from(5u64).epoch(spec.EPOCH_LENGTH) - ) - ); // return initialised vars (optimised_lmd_ghost, block_store, state_root) } @@ -115,7 +86,7 @@ fn setup_inital_state( #[test] fn test_optimised_lmd_ghost() { // set up logging - Builder::from_env(Env::default().default_filter_or("trace")).init(); + Builder::from_env(Env::default().default_filter_or("debug")).init(); // load the yaml let mut file = { @@ -141,6 +112,7 @@ fn test_optimised_lmd_ghost() { let mut block_slot: HashMap = HashMap::new(); // default vars + let spec = ChainSpec::foundation(); let zero_hash = Hash256::zero(); let eth1_data = Eth1Data { deposit_root: zero_hash.clone(), @@ -165,7 +137,7 @@ fn test_optimised_lmd_ghost() { // default params for genesis let mut block_hash = zero_hash.clone(); - let mut slot = Slot::from(0u64); + let mut slot = spec.genesis_slot; let mut parent_root = zero_hash; // set the slot and parent based off the YAML. Start with genesis; @@ -206,7 +178,9 @@ fn test_optimised_lmd_ghost() { // run add block for fork choice if not genesis if parent_id != block_id { - fork_choice.add_block(&beacon_block, &block_hash).unwrap(); + fork_choice + .add_block(&beacon_block, &block_hash, &spec) + .unwrap(); } } @@ -225,7 +199,7 @@ fn test_optimised_lmd_ghost() { for _ in 0..weight { assert!(current_validator <= total_emulated_validators); fork_choice - .add_attestation(current_validator as u64, &block_root) + .add_attestation(current_validator as u64, &block_root, &spec) .unwrap(); current_validator += 1; } @@ -233,16 +207,23 @@ fn test_optimised_lmd_ghost() { } // everything is set up, run the fork choice, using genesis as the head - println!("Running fork choice"); - let head = fork_choice.find_head(&zero_hash).unwrap(); + let head = fork_choice.find_head(&zero_hash, &spec).unwrap(); - let mut found_id = None; - for (id, block_hash) in block_id_map.iter() { - if *block_hash == head { - found_id = Some(id); - } - } + let (found_id, _) = block_id_map + .iter() + .find(|(_, hash)| **hash == head) + .unwrap(); + + // compare the result to the expected test + let success = test_case["heads"] + .clone() + .into_vec() + .unwrap() + .iter() + .find(|heads| heads["id"].as_str().unwrap() == found_id) + .is_some(); println!("Head Block ID: {:?}", found_id); + assert!(success, "Did not find one of the possible heads"); } } From fdfaf18dbd0da20a5758c8b7ee80937b0763eebb Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 19 Feb 2019 13:53:05 +1100 Subject: [PATCH 22/39] Add `ssz_derive` crate. It appears to be fully functional at this stage. --- Cargo.toml | 1 + eth2/types/Cargo.toml | 1 + eth2/utils/ssz_derive/Cargo.toml | 14 ++++++ eth2/utils/ssz_derive/src/lib.rs | 80 ++++++++++++++++++++++++++++++++ 4 files changed, 96 insertions(+) create mode 100644 eth2/utils/ssz_derive/Cargo.toml create mode 100644 eth2/utils/ssz_derive/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 5ab0ba847..f8b4cfe41 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ members = [ "eth2/utils/int_to_bytes", "eth2/utils/slot_clock", "eth2/utils/ssz", + "eth2/utils/ssz_derive", "eth2/utils/swap_or_not_shuffle", "eth2/utils/fisher_yates_shuffle", "beacon_node", diff --git a/eth2/types/Cargo.toml b/eth2/types/Cargo.toml index 4c5be65b5..f51e20236 100644 --- a/eth2/types/Cargo.toml +++ b/eth2/types/Cargo.toml @@ -18,6 +18,7 @@ serde_derive = "1.0" serde_json = "1.0" slog = "^2.2.3" ssz = { path = "../utils/ssz" } +ssz_derive = { path = "../utils/ssz_derive" } swap_or_not_shuffle = { path = "../utils/swap_or_not_shuffle" } [dev-dependencies] diff --git a/eth2/utils/ssz_derive/Cargo.toml b/eth2/utils/ssz_derive/Cargo.toml new file mode 100644 index 000000000..3e58d752b --- /dev/null +++ b/eth2/utils/ssz_derive/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "ssz_derive" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" +description = "Procedural derive macros for SSZ encoding and decoding." + +[lib] +proc-macro = true + +[dependencies] +syn = "0.15" +quote = "0.6" +ssz = { path = "../ssz" } diff --git a/eth2/utils/ssz_derive/src/lib.rs b/eth2/utils/ssz_derive/src/lib.rs new file mode 100644 index 000000000..fdee88dd9 --- /dev/null +++ b/eth2/utils/ssz_derive/src/lib.rs @@ -0,0 +1,80 @@ +extern crate proc_macro; + +use proc_macro::TokenStream; +use quote::quote; +use syn::{parse_macro_input, DeriveInput}; + +fn get_named_field_idents<'a>(struct_data: &'a syn::DataStruct) -> Vec<&'a syn::Ident> { + struct_data.fields.iter().map(|f| { + match &f.ident { + Some(ref ident) => ident, + _ => panic!("ssz_derive only supports named struct fields.") + } + }).collect() +} + +#[proc_macro_derive(Encode)] +pub fn ssz_encode_derive(input: TokenStream) -> TokenStream { + let item = parse_macro_input!(input as DeriveInput); + + let name = &item.ident; + + let struct_data = match &item.data { + syn::Data::Struct(s) => s, + _ => panic!("ssz_derive only supports structs.") + }; + + let field_idents = get_named_field_idents(&struct_data); + + let output = quote! { + impl Encodable for #name { + fn ssz_append(&self, s: &mut SszStream) { + #( + s.append(&self.#field_idents); + )* + } + } + }; + output.into() +} + +#[proc_macro_derive(Decode)] +pub fn ssz_decode_derive(input: TokenStream) -> TokenStream { + let item = parse_macro_input!(input as DeriveInput); + + let name = &item.ident; + + let struct_data = match &item.data { + syn::Data::Struct(s) => s, + _ => panic!("ssz_derive only supports structs.") + }; + + let field_idents = get_named_field_idents(&struct_data); + + // Using a var in an iteration always consumes the var, therefore we must make a `fields_a` and + // a `fields_b` in order to perform two loops. + // + // https://github.com/dtolnay/quote/issues/8 + let field_idents_a = &field_idents; + let field_idents_b = &field_idents; + + let output = quote! { + impl Decodable for #name { + fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { + #( + let (#field_idents_a, i) = <_>::ssz_decode(bytes, i)?; + )* + + Ok(( + Self { + #( + #field_idents_b, + )* + }, + i + )) + } + } + }; + output.into() +} From 345c527d3359759ef8eae9f59549c9224d3f1800 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 19 Feb 2019 14:31:09 +1100 Subject: [PATCH 23/39] Add SSZ encode/decode for `bool` --- eth2/utils/ssz/src/impl_decode.rs | 31 +++++++++++++++++++++++++++++++ eth2/utils/ssz/src/impl_encode.rs | 20 ++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/eth2/utils/ssz/src/impl_decode.rs b/eth2/utils/ssz/src/impl_decode.rs index 134e438e1..b9ca48f9b 100644 --- a/eth2/utils/ssz/src/impl_decode.rs +++ b/eth2/utils/ssz/src/impl_decode.rs @@ -39,6 +39,21 @@ impl Decodable for u8 { } } +impl Decodable for bool { + fn ssz_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), DecodeError> { + if index >= bytes.len() { + Err(DecodeError::TooShort) + } else { + let result = match bytes[index] { + 0b0000_0000 => false, + 0b1000_0000 => true, + _ => return Err(DecodeError::Invalid), + }; + Ok((result, index + 1)) + } + } +} + impl Decodable for H256 { fn ssz_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), DecodeError> { if bytes.len() < 32 || bytes.len() - 32 < index { @@ -215,4 +230,20 @@ mod tests { let result: u16 = decode_ssz(&vec![0, 0, 0, 0, 1], 3).unwrap().0; assert_eq!(result, 1); } + + #[test] + fn test_decode_ssz_bool() { + let ssz = vec![0b0000_0000, 0b1000_0000]; + let (result, index): (bool, usize) = decode_ssz(&ssz, 0).unwrap(); + assert_eq!(index, 1); + assert_eq!(result, false); + + let (result, index): (bool, usize) = decode_ssz(&ssz, 1).unwrap(); + assert_eq!(index, 2); + assert_eq!(result, true); + + let ssz = vec![0b0100_0000]; + let result: Result<(bool, usize), DecodeError> = decode_ssz(&ssz, 0); + assert_eq!(result, Err(DecodeError::Invalid)); + } } diff --git a/eth2/utils/ssz/src/impl_encode.rs b/eth2/utils/ssz/src/impl_encode.rs index 8714cf75f..5f73b8483 100644 --- a/eth2/utils/ssz/src/impl_encode.rs +++ b/eth2/utils/ssz/src/impl_encode.rs @@ -46,6 +46,13 @@ impl_encodable_for_uint!(u32, 32); impl_encodable_for_uint!(u64, 64); impl_encodable_for_uint!(usize, 64); +impl Encodable for bool { + fn ssz_append(&self, s: &mut SszStream) { + let byte = if *self { 0b1000_0000 } else { 0b0000_0000 }; + s.append_encoded_raw(&[byte]); + } +} + impl Encodable for H256 { fn ssz_append(&self, s: &mut SszStream) { s.append_encoded_raw(&self.to_vec()); @@ -206,4 +213,17 @@ mod tests { ssz.append(&x); assert_eq!(ssz.drain(), vec![255, 255, 255, 255, 255, 255, 255, 255]); } + + #[test] + fn test_ssz_encode_bool() { + let x: bool = false; + let mut ssz = SszStream::new(); + ssz.append(&x); + assert_eq!(ssz.drain(), vec![0b0000_0000]); + + let x: bool = true; + let mut ssz = SszStream::new(); + ssz.append(&x); + assert_eq!(ssz.drain(), vec![0b1000_0000]); + } } From fc0bf578f82f450bdc01a1173d93d87121990eb7 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 19 Feb 2019 14:32:00 +1100 Subject: [PATCH 24/39] Use SSZ enc/dec proc macros on most `types` Some types were skipped as they have non-standard serialization. --- eth2/types/src/attestation.rs | 29 +----- eth2/types/src/attestation_data.rs | 41 +------- .../src/attestation_data_and_custody_bit.rs | 21 +---- eth2/types/src/attester_slashing.rs | 25 +---- eth2/types/src/beacon_block.rs | 40 +------- eth2/types/src/beacon_block_body.rs | 34 +------ eth2/types/src/beacon_state.rs | 94 +------------------ eth2/types/src/casper_slashing.rs | 25 +---- eth2/types/src/crosslink.rs | 25 +---- eth2/types/src/deposit.rs | 28 +----- eth2/types/src/deposit_data.rs | 28 +----- eth2/types/src/deposit_input.rs | 28 +----- eth2/types/src/eth1_data.rs | 25 +---- eth2/types/src/eth1_data_vote.rs | 25 +---- eth2/types/src/exit.rs | 28 +----- eth2/types/src/fork.rs | 28 +----- eth2/types/src/pending_attestation.rs | 31 +----- eth2/types/src/proposal_signed_data.rs | 28 +----- eth2/types/src/proposer_slashing.rs | 34 +------ eth2/types/src/shard_reassignment_record.rs | 28 +----- eth2/types/src/slashable_attestation.rs | 31 +----- eth2/types/src/slashable_vote_data.rs | 31 +----- .../src/validator_registry_delta_block.rs | 34 +------ 23 files changed, 46 insertions(+), 695 deletions(-) diff --git a/eth2/types/src/attestation.rs b/eth2/types/src/attestation.rs index eb375d490..22bc2c81f 100644 --- a/eth2/types/src/attestation.rs +++ b/eth2/types/src/attestation.rs @@ -3,8 +3,9 @@ use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode}; -#[derive(Debug, Clone, PartialEq, Serialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode)] pub struct Attestation { pub aggregation_bitfield: Bitfield, pub data: AttestationData, @@ -33,32 +34,6 @@ impl Attestation { } } -impl Encodable for Attestation { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.aggregation_bitfield); - s.append(&self.data); - s.append(&self.custody_bitfield); - s.append(&self.aggregate_signature); - } -} - -impl Decodable for Attestation { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (aggregation_bitfield, i) = Bitfield::ssz_decode(bytes, i)?; - let (data, i) = AttestationData::ssz_decode(bytes, i)?; - let (custody_bitfield, i) = Bitfield::ssz_decode(bytes, i)?; - let (aggregate_signature, i) = AggregateSignature::ssz_decode(bytes, i)?; - - let attestation_record = Self { - aggregation_bitfield, - data, - custody_bitfield, - aggregate_signature, - }; - Ok((attestation_record, i)) - } -} - impl TreeHash for Attestation { fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data.rs index 702bba416..1cdcbbb35 100644 --- a/eth2/types/src/attestation_data.rs +++ b/eth2/types/src/attestation_data.rs @@ -3,6 +3,7 @@ use crate::{AttestationDataAndCustodyBit, Crosslink, Epoch, Hash256, Slot}; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode}; pub const SSZ_ATTESTION_DATA_LENGTH: usize = { 8 + // slot @@ -15,7 +16,7 @@ pub const SSZ_ATTESTION_DATA_LENGTH: usize = { 32 // justified_block_root }; -#[derive(Debug, Clone, PartialEq, Default, Serialize, Hash)] +#[derive(Debug, Clone, PartialEq, Default, Serialize, Hash, Encode, Decode)] pub struct AttestationData { pub slot: Slot, pub shard: u64, @@ -43,44 +44,6 @@ impl AttestationData { } } -impl Encodable for AttestationData { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.slot); - s.append(&self.shard); - s.append(&self.beacon_block_root); - s.append(&self.epoch_boundary_root); - s.append(&self.shard_block_root); - s.append(&self.latest_crosslink); - s.append(&self.justified_epoch); - s.append(&self.justified_block_root); - } -} - -impl Decodable for AttestationData { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (slot, i) = <_>::ssz_decode(bytes, i)?; - let (shard, i) = <_>::ssz_decode(bytes, i)?; - let (beacon_block_root, i) = <_>::ssz_decode(bytes, i)?; - let (epoch_boundary_root, i) = <_>::ssz_decode(bytes, i)?; - let (shard_block_root, i) = <_>::ssz_decode(bytes, i)?; - let (latest_crosslink, i) = <_>::ssz_decode(bytes, i)?; - let (justified_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (justified_block_root, i) = <_>::ssz_decode(bytes, i)?; - - let attestation_data = AttestationData { - slot, - shard, - beacon_block_root, - epoch_boundary_root, - shard_block_root, - latest_crosslink, - justified_epoch, - justified_block_root, - }; - Ok((attestation_data, i)) - } -} - impl TreeHash for AttestationData { fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; diff --git a/eth2/types/src/attestation_data_and_custody_bit.rs b/eth2/types/src/attestation_data_and_custody_bit.rs index 4e93dd893..acd68c96e 100644 --- a/eth2/types/src/attestation_data_and_custody_bit.rs +++ b/eth2/types/src/attestation_data_and_custody_bit.rs @@ -3,31 +3,14 @@ use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; use ssz::{Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode}; -#[derive(Debug, Clone, PartialEq, Default, Serialize)] +#[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode)] pub struct AttestationDataAndCustodyBit { pub data: AttestationData, pub custody_bit: bool, } -impl Encodable for AttestationDataAndCustodyBit { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.data); - // TODO: deal with bools - } -} - -impl Decodable for AttestationDataAndCustodyBit { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (data, i) = <_>::ssz_decode(bytes, i)?; - let custody_bit = false; - - let attestation_data_and_custody_bit = AttestationDataAndCustodyBit { data, custody_bit }; - - Ok((attestation_data_and_custody_bit, i)) - } -} - impl TreeHash for AttestationDataAndCustodyBit { fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; diff --git a/eth2/types/src/attester_slashing.rs b/eth2/types/src/attester_slashing.rs index 0b27d2030..6ff5c16db 100644 --- a/eth2/types/src/attester_slashing.rs +++ b/eth2/types/src/attester_slashing.rs @@ -2,35 +2,14 @@ use crate::{test_utils::TestRandom, SlashableAttestation}; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode}; -#[derive(Debug, PartialEq, Clone, Serialize)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] pub struct AttesterSlashing { pub slashable_attestation_1: SlashableAttestation, pub slashable_attestation_2: SlashableAttestation, } -impl Encodable for AttesterSlashing { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.slashable_attestation_1); - s.append(&self.slashable_attestation_2); - } -} - -impl Decodable for AttesterSlashing { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (slashable_attestation_1, i) = <_>::ssz_decode(bytes, i)?; - let (slashable_attestation_2, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - AttesterSlashing { - slashable_attestation_1, - slashable_attestation_2, - }, - i, - )) - } -} - impl TreeHash for AttesterSlashing { fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; diff --git a/eth2/types/src/beacon_block.rs b/eth2/types/src/beacon_block.rs index f6977595a..1dc2f735f 100644 --- a/eth2/types/src/beacon_block.rs +++ b/eth2/types/src/beacon_block.rs @@ -4,8 +4,9 @@ use bls::Signature; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode}; -#[derive(Debug, PartialEq, Clone, Serialize)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] pub struct BeaconBlock { pub slot: Slot, pub parent_root: Hash256, @@ -59,43 +60,6 @@ impl BeaconBlock { } } -impl Encodable for BeaconBlock { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.slot); - s.append(&self.parent_root); - s.append(&self.state_root); - s.append(&self.randao_reveal); - s.append(&self.eth1_data); - s.append(&self.signature); - s.append(&self.body); - } -} - -impl Decodable for BeaconBlock { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (slot, i) = <_>::ssz_decode(bytes, i)?; - let (parent_root, i) = <_>::ssz_decode(bytes, i)?; - let (state_root, i) = <_>::ssz_decode(bytes, i)?; - let (randao_reveal, i) = <_>::ssz_decode(bytes, i)?; - let (eth1_data, i) = <_>::ssz_decode(bytes, i)?; - let (signature, i) = <_>::ssz_decode(bytes, i)?; - let (body, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - Self { - slot, - parent_root, - state_root, - randao_reveal, - eth1_data, - signature, - body, - }, - i, - )) - } -} - impl TreeHash for BeaconBlock { fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; diff --git a/eth2/types/src/beacon_block_body.rs b/eth2/types/src/beacon_block_body.rs index d3a61f7ba..11958cd36 100644 --- a/eth2/types/src/beacon_block_body.rs +++ b/eth2/types/src/beacon_block_body.rs @@ -3,8 +3,9 @@ use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode}; -#[derive(Debug, PartialEq, Clone, Default, Serialize)] +#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode)] pub struct BeaconBlockBody { pub proposer_slashings: Vec, pub attester_slashings: Vec, @@ -13,37 +14,6 @@ pub struct BeaconBlockBody { pub exits: Vec, } -impl Encodable for BeaconBlockBody { - fn ssz_append(&self, s: &mut SszStream) { - s.append_vec(&self.proposer_slashings); - s.append_vec(&self.attester_slashings); - s.append_vec(&self.attestations); - s.append_vec(&self.deposits); - s.append_vec(&self.exits); - } -} - -impl Decodable for BeaconBlockBody { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (proposer_slashings, i) = <_>::ssz_decode(bytes, i)?; - let (attester_slashings, i) = <_>::ssz_decode(bytes, i)?; - let (attestations, i) = <_>::ssz_decode(bytes, i)?; - let (deposits, i) = <_>::ssz_decode(bytes, i)?; - let (exits, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - Self { - proposer_slashings, - attester_slashings, - attestations, - deposits, - exits, - }, - i, - )) - } -} - impl TreeHash for BeaconBlockBody { fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 22885adfe..a59f31141 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -9,6 +9,7 @@ use honey_badger_split::SplitExt; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode}; use swap_or_not_shuffle::get_permutated_index; mod tests; @@ -50,7 +51,7 @@ macro_rules! safe_sub_assign { }; } -#[derive(Debug, PartialEq, Clone, Default, Serialize)] +#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode)] pub struct BeaconState { // Misc pub slot: Slot, @@ -928,97 +929,6 @@ impl From for InclusionError { } } -impl Encodable for BeaconState { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.slot); - s.append(&self.genesis_time); - s.append(&self.fork); - s.append(&self.validator_registry); - s.append(&self.validator_balances); - s.append(&self.validator_registry_update_epoch); - s.append(&self.latest_randao_mixes); - s.append(&self.previous_epoch_start_shard); - s.append(&self.current_epoch_start_shard); - s.append(&self.previous_calculation_epoch); - s.append(&self.current_calculation_epoch); - s.append(&self.previous_epoch_seed); - s.append(&self.current_epoch_seed); - s.append(&self.previous_justified_epoch); - s.append(&self.justified_epoch); - s.append(&self.justification_bitfield); - s.append(&self.finalized_epoch); - s.append(&self.latest_crosslinks); - s.append(&self.latest_block_roots); - s.append(&self.latest_index_roots); - s.append(&self.latest_penalized_balances); - s.append(&self.latest_attestations); - s.append(&self.batched_block_roots); - s.append(&self.latest_eth1_data); - s.append(&self.eth1_data_votes); - } -} - -impl Decodable for BeaconState { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (slot, i) = <_>::ssz_decode(bytes, i)?; - let (genesis_time, i) = <_>::ssz_decode(bytes, i)?; - let (fork, i) = <_>::ssz_decode(bytes, i)?; - let (validator_registry, i) = <_>::ssz_decode(bytes, i)?; - let (validator_balances, i) = <_>::ssz_decode(bytes, i)?; - let (validator_registry_update_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (latest_randao_mixes, i) = <_>::ssz_decode(bytes, i)?; - let (previous_epoch_start_shard, i) = <_>::ssz_decode(bytes, i)?; - let (current_epoch_start_shard, i) = <_>::ssz_decode(bytes, i)?; - let (previous_calculation_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (current_calculation_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (previous_epoch_seed, i) = <_>::ssz_decode(bytes, i)?; - let (current_epoch_seed, i) = <_>::ssz_decode(bytes, i)?; - let (previous_justified_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (justified_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (justification_bitfield, i) = <_>::ssz_decode(bytes, i)?; - let (finalized_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (latest_crosslinks, i) = <_>::ssz_decode(bytes, i)?; - let (latest_block_roots, i) = <_>::ssz_decode(bytes, i)?; - let (latest_index_roots, i) = <_>::ssz_decode(bytes, i)?; - let (latest_penalized_balances, i) = <_>::ssz_decode(bytes, i)?; - let (latest_attestations, i) = <_>::ssz_decode(bytes, i)?; - let (batched_block_roots, i) = <_>::ssz_decode(bytes, i)?; - let (latest_eth1_data, i) = <_>::ssz_decode(bytes, i)?; - let (eth1_data_votes, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - Self { - slot, - genesis_time, - fork, - validator_registry, - validator_balances, - validator_registry_update_epoch, - latest_randao_mixes, - previous_epoch_start_shard, - current_epoch_start_shard, - previous_calculation_epoch, - current_calculation_epoch, - previous_epoch_seed, - current_epoch_seed, - previous_justified_epoch, - justified_epoch, - justification_bitfield, - finalized_epoch, - latest_crosslinks, - latest_block_roots, - latest_index_roots, - latest_penalized_balances, - latest_attestations, - batched_block_roots, - latest_eth1_data, - eth1_data_votes, - }, - i, - )) - } -} - impl TreeHash for BeaconState { fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; diff --git a/eth2/types/src/casper_slashing.rs b/eth2/types/src/casper_slashing.rs index 0eab069b4..c4b875402 100644 --- a/eth2/types/src/casper_slashing.rs +++ b/eth2/types/src/casper_slashing.rs @@ -3,35 +3,14 @@ use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode}; -#[derive(Debug, PartialEq, Clone, Serialize)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] pub struct CasperSlashing { pub slashable_vote_data_1: SlashableVoteData, pub slashable_vote_data_2: SlashableVoteData, } -impl Encodable for CasperSlashing { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.slashable_vote_data_1); - s.append(&self.slashable_vote_data_2); - } -} - -impl Decodable for CasperSlashing { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (slashable_vote_data_1, i) = <_>::ssz_decode(bytes, i)?; - let (slashable_vote_data_2, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - CasperSlashing { - slashable_vote_data_1, - slashable_vote_data_2, - }, - i, - )) - } -} - impl TreeHash for CasperSlashing { fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; diff --git a/eth2/types/src/crosslink.rs b/eth2/types/src/crosslink.rs index 3cb857ef4..9e527f47a 100644 --- a/eth2/types/src/crosslink.rs +++ b/eth2/types/src/crosslink.rs @@ -3,8 +3,9 @@ use crate::{Epoch, Hash256}; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode}; -#[derive(Debug, Clone, PartialEq, Default, Serialize, Hash)] +#[derive(Debug, Clone, PartialEq, Default, Serialize, Hash, Encode, Decode)] pub struct Crosslink { pub epoch: Epoch, pub shard_block_root: Hash256, @@ -20,28 +21,6 @@ impl Crosslink { } } -impl Encodable for Crosslink { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.epoch); - s.append(&self.shard_block_root); - } -} - -impl Decodable for Crosslink { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (epoch, i) = <_>::ssz_decode(bytes, i)?; - let (shard_block_root, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - Self { - epoch, - shard_block_root, - }, - i, - )) - } -} - impl TreeHash for Crosslink { fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; diff --git a/eth2/types/src/deposit.rs b/eth2/types/src/deposit.rs index 62349cbc1..6eae0def9 100644 --- a/eth2/types/src/deposit.rs +++ b/eth2/types/src/deposit.rs @@ -3,39 +3,15 @@ use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode}; -#[derive(Debug, PartialEq, Clone, Serialize)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] pub struct Deposit { pub branch: Vec, pub index: u64, pub deposit_data: DepositData, } -impl Encodable for Deposit { - fn ssz_append(&self, s: &mut SszStream) { - s.append_vec(&self.branch); - s.append(&self.index); - s.append(&self.deposit_data); - } -} - -impl Decodable for Deposit { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (branch, i) = <_>::ssz_decode(bytes, i)?; - let (index, i) = <_>::ssz_decode(bytes, i)?; - let (deposit_data, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - Self { - branch, - index, - deposit_data, - }, - i, - )) - } -} - impl TreeHash for Deposit { fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; diff --git a/eth2/types/src/deposit_data.rs b/eth2/types/src/deposit_data.rs index 5c8c302f4..0b0548487 100644 --- a/eth2/types/src/deposit_data.rs +++ b/eth2/types/src/deposit_data.rs @@ -3,39 +3,15 @@ use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode}; -#[derive(Debug, PartialEq, Clone, Serialize)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] pub struct DepositData { pub amount: u64, pub timestamp: u64, pub deposit_input: DepositInput, } -impl Encodable for DepositData { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.amount); - s.append(&self.timestamp); - s.append(&self.deposit_input); - } -} - -impl Decodable for DepositData { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (amount, i) = <_>::ssz_decode(bytes, i)?; - let (timestamp, i) = <_>::ssz_decode(bytes, i)?; - let (deposit_input, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - Self { - amount, - timestamp, - deposit_input, - }, - i, - )) - } -} - impl TreeHash for DepositData { fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; diff --git a/eth2/types/src/deposit_input.rs b/eth2/types/src/deposit_input.rs index fc53baae9..6872d19cc 100644 --- a/eth2/types/src/deposit_input.rs +++ b/eth2/types/src/deposit_input.rs @@ -4,39 +4,15 @@ use bls::{PublicKey, Signature}; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode}; -#[derive(Debug, PartialEq, Clone, Serialize)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] pub struct DepositInput { pub pubkey: PublicKey, pub withdrawal_credentials: Hash256, pub proof_of_possession: Signature, } -impl Encodable for DepositInput { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.pubkey); - s.append(&self.withdrawal_credentials); - s.append(&self.proof_of_possession); - } -} - -impl Decodable for DepositInput { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (pubkey, i) = <_>::ssz_decode(bytes, i)?; - let (withdrawal_credentials, i) = <_>::ssz_decode(bytes, i)?; - let (proof_of_possession, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - Self { - pubkey, - withdrawal_credentials, - proof_of_possession, - }, - i, - )) - } -} - impl TreeHash for DepositInput { fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; diff --git a/eth2/types/src/eth1_data.rs b/eth2/types/src/eth1_data.rs index 6e9bb7d26..f3c788a31 100644 --- a/eth2/types/src/eth1_data.rs +++ b/eth2/types/src/eth1_data.rs @@ -3,36 +3,15 @@ use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode}; // Note: this is refer to as DepositRootVote in specs -#[derive(Debug, PartialEq, Clone, Default, Serialize)] +#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode)] pub struct Eth1Data { pub deposit_root: Hash256, pub block_hash: Hash256, } -impl Encodable for Eth1Data { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.deposit_root); - s.append(&self.block_hash); - } -} - -impl Decodable for Eth1Data { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (deposit_root, i) = <_>::ssz_decode(bytes, i)?; - let (block_hash, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - Self { - deposit_root, - block_hash, - }, - i, - )) - } -} - impl TreeHash for Eth1Data { fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; diff --git a/eth2/types/src/eth1_data_vote.rs b/eth2/types/src/eth1_data_vote.rs index 2bfee4d02..82cb9d0f6 100644 --- a/eth2/types/src/eth1_data_vote.rs +++ b/eth2/types/src/eth1_data_vote.rs @@ -3,36 +3,15 @@ use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode}; // Note: this is refer to as DepositRootVote in specs -#[derive(Debug, PartialEq, Clone, Default, Serialize)] +#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode)] pub struct Eth1DataVote { pub eth1_data: Eth1Data, pub vote_count: u64, } -impl Encodable for Eth1DataVote { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.eth1_data); - s.append(&self.vote_count); - } -} - -impl Decodable for Eth1DataVote { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (eth1_data, i) = <_>::ssz_decode(bytes, i)?; - let (vote_count, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - Self { - eth1_data, - vote_count, - }, - i, - )) - } -} - impl TreeHash for Eth1DataVote { fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; diff --git a/eth2/types/src/exit.rs b/eth2/types/src/exit.rs index cd7746919..a1af44fdd 100644 --- a/eth2/types/src/exit.rs +++ b/eth2/types/src/exit.rs @@ -3,39 +3,15 @@ use bls::Signature; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode}; -#[derive(Debug, PartialEq, Clone, Serialize)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] pub struct Exit { pub epoch: Epoch, pub validator_index: u64, pub signature: Signature, } -impl Encodable for Exit { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.epoch); - s.append(&self.validator_index); - s.append(&self.signature); - } -} - -impl Decodable for Exit { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (epoch, i) = <_>::ssz_decode(bytes, i)?; - let (validator_index, i) = <_>::ssz_decode(bytes, i)?; - let (signature, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - Self { - epoch, - validator_index, - signature, - }, - i, - )) - } -} - impl TreeHash for Exit { fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs index 1c96a34ac..f59cd1b79 100644 --- a/eth2/types/src/fork.rs +++ b/eth2/types/src/fork.rs @@ -2,39 +2,15 @@ use crate::{test_utils::TestRandom, Epoch}; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode}; -#[derive(Debug, Clone, PartialEq, Default, Serialize)] +#[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode)] pub struct Fork { pub previous_version: u64, pub current_version: u64, pub epoch: Epoch, } -impl Encodable for Fork { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.previous_version); - s.append(&self.current_version); - s.append(&self.epoch); - } -} - -impl Decodable for Fork { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (previous_version, i) = <_>::ssz_decode(bytes, i)?; - let (current_version, i) = <_>::ssz_decode(bytes, i)?; - let (epoch, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - Self { - previous_version, - current_version, - epoch, - }, - i, - )) - } -} - impl TreeHash for Fork { fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; diff --git a/eth2/types/src/pending_attestation.rs b/eth2/types/src/pending_attestation.rs index 25ec109d7..06ad767e0 100644 --- a/eth2/types/src/pending_attestation.rs +++ b/eth2/types/src/pending_attestation.rs @@ -3,8 +3,9 @@ use crate::{AttestationData, Bitfield, Slot}; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode}; -#[derive(Debug, Clone, PartialEq, Serialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode)] pub struct PendingAttestation { pub aggregation_bitfield: Bitfield, pub data: AttestationData, @@ -12,34 +13,6 @@ pub struct PendingAttestation { pub inclusion_slot: Slot, } -impl Encodable for PendingAttestation { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.aggregation_bitfield); - s.append(&self.data); - s.append(&self.custody_bitfield); - s.append(&self.inclusion_slot); - } -} - -impl Decodable for PendingAttestation { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (aggregation_bitfield, i) = <_>::ssz_decode(bytes, i)?; - let (data, i) = <_>::ssz_decode(bytes, i)?; - let (custody_bitfield, i) = <_>::ssz_decode(bytes, i)?; - let (inclusion_slot, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - Self { - data, - aggregation_bitfield, - custody_bitfield, - inclusion_slot, - }, - i, - )) - } -} - impl TreeHash for PendingAttestation { fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; diff --git a/eth2/types/src/proposal_signed_data.rs b/eth2/types/src/proposal_signed_data.rs index c57eb1e2a..ea7fa56a0 100644 --- a/eth2/types/src/proposal_signed_data.rs +++ b/eth2/types/src/proposal_signed_data.rs @@ -3,39 +3,15 @@ use crate::{Hash256, Slot}; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode}; -#[derive(Debug, PartialEq, Clone, Default, Serialize)] +#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode)] pub struct ProposalSignedData { pub slot: Slot, pub shard: u64, pub block_root: Hash256, } -impl Encodable for ProposalSignedData { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.slot); - s.append(&self.shard); - s.append(&self.block_root); - } -} - -impl Decodable for ProposalSignedData { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (slot, i) = <_>::ssz_decode(bytes, i)?; - let (shard, i) = <_>::ssz_decode(bytes, i)?; - let (block_root, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - ProposalSignedData { - slot, - shard, - block_root, - }, - i, - )) - } -} - impl TreeHash for ProposalSignedData { fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; diff --git a/eth2/types/src/proposer_slashing.rs b/eth2/types/src/proposer_slashing.rs index 417d23dbc..1c25aabcf 100644 --- a/eth2/types/src/proposer_slashing.rs +++ b/eth2/types/src/proposer_slashing.rs @@ -4,8 +4,9 @@ use bls::Signature; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode}; -#[derive(Debug, PartialEq, Clone, Serialize)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] pub struct ProposerSlashing { pub proposer_index: u64, pub proposal_data_1: ProposalSignedData, @@ -14,37 +15,6 @@ pub struct ProposerSlashing { pub proposal_signature_2: Signature, } -impl Encodable for ProposerSlashing { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.proposer_index); - s.append(&self.proposal_data_1); - s.append(&self.proposal_signature_1); - s.append(&self.proposal_data_2); - s.append(&self.proposal_signature_2); - } -} - -impl Decodable for ProposerSlashing { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (proposer_index, i) = <_>::ssz_decode(bytes, i)?; - let (proposal_data_1, i) = <_>::ssz_decode(bytes, i)?; - let (proposal_signature_1, i) = <_>::ssz_decode(bytes, i)?; - let (proposal_data_2, i) = <_>::ssz_decode(bytes, i)?; - let (proposal_signature_2, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - ProposerSlashing { - proposer_index, - proposal_data_1, - proposal_signature_1, - proposal_data_2, - proposal_signature_2, - }, - i, - )) - } -} - impl TreeHash for ProposerSlashing { fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; diff --git a/eth2/types/src/shard_reassignment_record.rs b/eth2/types/src/shard_reassignment_record.rs index 61f68ac05..28d10b321 100644 --- a/eth2/types/src/shard_reassignment_record.rs +++ b/eth2/types/src/shard_reassignment_record.rs @@ -2,39 +2,15 @@ use crate::{test_utils::TestRandom, Slot}; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode}; -#[derive(Debug, PartialEq, Clone, Serialize)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] pub struct ShardReassignmentRecord { pub validator_index: u64, pub shard: u64, pub slot: Slot, } -impl Encodable for ShardReassignmentRecord { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.validator_index); - s.append(&self.shard); - s.append(&self.slot); - } -} - -impl Decodable for ShardReassignmentRecord { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (validator_index, i) = <_>::ssz_decode(bytes, i)?; - let (shard, i) = <_>::ssz_decode(bytes, i)?; - let (slot, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - Self { - validator_index, - shard, - slot, - }, - i, - )) - } -} - impl TreeHash for ShardReassignmentRecord { fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; diff --git a/eth2/types/src/slashable_attestation.rs b/eth2/types/src/slashable_attestation.rs index 6d83ad147..6080b664f 100644 --- a/eth2/types/src/slashable_attestation.rs +++ b/eth2/types/src/slashable_attestation.rs @@ -2,8 +2,9 @@ use crate::{test_utils::TestRandom, AggregateSignature, AttestationData, Bitfiel use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode}; -#[derive(Debug, PartialEq, Clone, Serialize)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] pub struct SlashableAttestation { pub validator_indices: Vec, pub data: AttestationData, @@ -11,34 +12,6 @@ pub struct SlashableAttestation { pub aggregate_signature: AggregateSignature, } -impl Encodable for SlashableAttestation { - fn ssz_append(&self, s: &mut SszStream) { - s.append_vec(&self.validator_indices); - s.append(&self.data); - s.append(&self.custody_bitfield); - s.append(&self.aggregate_signature); - } -} - -impl Decodable for SlashableAttestation { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (validator_indices, i) = <_>::ssz_decode(bytes, i)?; - let (data, i) = <_>::ssz_decode(bytes, i)?; - let (custody_bitfield, i) = <_>::ssz_decode(bytes, i)?; - let (aggregate_signature, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - SlashableAttestation { - validator_indices, - data, - custody_bitfield, - aggregate_signature, - }, - i, - )) - } -} - impl TreeHash for SlashableAttestation { fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; diff --git a/eth2/types/src/slashable_vote_data.rs b/eth2/types/src/slashable_vote_data.rs index acffca26d..f0a10745a 100644 --- a/eth2/types/src/slashable_vote_data.rs +++ b/eth2/types/src/slashable_vote_data.rs @@ -4,8 +4,9 @@ use bls::AggregateSignature; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode}; -#[derive(Debug, PartialEq, Clone, Serialize)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] pub struct SlashableVoteData { pub custody_bit_0_indices: Vec, pub custody_bit_1_indices: Vec, @@ -13,34 +14,6 @@ pub struct SlashableVoteData { pub aggregate_signature: AggregateSignature, } -impl Encodable for SlashableVoteData { - fn ssz_append(&self, s: &mut SszStream) { - s.append_vec(&self.custody_bit_0_indices); - s.append_vec(&self.custody_bit_1_indices); - s.append(&self.data); - s.append(&self.aggregate_signature); - } -} - -impl Decodable for SlashableVoteData { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (custody_bit_0_indices, i) = <_>::ssz_decode(bytes, i)?; - let (custody_bit_1_indices, i) = <_>::ssz_decode(bytes, i)?; - let (data, i) = <_>::ssz_decode(bytes, i)?; - let (aggregate_signature, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - SlashableVoteData { - custody_bit_0_indices, - custody_bit_1_indices, - data, - aggregate_signature, - }, - i, - )) - } -} - impl TreeHash for SlashableVoteData { fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; diff --git a/eth2/types/src/validator_registry_delta_block.rs b/eth2/types/src/validator_registry_delta_block.rs index 3142e0263..a142d64a2 100644 --- a/eth2/types/src/validator_registry_delta_block.rs +++ b/eth2/types/src/validator_registry_delta_block.rs @@ -3,9 +3,10 @@ use bls::PublicKey; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode}; // The information gathered from the PoW chain validator registration function. -#[derive(Debug, Clone, PartialEq, Serialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode)] pub struct ValidatorRegistryDeltaBlock { pub latest_registry_delta_root: Hash256, pub validator_index: u32, @@ -27,37 +28,6 @@ impl Default for ValidatorRegistryDeltaBlock { } } -impl Encodable for ValidatorRegistryDeltaBlock { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.latest_registry_delta_root); - s.append(&self.validator_index); - s.append(&self.pubkey); - s.append(&self.slot); - s.append(&self.flag); - } -} - -impl Decodable for ValidatorRegistryDeltaBlock { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (latest_registry_delta_root, i) = <_>::ssz_decode(bytes, i)?; - let (validator_index, i) = <_>::ssz_decode(bytes, i)?; - let (pubkey, i) = <_>::ssz_decode(bytes, i)?; - let (slot, i) = <_>::ssz_decode(bytes, i)?; - let (flag, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - Self { - latest_registry_delta_root, - validator_index, - pubkey, - slot, - flag, - }, - i, - )) - } -} - impl TreeHash for ValidatorRegistryDeltaBlock { fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; From 846cbdd7f7debf49713f221ffb0b792a41a3c9d6 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 19 Feb 2019 14:37:17 +1100 Subject: [PATCH 25/39] Generalise fork choice tests. --- beacon_node/beacon_chain/src/lib.rs | 2 +- eth2/fork_choice/src/lib.rs | 13 +- eth2/fork_choice/src/slow_lmd_ghost.rs | 9 +- ... => optimised_lmd_ghost_test_vectors.yaml} | 0 .../{optimised_lmd_ghost_test.rs => tests.rs} | 176 ++++++++++-------- 5 files changed, 115 insertions(+), 85 deletions(-) rename eth2/fork_choice/tests/{fork_choice_test_vectors.yaml => optimised_lmd_ghost_test_vectors.yaml} (100%) rename eth2/fork_choice/tests/{optimised_lmd_ghost_test.rs => tests.rs} (82%) diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index bc9085fbe..fdacdd2b1 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -5,4 +5,4 @@ mod checkpoint; pub use self::beacon_chain::{BeaconChain, Error}; pub use self::checkpoint::CheckPoint; -pub use fork_choice::{ForkChoice, ForkChoiceAlgorithms, ForkChoiceError}; +pub use fork_choice::{ForkChoice, ForkChoiceAlgorithm, ForkChoiceError}; diff --git a/eth2/fork_choice/src/lib.rs b/eth2/fork_choice/src/lib.rs index 3923fca9b..90fb0e82b 100644 --- a/eth2/fork_choice/src/lib.rs +++ b/eth2/fork_choice/src/lib.rs @@ -1,22 +1,20 @@ //! This crate stores the various implementations of fork-choice rules that can be used for the //! beacon blockchain. //! -//! There are four implementations. One is the naive longest chain rule (primarily for testing -//! purposes). The other three are proposed implementations of the LMD-GHOST fork-choice rule with various forms of optimisation. +//! There are three implementations. One is the naive longest chain rule (primarily for testing +//! purposes). The other two are proposed implementations of the LMD-GHOST fork-choice rule with various forms of optimisation. //! //! The current implementations are: //! - [`longest-chain`]: Simplistic longest-chain fork choice - primarily for testing, **not for //! production**. //! - [`slow_lmd_ghost`]: This is a simple and very inefficient implementation given in the ethereum 2.0 //! specifications (https://github.com/ethereum/eth2.0-specs/blob/v0.1/specs/core/0_beacon-chain.md#get_block_root). -//! - [`optimised_lmd_ghost`]: This is an optimised version of the naive implementation as proposed +//! - [`optimised_lmd_ghost`]: This is an optimised version of bitwise LMD-GHOST as proposed //! by Vitalik. The reference implementation can be found at: https://github.com/ethereum/research/blob/master/ghost/ghost.py -//! - [`protolambda_lmd_ghost`]: Another optimised version of LMD-GHOST designed by @protolambda. -//! The go implementation can be found here: https://github.com/protolambda/lmd-ghost. //! +//! [`longest-chain`]: struct.LongestChain.html //! [`slow_lmd_ghost`]: struct.SlowLmdGhost.html //! [`optimised_lmd_ghost`]: struct.OptimisedLmdGhost.html -//! [`protolambda_lmd_ghost`]: struct.ProtolambdaLmdGhost.html extern crate db; extern crate ssz; @@ -34,6 +32,7 @@ use types::{BeaconBlock, ChainSpec, Hash256}; pub use longest_chain::LongestChain; pub use optimised_lmd_ghost::OptimisedLMDGhost; +pub use slow_lmd_ghost::SlowLMDGhost; /// Defines the interface for Fork Choices. Each Fork choice will define their own data structures /// which can be built in block processing through the `add_block` and `add_attestation` functions. @@ -97,7 +96,7 @@ impl From for ForkChoiceError { } /// Fork choice options that are currently implemented. -pub enum ForkChoiceAlgorithms { +pub enum ForkChoiceAlgorithm { /// Chooses the longest chain becomes the head. Not for production. LongestChain, /// A simple and highly inefficient implementation of LMD ghost. diff --git a/eth2/fork_choice/src/slow_lmd_ghost.rs b/eth2/fork_choice/src/slow_lmd_ghost.rs index 7041b1b46..3412f5473 100644 --- a/eth2/fork_choice/src/slow_lmd_ghost.rs +++ b/eth2/fork_choice/src/slow_lmd_ghost.rs @@ -30,12 +30,15 @@ impl SlowLMDGhost where T: ClientDB + Sized, { - pub fn new(block_store: BeaconBlockStore, state_store: BeaconStateStore) -> Self { + pub fn new( + block_store: Arc>, + state_store: Arc>, + ) -> Self { SlowLMDGhost { latest_attestation_targets: HashMap::new(), children: HashMap::new(), - block_store: Arc::new(block_store), - state_store: Arc::new(state_store), + block_store, + state_store, } } diff --git a/eth2/fork_choice/tests/fork_choice_test_vectors.yaml b/eth2/fork_choice/tests/optimised_lmd_ghost_test_vectors.yaml similarity index 100% rename from eth2/fork_choice/tests/fork_choice_test_vectors.yaml rename to eth2/fork_choice/tests/optimised_lmd_ghost_test_vectors.yaml diff --git a/eth2/fork_choice/tests/optimised_lmd_ghost_test.rs b/eth2/fork_choice/tests/tests.rs similarity index 82% rename from eth2/fork_choice/tests/optimised_lmd_ghost_test.rs rename to eth2/fork_choice/tests/tests.rs index 3cff5b546..83baa8bd8 100644 --- a/eth2/fork_choice/tests/optimised_lmd_ghost_test.rs +++ b/eth2/fork_choice/tests/tests.rs @@ -15,7 +15,7 @@ use bls::{PublicKey, Signature}; use db::stores::{BeaconBlockStore, BeaconStateStore}; use db::MemoryDB; use env_logger::{Builder, Env}; -use fork_choice::{ForkChoice, OptimisedLMDGhost}; +use fork_choice::{ForkChoice, ForkChoiceAlgorithm, LongestChain, OptimisedLMDGhost, SlowLMDGhost}; use ssz::ssz_encode; use std::collections::HashMap; use std::sync::Arc; @@ -25,86 +25,32 @@ use types::{ }; use yaml_rust::yaml; -// initialise a single validator and state. All blocks will reference this state root. -fn setup_inital_state( - no_validators: usize, -) -> (impl ForkChoice, Arc>, Hash256) { - let zero_hash = Hash256::zero(); - - let db = Arc::new(MemoryDB::open()); - let block_store = Arc::new(BeaconBlockStore::new(db.clone())); - let state_store = Arc::new(BeaconStateStore::new(db.clone())); - - // the fork choice instantiation - let optimised_lmd_ghost = OptimisedLMDGhost::new(block_store.clone(), state_store.clone()); - - // misc vars for setting up the state - let genesis_time = 1_550_381_159; - - let latest_eth1_data = Eth1Data { - deposit_root: zero_hash.clone(), - block_hash: zero_hash.clone(), - }; - - let initial_validator_deposits = vec![]; - let spec = ChainSpec::foundation(); - - // create the state - let mut state = BeaconState::genesis( - genesis_time, - initial_validator_deposits, - latest_eth1_data, - &spec, - ) - .unwrap(); - - let default_validator = Validator { - pubkey: PublicKey::default(), - withdrawal_credentials: zero_hash, - activation_epoch: Epoch::from(0u64), - exit_epoch: spec.far_future_epoch, - withdrawal_epoch: spec.far_future_epoch, - penalized_epoch: spec.far_future_epoch, - status_flags: None, - }; - // activate the validators - for _ in 0..no_validators { - state.validator_registry.push(default_validator.clone()); - state.validator_balances.push(32_000_000_000); - } - - let state_root = state.canonical_root(); - state_store - .put(&state_root, &ssz_encode(&state)[..]) - .unwrap(); - - // return initialised vars - (optimised_lmd_ghost, block_store, state_root) -} - -// YAML test vectors +// run tests #[test] fn test_optimised_lmd_ghost() { + test_yaml_vectors( + ForkChoiceAlgorithm::OptimisedLMDGhost, + "tests/optimised_lmd_ghost_test_vectors.yaml", + 100, + ); +} + +// run a generic test over given YAML test vectors +fn test_yaml_vectors( + fork_choice_algo: ForkChoiceAlgorithm, + yaml_file_path: &str, + max_validators: usize, +) { // set up logging Builder::from_env(Env::default().default_filter_or("debug")).init(); - // load the yaml - let mut file = { - let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - file_path_buf.push("tests/fork_choice_test_vectors.yaml"); - - File::open(file_path_buf).unwrap() - }; - - let mut yaml_str = String::new(); - file.read_to_string(&mut yaml_str).unwrap(); - let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap(); - let doc = &docs[0]; - let test_cases = doc["test_cases"].as_vec().unwrap(); + // load test cases from yaml + let test_cases = load_test_cases_from_yaml(yaml_file_path); // set up the test - let total_emulated_validators = 20; // the number of validators used to give weights. - let (mut fork_choice, block_store, state_root) = setup_inital_state(total_emulated_validators); + let total_emulated_validators = max_validators; // the number of validators used to give weights. + let (mut fork_choice, block_store, state_root) = + setup_inital_state(fork_choice_algo, total_emulated_validators); // keep a hashmap of block_id's to block_hashes (random hashes to abstract block_id) let mut block_id_map: HashMap = HashMap::new(); @@ -227,3 +173,85 @@ fn test_optimised_lmd_ghost() { assert!(success, "Did not find one of the possible heads"); } } + +// loads the test_cases from the supplied yaml file +fn load_test_cases_from_yaml(file_path: &str) -> Vec { + // load the yaml + let mut file = { + let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + file_path_buf.push(file_path); + File::open(file_path_buf).unwrap() + }; + let mut yaml_str = String::new(); + file.read_to_string(&mut yaml_str).unwrap(); + let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap(); + let doc = &docs[0]; + doc["test_cases"].as_vec().unwrap().clone() +} + +// initialise a single validator and state. All blocks will reference this state root. +fn setup_inital_state( + fork_choice_algo: ForkChoiceAlgorithm, + no_validators: usize, +) -> (Box, Arc>, Hash256) { + let zero_hash = Hash256::zero(); + + let db = Arc::new(MemoryDB::open()); + let block_store = Arc::new(BeaconBlockStore::new(db.clone())); + let state_store = Arc::new(BeaconStateStore::new(db.clone())); + + // the fork choice instantiation + let fork_choice: Box = match fork_choice_algo { + ForkChoiceAlgorithm::OptimisedLMDGhost => Box::new(OptimisedLMDGhost::new( + block_store.clone(), + state_store.clone(), + )), + ForkChoiceAlgorithm::SlowLMDGhost => { + Box::new(SlowLMDGhost::new(block_store.clone(), state_store.clone())) + } + ForkChoiceAlgorithm::LongestChain => Box::new(LongestChain::new(block_store.clone())), + }; + + // misc vars for setting up the state + let genesis_time = 1_550_381_159; + + let latest_eth1_data = Eth1Data { + deposit_root: zero_hash.clone(), + block_hash: zero_hash.clone(), + }; + + let initial_validator_deposits = vec![]; + let spec = ChainSpec::foundation(); + + // create the state + let mut state = BeaconState::genesis( + genesis_time, + initial_validator_deposits, + latest_eth1_data, + &spec, + ) + .unwrap(); + + let default_validator = Validator { + pubkey: PublicKey::default(), + withdrawal_credentials: zero_hash, + activation_epoch: Epoch::from(0u64), + exit_epoch: spec.far_future_epoch, + withdrawal_epoch: spec.far_future_epoch, + penalized_epoch: spec.far_future_epoch, + status_flags: None, + }; + // activate the validators + for _ in 0..no_validators { + state.validator_registry.push(default_validator.clone()); + state.validator_balances.push(32_000_000_000); + } + + let state_root = state.canonical_root(); + state_store + .put(&state_root, &ssz_encode(&state)[..]) + .unwrap(); + + // return initialised vars + (fork_choice, block_store, state_root) +} From bd66a02cb351e179ea97a6d6cba4297494b55e70 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 19 Feb 2019 15:08:55 +1100 Subject: [PATCH 26/39] Add slow LMD Ghost working tests. --- eth2/fork_choice/src/slow_lmd_ghost.rs | 38 ++++++++++++++----- .../tests/lmd_ghost_test_vectors.yaml | 21 ++++++++++ .../optimised_lmd_ghost_test_vectors.yaml | 2 +- eth2/fork_choice/tests/tests.rs | 15 +++++++- 4 files changed, 64 insertions(+), 12 deletions(-) create mode 100644 eth2/fork_choice/tests/lmd_ghost_test_vectors.yaml diff --git a/eth2/fork_choice/src/slow_lmd_ghost.rs b/eth2/fork_choice/src/slow_lmd_ghost.rs index 3412f5473..c7cd9bde2 100644 --- a/eth2/fork_choice/src/slow_lmd_ghost.rs +++ b/eth2/fork_choice/src/slow_lmd_ghost.rs @@ -54,7 +54,6 @@ where // Note: Votes are weighted by min(balance, MAX_DEPOSIT_AMOUNT) // // FORK_CHOICE_BALANCE_INCREMENT // build a hashmap of block_hash to weighted votes - trace!("FORKCHOICE: Getting the latest votes"); let mut latest_votes: HashMap = HashMap::new(); // gets the current weighted votes let current_state = self @@ -66,10 +65,6 @@ where ¤t_state.validator_registry[..], block_slot.epoch(spec.epoch_length), ); - trace!( - "FORKCHOICE: Active validator indicies: {:?}", - active_validator_indices - ); for index in active_validator_indices { let balance = std::cmp::min( @@ -101,12 +96,12 @@ where .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*block_root))? .slot(); - for (target_hash, votes) in latest_votes.iter() { + for (vote_hash, votes) in latest_votes.iter() { let (root_at_slot, _) = self .block_store - .block_at_slot(&block_root, block_slot)? + .block_at_slot(&vote_hash, block_slot)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*block_root))?; - if root_at_slot == *target_hash { + if root_at_slot == *block_root { count += votes; } } @@ -142,12 +137,21 @@ impl ForkChoice for SlowLMDGhost { ) -> Result<(), ForkChoiceError> { // simply add the attestation to the latest_attestation_target if the block_height is // larger + trace!( + "FORKCHOICE: Adding attestation of validator: {:?} for block: {}", + validator_index, + target_block_root + ); let attestation_target = self .latest_attestation_targets .entry(validator_index) .or_insert_with(|| *target_block_root); // if we already have a value if attestation_target != target_block_root { + trace!( + "FORKCHOICE: Old attestation found: {:?}", + attestation_target + ); // get the height of the target block let block_height = self .block_store @@ -165,6 +169,7 @@ impl ForkChoice for SlowLMDGhost { .height(spec.genesis_slot); // update the attestation only if the new target is higher if past_block_height < block_height { + trace!("FORKCHOICE: Updating old attestation"); *attestation_target = *target_block_root; } } @@ -177,6 +182,7 @@ impl ForkChoice for SlowLMDGhost { justified_block_start: &Hash256, spec: &ChainSpec, ) -> Result { + debug!("FORKCHOICE: Running LMD Ghost Fork-choice rule"); let start = self .block_store .get_deserialized(&justified_block_start)? @@ -189,7 +195,7 @@ impl ForkChoice for SlowLMDGhost { let mut head_hash = Hash256::zero(); loop { - let mut head_vote_count = 0; + debug!("FORKCHOICE: Iteration for block: {}", head_hash); let children = match self.children.get(&head_hash) { Some(children) => children, @@ -197,8 +203,22 @@ impl ForkChoice for SlowLMDGhost { None => break, }; + // if we only have one child, use it + if children.len() == 1 { + trace!("FORKCHOICE: Single child found."); + head_hash = children[0]; + continue; + } + trace!("FORKCHOICE: Children found: {:?}", children); + + let mut head_vote_count = 0; for child_hash in children { let vote_count = self.get_vote_count(&latest_votes, &child_hash)?; + trace!( + "FORKCHOICE: Vote count for child: {} is: {}", + child_hash, + vote_count + ); if vote_count > head_vote_count { head_hash = *child_hash; diff --git a/eth2/fork_choice/tests/lmd_ghost_test_vectors.yaml b/eth2/fork_choice/tests/lmd_ghost_test_vectors.yaml new file mode 100644 index 000000000..3c2118222 --- /dev/null +++ b/eth2/fork_choice/tests/lmd_ghost_test_vectors.yaml @@ -0,0 +1,21 @@ +title: Fork-choice Tests +summary: A collection of abstract fork-choice tests for lmd ghost. +test_suite: Fork-Choice + +test_cases: +- blocks: + - id: 'b0' + parent: 'b0' + - id: 'b1' + parent: 'b0' + - id: 'b2' + parent: 'b1' + - id: 'b3' + parent: 'b1' + weights: + - b0: 0 + - b1: 0 + - b2: 5 + - b3: 10 + heads: + - id: 'b3' diff --git a/eth2/fork_choice/tests/optimised_lmd_ghost_test_vectors.yaml b/eth2/fork_choice/tests/optimised_lmd_ghost_test_vectors.yaml index 5a8869e8b..1c2f6b696 100644 --- a/eth2/fork_choice/tests/optimised_lmd_ghost_test_vectors.yaml +++ b/eth2/fork_choice/tests/optimised_lmd_ghost_test_vectors.yaml @@ -1,5 +1,5 @@ title: Fork-choice Tests -summary: A collection of abstract fork-choice tests. +summary: A collection of abstract fork-choice tests for bitwise lmd ghost. test_suite: Fork-Choice test_cases: diff --git a/eth2/fork_choice/tests/tests.rs b/eth2/fork_choice/tests/tests.rs index 83baa8bd8..0eafb58ab 100644 --- a/eth2/fork_choice/tests/tests.rs +++ b/eth2/fork_choice/tests/tests.rs @@ -25,13 +25,23 @@ use types::{ }; use yaml_rust::yaml; -// run tests #[test] fn test_optimised_lmd_ghost() { test_yaml_vectors( ForkChoiceAlgorithm::OptimisedLMDGhost, "tests/optimised_lmd_ghost_test_vectors.yaml", 100, + "debug", + ); +} + +#[test] +fn test_slow_lmd_ghost() { + test_yaml_vectors( + ForkChoiceAlgorithm::SlowLMDGhost, + "tests/lmd_ghost_test_vectors.yaml", + 100, + "debug", ); } @@ -40,9 +50,10 @@ fn test_yaml_vectors( fork_choice_algo: ForkChoiceAlgorithm, yaml_file_path: &str, max_validators: usize, + log_level: &str, ) { // set up logging - Builder::from_env(Env::default().default_filter_or("debug")).init(); + Builder::from_env(Env::default().default_filter_or(log_level)).init(); // load test cases from yaml let test_cases = load_test_cases_from_yaml(yaml_file_path); From 2394f64329f4f9bcd08d9f31bce66517a2fc9e7c Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 19 Feb 2019 15:22:35 +1100 Subject: [PATCH 27/39] Add longest chain tests and test vectors. --- .../tests/longest_chain_test_vectors.yaml | 51 +++++++++++++++++++ eth2/fork_choice/tests/tests.rs | 10 ++++ 2 files changed, 61 insertions(+) create mode 100644 eth2/fork_choice/tests/longest_chain_test_vectors.yaml diff --git a/eth2/fork_choice/tests/longest_chain_test_vectors.yaml b/eth2/fork_choice/tests/longest_chain_test_vectors.yaml new file mode 100644 index 000000000..e1cd61f06 --- /dev/null +++ b/eth2/fork_choice/tests/longest_chain_test_vectors.yaml @@ -0,0 +1,51 @@ +title: Fork-choice Tests +summary: A collection of abstract fork-choice tests to verify the longest chain fork-choice rule. +test_suite: Fork-Choice + +test_cases: +- blocks: + - id: 'b0' + parent: 'b0' + - id: 'b1' + parent: 'b0' + - id: 'b2' + parent: 'b1' + - id: 'b3' + parent: 'b1' + - id: 'b4' + parent: 'b3' + weights: + - b0: 0 + - b1: 0 + - b2: 10 + - b3: 1 + heads: + - id: 'b4' +- blocks: + - id: 'b0' + parent: 'b0' + - id: 'b1' + parent: 'b0' + - id: 'b2' + parent: 'b1' + - id: 'b3' + parent: 'b2' + - id: 'b4' + parent: 'b3' + - id: 'b5' + parent: 'b0' + - id: 'b6' + parent: 'b5' + - id: 'b7' + parent: 'b6' + - id: 'b8' + parent: 'b7' + - id: 'b9' + parent: 'b8' + weights: + - b0: 5 + - b1: 20 + - b2: 10 + - b3: 10 + heads: + - id: 'b9' diff --git a/eth2/fork_choice/tests/tests.rs b/eth2/fork_choice/tests/tests.rs index 0eafb58ab..b08c7031e 100644 --- a/eth2/fork_choice/tests/tests.rs +++ b/eth2/fork_choice/tests/tests.rs @@ -45,6 +45,16 @@ fn test_slow_lmd_ghost() { ); } +#[test] +fn test_longest_chain() { + test_yaml_vectors( + ForkChoiceAlgorithm::LongestChain, + "tests/longest_chain_test_vectors.yaml", + 100, + "debug", + ); +} + // run a generic test over given YAML test vectors fn test_yaml_vectors( fork_choice_algo: ForkChoiceAlgorithm, From b6f3156b4ee9e4395a8d54c83bd8f804e4434a24 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 19 Feb 2019 18:04:29 +1300 Subject: [PATCH 28/39] Run rustfmt on `ssz_derive` --- eth2/utils/ssz_derive/src/lib.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/eth2/utils/ssz_derive/src/lib.rs b/eth2/utils/ssz_derive/src/lib.rs index fdee88dd9..cb14e684f 100644 --- a/eth2/utils/ssz_derive/src/lib.rs +++ b/eth2/utils/ssz_derive/src/lib.rs @@ -5,12 +5,14 @@ use quote::quote; use syn::{parse_macro_input, DeriveInput}; fn get_named_field_idents<'a>(struct_data: &'a syn::DataStruct) -> Vec<&'a syn::Ident> { - struct_data.fields.iter().map(|f| { - match &f.ident { + struct_data + .fields + .iter() + .map(|f| match &f.ident { Some(ref ident) => ident, - _ => panic!("ssz_derive only supports named struct fields.") - } - }).collect() + _ => panic!("ssz_derive only supports named struct fields."), + }) + .collect() } #[proc_macro_derive(Encode)] @@ -21,7 +23,7 @@ pub fn ssz_encode_derive(input: TokenStream) -> TokenStream { let struct_data = match &item.data { syn::Data::Struct(s) => s, - _ => panic!("ssz_derive only supports structs.") + _ => panic!("ssz_derive only supports structs."), }; let field_idents = get_named_field_idents(&struct_data); @@ -46,7 +48,7 @@ pub fn ssz_decode_derive(input: TokenStream) -> TokenStream { let struct_data = match &item.data { syn::Data::Struct(s) => s, - _ => panic!("ssz_derive only supports structs.") + _ => panic!("ssz_derive only supports structs."), }; let field_idents = get_named_field_idents(&struct_data); From 5e67ddd498692e53058f90312bc89647edb98253 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 19 Feb 2019 20:43:09 +1300 Subject: [PATCH 29/39] Add docs to ssz_derive --- eth2/utils/ssz_derive/src/lib.rs | 41 ++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/eth2/utils/ssz_derive/src/lib.rs b/eth2/utils/ssz_derive/src/lib.rs index cb14e684f..b40cc1b69 100644 --- a/eth2/utils/ssz_derive/src/lib.rs +++ b/eth2/utils/ssz_derive/src/lib.rs @@ -1,3 +1,38 @@ +//! Provides the following procedural derive macros: +//! +//! - `#[derive(Encode)]` +//! - `#[derive(Decode)]` +//! +//! These macros provide SSZ encoding/decoding for a `struct`. Fields are encoded/decoded in the +//! order they are defined. +//! +//! Presently, only `structs` with named fields are supported. `enum`s and tuple-structs are +//! unsupported. +//! +//! Example: +//! ``` +//! use ssz::{ssz_encode, Decodable, Encodable}; +//! +//! #[derive(Encodable, Decodable)] +//! struct Foo { +//! bar: bool, +//! baz: u64, +//! } +//! +//! fn main() { +//! let foo = Foo { +//! bar: true, +//! baz: 42, +//! }; +//! +//! let bytes = ssz_encode(&foo); +//! +//! let decoded_foo = Foo::ssz_decode(bytes, 0).unwrap(); +//! +//! assert_eq!(foo.baz, decoded_foo.baz); +//! } +//! ``` + extern crate proc_macro; use proc_macro::TokenStream; @@ -15,6 +50,9 @@ fn get_named_field_idents<'a>(struct_data: &'a syn::DataStruct) -> Vec<&'a syn:: .collect() } +/// Implements `ssz::Encodable` on some `struct`. +/// +/// Fields are encoded in the order they are defined. #[proc_macro_derive(Encode)] pub fn ssz_encode_derive(input: TokenStream) -> TokenStream { let item = parse_macro_input!(input as DeriveInput); @@ -40,6 +78,9 @@ pub fn ssz_encode_derive(input: TokenStream) -> TokenStream { output.into() } +/// Implements `ssz::Decodable` on some `struct`. +/// +/// Fields are decoded in the order they are defined. #[proc_macro_derive(Decode)] pub fn ssz_decode_derive(input: TokenStream) -> TokenStream { let item = parse_macro_input!(input as DeriveInput); From fd1edaf8056471e552905e29ff06c93807ab963a Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 19 Feb 2019 23:06:35 +1100 Subject: [PATCH 30/39] Add fork choice bug fixes. - Further bug fixes from testing. - Simplify the testing framework. - Add tests for longest chain and GHOST vs bitwise GHOST. --- eth2/fork_choice/Cargo.toml | 1 + eth2/fork_choice/src/optimised_lmd_ghost.rs | 24 +++-- eth2/fork_choice/src/slow_lmd_ghost.rs | 2 +- .../tests/lmd_ghost_test_vectors.yaml | 16 ++++ .../optimised_lmd_ghost_test_vectors.yaml | 16 ++++ eth2/fork_choice/tests/tests.rs | 93 ++++++++++--------- 6 files changed, 99 insertions(+), 53 deletions(-) diff --git a/eth2/fork_choice/Cargo.toml b/eth2/fork_choice/Cargo.toml index d12f5ae7a..210f3c235 100644 --- a/eth2/fork_choice/Cargo.toml +++ b/eth2/fork_choice/Cargo.toml @@ -13,6 +13,7 @@ log = "0.4.6" bit-vec = "0.5.0" [dev-dependencies] +hex = "0.3.2" yaml-rust = "0.4.2" bls = { path = "../utils/bls" } slot_clock = { path = "../utils/slot_clock" } diff --git a/eth2/fork_choice/src/optimised_lmd_ghost.rs b/eth2/fork_choice/src/optimised_lmd_ghost.rs index f211359ea..e85a052ca 100644 --- a/eth2/fork_choice/src/optimised_lmd_ghost.rs +++ b/eth2/fork_choice/src/optimised_lmd_ghost.rs @@ -203,13 +203,22 @@ where } let mut bitmask: BitVec = BitVec::new(); // loop through bytes then bits - for bit in 0..256 { + for bit in 0..=256 { let mut zero_votes = 0; let mut one_votes = 0; - let mut single_candidate = None; + let mut single_candidate = (None, false); + trace!("FORKCHOICE: Child vote length: {}", votes.len()); for (candidate, votes) in votes.iter() { let candidate_bit: BitVec = BitVec::from_bytes(&candidate); + /* + trace!( + "FORKCHOICE: Child: {} in bits: {:?}", + candidate, + candidate_bit + ); + trace!("FORKCHOICE: Current bitmask: {:?}", bitmask); + */ // if the bitmasks don't match if !bitmask.iter().eq(candidate_bit.iter().take(bit)) { @@ -227,15 +236,16 @@ where one_votes += votes; } - if single_candidate.is_none() { - single_candidate = Some(candidate); + if single_candidate.0.is_none() { + single_candidate.0 = Some(candidate); + single_candidate.1 = true; } else { - single_candidate = None; + single_candidate.1 = false; } } bitmask.push(one_votes > zero_votes); - if let Some(candidate) = single_candidate { - return Some(*candidate); + if single_candidate.1 { + return Some(*single_candidate.0.expect("Cannot reach this")); } } // should never reach here diff --git a/eth2/fork_choice/src/slow_lmd_ghost.rs b/eth2/fork_choice/src/slow_lmd_ghost.rs index c7cd9bde2..d7dc4d1ad 100644 --- a/eth2/fork_choice/src/slow_lmd_ghost.rs +++ b/eth2/fork_choice/src/slow_lmd_ghost.rs @@ -192,7 +192,7 @@ impl ForkChoice for SlowLMDGhost { let latest_votes = self.get_latest_votes(&start_state_root, start.slot(), spec)?; - let mut head_hash = Hash256::zero(); + let mut head_hash = *justified_block_start; loop { debug!("FORKCHOICE: Iteration for block: {}", head_hash); diff --git a/eth2/fork_choice/tests/lmd_ghost_test_vectors.yaml b/eth2/fork_choice/tests/lmd_ghost_test_vectors.yaml index 3c2118222..4676d8201 100644 --- a/eth2/fork_choice/tests/lmd_ghost_test_vectors.yaml +++ b/eth2/fork_choice/tests/lmd_ghost_test_vectors.yaml @@ -19,3 +19,19 @@ test_cases: - b3: 10 heads: - id: 'b3' +# bitwise LMD ghost example. GHOST gives b1 +- blocks: + - id: 'b0' + parent: 'b0' + - id: 'b1' + parent: 'b0' + - id: 'b2' + parent: 'b0' + - id: 'b3' + parent: 'b0' + weights: + - b1: 5 + - b2: 4 + - b3: 3 + heads: + - id: 'b1' diff --git a/eth2/fork_choice/tests/optimised_lmd_ghost_test_vectors.yaml b/eth2/fork_choice/tests/optimised_lmd_ghost_test_vectors.yaml index 1c2f6b696..1578673cd 100644 --- a/eth2/fork_choice/tests/optimised_lmd_ghost_test_vectors.yaml +++ b/eth2/fork_choice/tests/optimised_lmd_ghost_test_vectors.yaml @@ -19,3 +19,19 @@ test_cases: - b3: 10 heads: - id: 'b3' +# bitwise LMD ghost example. bitwise GHOST gives b2 +- blocks: + - id: 'b0' + parent: 'b0' + - id: 'b1' + parent: 'b0' + - id: 'b2' + parent: 'b0' + - id: 'b3' + parent: 'b0' + weights: + - b1: 5 + - b2: 4 + - b3: 3 + heads: + - id: 'b2' diff --git a/eth2/fork_choice/tests/tests.rs b/eth2/fork_choice/tests/tests.rs index b08c7031e..01627eb73 100644 --- a/eth2/fork_choice/tests/tests.rs +++ b/eth2/fork_choice/tests/tests.rs @@ -3,8 +3,9 @@ extern crate beacon_chain; extern crate bls; extern crate db; -extern crate env_logger; +//extern crate env_logger; // for debugging extern crate fork_choice; +extern crate hex; extern crate log; extern crate slot_clock; extern crate types; @@ -14,7 +15,7 @@ pub use beacon_chain::BeaconChain; use bls::{PublicKey, Signature}; use db::stores::{BeaconBlockStore, BeaconStateStore}; use db::MemoryDB; -use env_logger::{Builder, Env}; +//use env_logger::{Builder, Env}; use fork_choice::{ForkChoice, ForkChoiceAlgorithm, LongestChain, OptimisedLMDGhost, SlowLMDGhost}; use ssz::ssz_encode; use std::collections::HashMap; @@ -25,13 +26,17 @@ use types::{ }; use yaml_rust::yaml; +// Note: We Assume the block Id's are hex-encoded. + #[test] fn test_optimised_lmd_ghost() { + // set up logging + //Builder::from_env(Env::default().default_filter_or("trace")).init(); + test_yaml_vectors( ForkChoiceAlgorithm::OptimisedLMDGhost, "tests/optimised_lmd_ghost_test_vectors.yaml", 100, - "debug", ); } @@ -41,7 +46,6 @@ fn test_slow_lmd_ghost() { ForkChoiceAlgorithm::SlowLMDGhost, "tests/lmd_ghost_test_vectors.yaml", 100, - "debug", ); } @@ -51,7 +55,6 @@ fn test_longest_chain() { ForkChoiceAlgorithm::LongestChain, "tests/longest_chain_test_vectors.yaml", 100, - "debug", ); } @@ -59,25 +62,11 @@ fn test_longest_chain() { fn test_yaml_vectors( fork_choice_algo: ForkChoiceAlgorithm, yaml_file_path: &str, - max_validators: usize, - log_level: &str, + emulated_validators: usize, // the number of validators used to give weights. ) { - // set up logging - Builder::from_env(Env::default().default_filter_or(log_level)).init(); - // load test cases from yaml let test_cases = load_test_cases_from_yaml(yaml_file_path); - // set up the test - let total_emulated_validators = max_validators; // the number of validators used to give weights. - let (mut fork_choice, block_store, state_root) = - setup_inital_state(fork_choice_algo, total_emulated_validators); - - // keep a hashmap of block_id's to block_hashes (random hashes to abstract block_id) - let mut block_id_map: HashMap = HashMap::new(); - // keep a list of hash to slot - let mut block_slot: HashMap = HashMap::new(); - // default vars let spec = ChainSpec::foundation(); let zero_hash = Hash256::zero(); @@ -97,33 +86,37 @@ fn test_yaml_vectors( // process the tests for test_case in test_cases { + // setup a fresh test + let (mut fork_choice, block_store, state_root) = + setup_inital_state(&fork_choice_algo, emulated_validators); + + // keep a hashmap of block_id's to block_hashes (random hashes to abstract block_id) + //let mut block_id_map: HashMap = HashMap::new(); + // keep a list of hash to slot + let mut block_slot: HashMap = HashMap::new(); // assume the block tree is given to us in order. + let mut genesis_hash = None; for block in test_case["blocks"].clone().into_vec().unwrap() { let block_id = block["id"].as_str().unwrap().to_string(); - let parent_id = block["parent"].as_str().unwrap(); + let parent_id = block["parent"].as_str().unwrap().to_string(); // default params for genesis - let mut block_hash = zero_hash.clone(); + let block_hash = id_to_hash(&block_id); let mut slot = spec.genesis_slot; - let mut parent_root = zero_hash; + let parent_root = id_to_hash(&parent_id); // set the slot and parent based off the YAML. Start with genesis; - // if not the genesis, update slot and parent + // if not the genesis, update slot if parent_id != block_id { - // generate a random hash for the block_hash - block_hash = Hash256::random(); - // find the parent hash - parent_root = *block_id_map - .get(parent_id) - .expect(&format!("Parent not found: {}", parent_id)); + // find parent slot slot = *(block_slot .get(&parent_root) .expect("Parent should have a slot number")) + 1; + } else { + genesis_hash = Some(block_hash); } - block_id_map.insert(block_id.clone(), block_hash.clone()); - // update slot mapping block_slot.insert(block_hash, slot); @@ -138,7 +131,7 @@ fn test_yaml_vectors( body: body.clone(), }; - // Store the block and state. + // Store the block. block_store .put(&block_hash, &ssz_encode(&beacon_block)[..]) .unwrap(); @@ -157,14 +150,15 @@ fn test_yaml_vectors( // get the block id and weights for (map_id, map_weight) in id_map.as_hash().unwrap().iter() { let id = map_id.as_str().unwrap(); - let block_root = block_id_map - .get(id) - .expect(&format!("Cannot find block id: {} in weights", id)); + let block_root = id_to_hash(&id.to_string()); let weight = map_weight.as_i64().unwrap(); // we assume a validator has a value 1 and add an attestation for to achieve the // correct weight for _ in 0..weight { - assert!(current_validator <= total_emulated_validators); + assert!( + current_validator <= emulated_validators, + "Not enough validators to emulate weights" + ); fork_choice .add_attestation(current_validator as u64, &block_root, &spec) .unwrap(); @@ -174,11 +168,8 @@ fn test_yaml_vectors( } // everything is set up, run the fork choice, using genesis as the head - let head = fork_choice.find_head(&zero_hash, &spec).unwrap(); - - let (found_id, _) = block_id_map - .iter() - .find(|(_, hash)| **hash == head) + let head = fork_choice + .find_head(&genesis_hash.unwrap(), &spec) .unwrap(); // compare the result to the expected test @@ -187,10 +178,10 @@ fn test_yaml_vectors( .into_vec() .unwrap() .iter() - .find(|heads| heads["id"].as_str().unwrap() == found_id) + .find(|heads| id_to_hash(&heads["id"].as_str().unwrap().to_string()) == head) .is_some(); - println!("Head Block ID: {:?}", found_id); + println!("Head found: {}", head); assert!(success, "Did not find one of the possible heads"); } } @@ -212,7 +203,7 @@ fn load_test_cases_from_yaml(file_path: &str) -> Vec { // initialise a single validator and state. All blocks will reference this state root. fn setup_inital_state( - fork_choice_algo: ForkChoiceAlgorithm, + fork_choice_algo: &ForkChoiceAlgorithm, no_validators: usize, ) -> (Box, Arc>, Hash256) { let zero_hash = Hash256::zero(); @@ -276,3 +267,15 @@ fn setup_inital_state( // return initialised vars (fork_choice, block_store, state_root) } + +// convert a block_id into a Hash256 -- assume input is hex encoded; +fn id_to_hash(id: &String) -> Hash256 { + let bytes = hex::decode(id).expect("Block ID should be hex"); + + let len = std::cmp::min(bytes.len(), 32); + let mut fixed_bytes = [0u8; 32]; + for (index, byte) in bytes.iter().take(32).enumerate() { + fixed_bytes[32 - len + index] = *byte; + } + Hash256::from(fixed_bytes) +} From 6f74ffc7e6967be56637b567c2e032fdc60baa6b Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 19 Feb 2019 23:20:45 +1100 Subject: [PATCH 31/39] Correct minor comment. --- eth2/fork_choice/tests/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth2/fork_choice/tests/tests.rs b/eth2/fork_choice/tests/tests.rs index 01627eb73..0bebe6a77 100644 --- a/eth2/fork_choice/tests/tests.rs +++ b/eth2/fork_choice/tests/tests.rs @@ -1,4 +1,4 @@ -// Tests the optimised LMD Ghost Algorithm +// Tests the available fork-choice algorithms extern crate beacon_chain; extern crate bls; From abef6698b1244a670342c7f24f15a053eb1c4a99 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 20 Feb 2019 10:12:18 +1300 Subject: [PATCH 32/39] Fix failing doc examples in ssz_derive --- eth2/utils/ssz_derive/src/lib.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/eth2/utils/ssz_derive/src/lib.rs b/eth2/utils/ssz_derive/src/lib.rs index b40cc1b69..41c47a496 100644 --- a/eth2/utils/ssz_derive/src/lib.rs +++ b/eth2/utils/ssz_derive/src/lib.rs @@ -11,12 +11,13 @@ //! //! Example: //! ``` -//! use ssz::{ssz_encode, Decodable, Encodable}; +//! use ssz::{ssz_encode, Decodable, Encodable, SszStream, DecodeError}; +//! use ssz_derive::{Encode, Decode}; //! -//! #[derive(Encodable, Decodable)] +//! #[derive(Encode, Decode)] //! struct Foo { -//! bar: bool, -//! baz: u64, +//! pub bar: bool, +//! pub baz: u64, //! } //! //! fn main() { @@ -27,7 +28,7 @@ //! //! let bytes = ssz_encode(&foo); //! -//! let decoded_foo = Foo::ssz_decode(bytes, 0).unwrap(); +//! let (decoded_foo, _i) = Foo::ssz_decode(&bytes, 0).unwrap(); //! //! assert_eq!(foo.baz, decoded_foo.baz); //! } @@ -50,7 +51,7 @@ fn get_named_field_idents<'a>(struct_data: &'a syn::DataStruct) -> Vec<&'a syn:: .collect() } -/// Implements `ssz::Encodable` on some `struct`. +/// Implements `ssz::Encodable` for some `struct`. /// /// Fields are encoded in the order they are defined. #[proc_macro_derive(Encode)] @@ -78,7 +79,7 @@ pub fn ssz_encode_derive(input: TokenStream) -> TokenStream { output.into() } -/// Implements `ssz::Decodable` on some `struct`. +/// Implements `ssz::Decodable` for some `struct`. /// /// Fields are decoded in the order they are defined. #[proc_macro_derive(Decode)] From 586bb09e02adbfd0c888cdb524579b17c529dad5 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 20 Feb 2019 11:06:03 +1300 Subject: [PATCH 33/39] Set ssz_derive to import from ssz:: Previously it was expecting `Encodable`, `Decodable`, etc to be in scope, now it uses `ssz::Encodable`. --- eth2/types/src/attestation.rs | 4 ++-- eth2/types/src/attestation_data.rs | 4 ++-- eth2/types/src/attestation_data_and_custody_bit.rs | 4 ++-- eth2/types/src/attester_slashing.rs | 4 ++-- eth2/types/src/beacon_block.rs | 4 ++-- eth2/types/src/beacon_block_body.rs | 4 ++-- eth2/types/src/beacon_state.rs | 2 +- eth2/types/src/beacon_state/tests.rs | 2 +- eth2/types/src/casper_slashing.rs | 4 ++-- eth2/types/src/crosslink.rs | 4 ++-- eth2/types/src/deposit.rs | 4 ++-- eth2/types/src/deposit_data.rs | 4 ++-- eth2/types/src/deposit_input.rs | 4 ++-- eth2/types/src/eth1_data.rs | 4 ++-- eth2/types/src/eth1_data_vote.rs | 4 ++-- eth2/types/src/exit.rs | 4 ++-- eth2/types/src/fork.rs | 4 ++-- eth2/types/src/pending_attestation.rs | 4 ++-- eth2/types/src/proposal_signed_data.rs | 4 ++-- eth2/types/src/proposer_slashing.rs | 4 ++-- eth2/types/src/shard_reassignment_record.rs | 4 ++-- eth2/types/src/slashable_attestation.rs | 4 ++-- eth2/types/src/slashable_vote_data.rs | 4 ++-- eth2/types/src/validator_registry_delta_block.rs | 4 ++-- eth2/utils/ssz_derive/src/lib.rs | 10 +++++----- 25 files changed, 51 insertions(+), 51 deletions(-) diff --git a/eth2/types/src/attestation.rs b/eth2/types/src/attestation.rs index 1b19e59cb..7388a8e49 100644 --- a/eth2/types/src/attestation.rs +++ b/eth2/types/src/attestation.rs @@ -2,7 +2,7 @@ use super::{AggregatePublicKey, AggregateSignature, AttestationData, Bitfield, H use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; #[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode)] @@ -60,7 +60,7 @@ impl TestRandom for Attestation { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; + use ssz::{ssz_encode, Decodable}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data.rs index a99d50f58..7edb0b72b 100644 --- a/eth2/types/src/attestation_data.rs +++ b/eth2/types/src/attestation_data.rs @@ -2,7 +2,7 @@ use crate::test_utils::TestRandom; use crate::{AttestationDataAndCustodyBit, Crosslink, Epoch, Hash256, Slot}; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; pub const SSZ_ATTESTION_DATA_LENGTH: usize = { @@ -78,7 +78,7 @@ impl TestRandom for AttestationData { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; + use ssz::{ssz_encode, Decodable}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/attestation_data_and_custody_bit.rs b/eth2/types/src/attestation_data_and_custody_bit.rs index afc9499d0..3f107be82 100644 --- a/eth2/types/src/attestation_data_and_custody_bit.rs +++ b/eth2/types/src/attestation_data_and_custody_bit.rs @@ -2,7 +2,7 @@ use super::AttestationData; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz::{Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::TreeHash; use ssz_derive::{Decode, Encode}; #[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode)] @@ -35,7 +35,7 @@ impl TestRandom for AttestationDataAndCustodyBit { mod test { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; + use ssz::{ssz_encode, Decodable}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/attester_slashing.rs b/eth2/types/src/attester_slashing.rs index d6578c917..f84998324 100644 --- a/eth2/types/src/attester_slashing.rs +++ b/eth2/types/src/attester_slashing.rs @@ -1,7 +1,7 @@ use crate::{test_utils::TestRandom, SlashableAttestation}; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; #[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] @@ -32,7 +32,7 @@ impl TestRandom for AttesterSlashing { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; + use ssz::{ssz_encode, Decodable}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/beacon_block.rs b/eth2/types/src/beacon_block.rs index 08c7841d8..c252d03f7 100644 --- a/eth2/types/src/beacon_block.rs +++ b/eth2/types/src/beacon_block.rs @@ -3,7 +3,7 @@ use crate::{BeaconBlockBody, ChainSpec, Eth1Data, Hash256, ProposalSignedData, S use bls::Signature; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; #[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] @@ -92,7 +92,7 @@ impl TestRandom for BeaconBlock { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; + use ssz::{ssz_encode, Decodable}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/beacon_block_body.rs b/eth2/types/src/beacon_block_body.rs index ffb2ea9d3..e051f5940 100644 --- a/eth2/types/src/beacon_block_body.rs +++ b/eth2/types/src/beacon_block_body.rs @@ -2,7 +2,7 @@ use super::{Attestation, AttesterSlashing, Deposit, Exit, ProposerSlashing}; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; #[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode)] @@ -42,7 +42,7 @@ impl TestRandom for BeaconBlockBody { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; + use ssz::{ssz_encode, Decodable}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 85aaabf24..21deb6fe7 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -9,7 +9,7 @@ use honey_badger_split::SplitExt; use log::trace; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; use swap_or_not_shuffle::get_permutated_index; diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs index 2b7c5b539..7e25d4dba 100644 --- a/eth2/types/src/beacon_state/tests.rs +++ b/eth2/types/src/beacon_state/tests.rs @@ -7,7 +7,7 @@ use crate::{ Eth1Data, Hash256, Keypair, }; use bls::create_proof_of_possession; -use ssz::ssz_encode; +use ssz::{ssz_encode, Decodable}; struct BeaconStateTestBuilder { pub genesis_time: u64, diff --git a/eth2/types/src/casper_slashing.rs b/eth2/types/src/casper_slashing.rs index 27fce54a9..6346db65c 100644 --- a/eth2/types/src/casper_slashing.rs +++ b/eth2/types/src/casper_slashing.rs @@ -2,7 +2,7 @@ use super::SlashableVoteData; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; #[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] @@ -33,7 +33,7 @@ impl TestRandom for CasperSlashing { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; + use ssz::{ssz_encode, Decodable}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/crosslink.rs b/eth2/types/src/crosslink.rs index a1d467083..19c71f604 100644 --- a/eth2/types/src/crosslink.rs +++ b/eth2/types/src/crosslink.rs @@ -2,7 +2,7 @@ use crate::test_utils::TestRandom; use crate::{Epoch, Hash256}; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; #[derive(Debug, Clone, PartialEq, Default, Serialize, Hash, Encode, Decode)] @@ -43,7 +43,7 @@ impl TestRandom for Crosslink { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; + use ssz::{ssz_encode, Decodable}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/deposit.rs b/eth2/types/src/deposit.rs index eb54df6f3..78f43532a 100644 --- a/eth2/types/src/deposit.rs +++ b/eth2/types/src/deposit.rs @@ -2,7 +2,7 @@ use super::{DepositData, Hash256}; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; #[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] @@ -36,7 +36,7 @@ impl TestRandom for Deposit { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; + use ssz::{ssz_encode, Decodable}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/deposit_data.rs b/eth2/types/src/deposit_data.rs index f22a1cf1a..8f49deb3c 100644 --- a/eth2/types/src/deposit_data.rs +++ b/eth2/types/src/deposit_data.rs @@ -2,7 +2,7 @@ use super::DepositInput; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; #[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] @@ -36,7 +36,7 @@ impl TestRandom for DepositData { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; + use ssz::{ssz_encode, Decodable}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/deposit_input.rs b/eth2/types/src/deposit_input.rs index a8299c455..7556fc2ca 100644 --- a/eth2/types/src/deposit_input.rs +++ b/eth2/types/src/deposit_input.rs @@ -3,7 +3,7 @@ use crate::test_utils::TestRandom; use bls::{PublicKey, Signature}; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; #[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] @@ -37,7 +37,7 @@ impl TestRandom for DepositInput { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; + use ssz::{ssz_encode, Decodable}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/eth1_data.rs b/eth2/types/src/eth1_data.rs index c88627548..b0dc14e7a 100644 --- a/eth2/types/src/eth1_data.rs +++ b/eth2/types/src/eth1_data.rs @@ -2,7 +2,7 @@ use super::Hash256; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; // Note: this is refer to as DepositRootVote in specs @@ -34,7 +34,7 @@ impl TestRandom for Eth1Data { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; + use ssz::{ssz_encode, Decodable}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/eth1_data_vote.rs b/eth2/types/src/eth1_data_vote.rs index ee891523b..eda6e6a6a 100644 --- a/eth2/types/src/eth1_data_vote.rs +++ b/eth2/types/src/eth1_data_vote.rs @@ -2,7 +2,7 @@ use super::Eth1Data; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; // Note: this is refer to as DepositRootVote in specs @@ -34,7 +34,7 @@ impl TestRandom for Eth1DataVote { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; + use ssz::{ssz_encode, Decodable}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/exit.rs b/eth2/types/src/exit.rs index 1379497ae..18d743b83 100644 --- a/eth2/types/src/exit.rs +++ b/eth2/types/src/exit.rs @@ -2,7 +2,7 @@ use crate::{test_utils::TestRandom, Epoch}; use bls::Signature; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; #[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] @@ -36,7 +36,7 @@ impl TestRandom for Exit { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; + use ssz::{ssz_encode, Decodable}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs index 5c1fa6fb0..85d530e19 100644 --- a/eth2/types/src/fork.rs +++ b/eth2/types/src/fork.rs @@ -1,7 +1,7 @@ use crate::{test_utils::TestRandom, Epoch}; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; #[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode)] @@ -35,7 +35,7 @@ impl TestRandom for Fork { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; + use ssz::{ssz_encode, Decodable}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/pending_attestation.rs b/eth2/types/src/pending_attestation.rs index c3d6023c4..42f990210 100644 --- a/eth2/types/src/pending_attestation.rs +++ b/eth2/types/src/pending_attestation.rs @@ -2,7 +2,7 @@ use crate::test_utils::TestRandom; use crate::{AttestationData, Bitfield, Slot}; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; #[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode)] @@ -39,7 +39,7 @@ impl TestRandom for PendingAttestation { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; + use ssz::{ssz_encode, Decodable}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/proposal_signed_data.rs b/eth2/types/src/proposal_signed_data.rs index b544c807b..63c0f1ce6 100644 --- a/eth2/types/src/proposal_signed_data.rs +++ b/eth2/types/src/proposal_signed_data.rs @@ -2,7 +2,7 @@ use crate::test_utils::TestRandom; use crate::{Hash256, Slot}; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; #[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode)] @@ -36,7 +36,7 @@ impl TestRandom for ProposalSignedData { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; + use ssz::{ssz_encode, Decodable}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/proposer_slashing.rs b/eth2/types/src/proposer_slashing.rs index 1b1620df8..b3a819a7f 100644 --- a/eth2/types/src/proposer_slashing.rs +++ b/eth2/types/src/proposer_slashing.rs @@ -3,7 +3,7 @@ use crate::test_utils::TestRandom; use bls::Signature; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; #[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] @@ -43,7 +43,7 @@ impl TestRandom for ProposerSlashing { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; + use ssz::{ssz_encode, Decodable}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/shard_reassignment_record.rs b/eth2/types/src/shard_reassignment_record.rs index 0a4650e6e..511fe13ca 100644 --- a/eth2/types/src/shard_reassignment_record.rs +++ b/eth2/types/src/shard_reassignment_record.rs @@ -1,7 +1,7 @@ use crate::{test_utils::TestRandom, Slot}; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; #[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] @@ -35,7 +35,7 @@ impl TestRandom for ShardReassignmentRecord { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; + use ssz::{ssz_encode, Decodable}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/slashable_attestation.rs b/eth2/types/src/slashable_attestation.rs index 2ae625c23..676954ec2 100644 --- a/eth2/types/src/slashable_attestation.rs +++ b/eth2/types/src/slashable_attestation.rs @@ -1,7 +1,7 @@ use crate::{test_utils::TestRandom, AggregateSignature, AttestationData, Bitfield}; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; #[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] @@ -38,7 +38,7 @@ impl TestRandom for SlashableAttestation { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; + use ssz::{ssz_encode, Decodable}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/slashable_vote_data.rs b/eth2/types/src/slashable_vote_data.rs index ba7b9ae92..bdd1d0619 100644 --- a/eth2/types/src/slashable_vote_data.rs +++ b/eth2/types/src/slashable_vote_data.rs @@ -4,7 +4,7 @@ use crate::test_utils::TestRandom; use bls::AggregateSignature; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; #[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] @@ -64,7 +64,7 @@ mod tests { use crate::chain_spec::ChainSpec; use crate::slot_epoch::{Epoch, Slot}; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; + use ssz::{ssz_encode, Decodable}; #[test] pub fn test_is_double_vote_true() { diff --git a/eth2/types/src/validator_registry_delta_block.rs b/eth2/types/src/validator_registry_delta_block.rs index f171880b0..14f9c6ce5 100644 --- a/eth2/types/src/validator_registry_delta_block.rs +++ b/eth2/types/src/validator_registry_delta_block.rs @@ -2,7 +2,7 @@ use crate::{test_utils::TestRandom, Hash256, Slot}; use bls::PublicKey; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; // The information gathered from the PoW chain validator registration function. @@ -56,7 +56,7 @@ impl TestRandom for ValidatorRegistryDeltaBlock { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; + use ssz::{ssz_encode, Decodable}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/utils/ssz_derive/src/lib.rs b/eth2/utils/ssz_derive/src/lib.rs index 41c47a496..dbf7638f2 100644 --- a/eth2/utils/ssz_derive/src/lib.rs +++ b/eth2/utils/ssz_derive/src/lib.rs @@ -11,7 +11,7 @@ //! //! Example: //! ``` -//! use ssz::{ssz_encode, Decodable, Encodable, SszStream, DecodeError}; +//! use ssz::{ssz_encode, Decodable}; //! use ssz_derive::{Encode, Decode}; //! //! #[derive(Encode, Decode)] @@ -68,8 +68,8 @@ pub fn ssz_encode_derive(input: TokenStream) -> TokenStream { let field_idents = get_named_field_idents(&struct_data); let output = quote! { - impl Encodable for #name { - fn ssz_append(&self, s: &mut SszStream) { + impl ssz::Encodable for #name { + fn ssz_append(&self, s: &mut ssz::SszStream) { #( s.append(&self.#field_idents); )* @@ -103,8 +103,8 @@ pub fn ssz_decode_derive(input: TokenStream) -> TokenStream { let field_idents_b = &field_idents; let output = quote! { - impl Decodable for #name { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { + impl ssz::Decodable for #name { + fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), ssz::DecodeError> { #( let (#field_idents_a, i) = <_>::ssz_decode(bytes, i)?; )* From 59fd7162868b6998a9674a6701fc18bbaa5fb92b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 20 Feb 2019 11:23:35 +1300 Subject: [PATCH 34/39] Add extra comment to ssz_derive --- eth2/utils/ssz_derive/src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eth2/utils/ssz_derive/src/lib.rs b/eth2/utils/ssz_derive/src/lib.rs index dbf7638f2..1bc5caef1 100644 --- a/eth2/utils/ssz_derive/src/lib.rs +++ b/eth2/utils/ssz_derive/src/lib.rs @@ -40,6 +40,10 @@ use proc_macro::TokenStream; use quote::quote; use syn::{parse_macro_input, DeriveInput}; +/// Returns a Vec of `syn::Ident` for each named field in the struct. +/// +/// # Panics +/// Any unnamed struct field (like in a tuple struct) will raise a panic at compile time. fn get_named_field_idents<'a>(struct_data: &'a syn::DataStruct) -> Vec<&'a syn::Ident> { struct_data .fields From c7acde4fc27ad7c05aa65aa341a83930d6c17b17 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 20 Feb 2019 11:56:58 +1100 Subject: [PATCH 35/39] Rename OptimisedLMDGhost to BitwiseLMDGhost. --- .../test_harness/src/beacon_chain_harness.rs | 6 +++--- .../test_harness/src/validator_harness/mod.rs | 16 ++++++++-------- beacon_node/src/main.rs | 4 ++-- ...timised_lmd_ghost.rs => bitwise_lmd_ghost.rs} | 8 ++++---- eth2/fork_choice/src/lib.rs | 12 ++++++------ ....yaml => bitwise_lmd_ghost_test_vectors.yaml} | 0 eth2/fork_choice/tests/tests.rs | 10 +++++----- 7 files changed, 28 insertions(+), 28 deletions(-) rename eth2/fork_choice/src/{optimised_lmd_ghost.rs => bitwise_lmd_ghost.rs} (99%) rename eth2/fork_choice/tests/{optimised_lmd_ghost_test_vectors.yaml => bitwise_lmd_ghost_test_vectors.yaml} (100%) diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index 9d61952f0..a20441beb 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -6,7 +6,7 @@ use db::{ stores::{BeaconBlockStore, BeaconStateStore}, MemoryDB, }; -use fork_choice::OptimisedLMDGhost; +use fork_choice::BitwiseLMDGhost; use log::debug; use rayon::prelude::*; use slot_clock::TestingSlotClock; @@ -28,7 +28,7 @@ use types::{ /// is not useful for testing that multiple beacon nodes can reach consensus. pub struct BeaconChainHarness { pub db: Arc, - pub beacon_chain: Arc>>, + pub beacon_chain: Arc>>, pub block_store: Arc>, pub state_store: Arc>, pub validators: Vec, @@ -46,7 +46,7 @@ impl BeaconChainHarness { let state_store = Arc::new(BeaconStateStore::new(db.clone())); let genesis_time = 1_549_935_547; // 12th Feb 2018 (arbitrary value in the past). let slot_clock = TestingSlotClock::new(spec.genesis_slot.as_u64()); - let fork_choice = OptimisedLMDGhost::new(block_store.clone(), state_store.clone()); + let fork_choice = BitwiseLMDGhost::new(block_store.clone(), state_store.clone()); let latest_eth1_data = Eth1Data { deposit_root: Hash256::zero(), block_hash: Hash256::zero(), diff --git a/beacon_node/beacon_chain/test_harness/src/validator_harness/mod.rs b/beacon_node/beacon_chain/test_harness/src/validator_harness/mod.rs index f48309541..60c2f8ecf 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator_harness/mod.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator_harness/mod.rs @@ -10,7 +10,7 @@ use block_proposer::{BlockProducer, Error as BlockPollError}; use db::MemoryDB; use direct_beacon_node::DirectBeaconNode; use direct_duties::DirectDuties; -use fork_choice::OptimisedLMDGhost; +use fork_choice::BitwiseLMDGhost; use local_signer::LocalSigner; use slot_clock::TestingSlotClock; use std::sync::Arc; @@ -36,20 +36,20 @@ pub enum AttestationProduceError { pub struct ValidatorHarness { pub block_producer: BlockProducer< TestingSlotClock, - DirectBeaconNode>, - DirectDuties>, + DirectBeaconNode>, + DirectDuties>, LocalSigner, >, pub attester: Attester< TestingSlotClock, - DirectBeaconNode>, - DirectDuties>, + DirectBeaconNode>, + DirectDuties>, LocalSigner, >, pub spec: Arc, - pub epoch_map: Arc>>, + pub epoch_map: Arc>>, pub keypair: Keypair, - pub beacon_node: Arc>>, + pub beacon_node: Arc>>, pub slot_clock: Arc, pub signer: Arc, } @@ -61,7 +61,7 @@ impl ValidatorHarness { /// A `BlockProducer` and `Attester` is created.. pub fn new( keypair: Keypair, - beacon_chain: Arc>>, + beacon_chain: Arc>>, spec: Arc, ) -> Self { let slot_clock = Arc::new(TestingSlotClock::new(spec.genesis_slot.as_u64())); diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index 2b6cdddcd..b9ef2c8a7 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -14,7 +14,7 @@ use db::{ stores::{BeaconBlockStore, BeaconStateStore}, MemoryDB, }; -use fork_choice::optimised_lmd_ghost::OptimisedLMDGhost; +use fork_choice::BitwiseLMDGhost; use slog::{error, info, o, Drain}; use slot_clock::SystemTimeSlotClock; use std::sync::Arc; @@ -81,7 +81,7 @@ fn main() { let slot_clock = SystemTimeSlotClock::new(genesis_time, spec.slot_duration) .expect("Unable to load SystemTimeSlotClock"); // Choose the fork choice - let fork_choice = OptimisedLMDGhost::new(block_store.clone(), state_store.clone()); + let fork_choice = BitwiseLMDGhost::new(block_store.clone(), state_store.clone()); /* * Generate some random data to start a chain with. diff --git a/eth2/fork_choice/src/optimised_lmd_ghost.rs b/eth2/fork_choice/src/bitwise_lmd_ghost.rs similarity index 99% rename from eth2/fork_choice/src/optimised_lmd_ghost.rs rename to eth2/fork_choice/src/bitwise_lmd_ghost.rs index e85a052ca..c09de1529 100644 --- a/eth2/fork_choice/src/optimised_lmd_ghost.rs +++ b/eth2/fork_choice/src/bitwise_lmd_ghost.rs @@ -37,7 +37,7 @@ fn power_of_2_below(x: u32) -> u32 { } /// Stores the necessary data structures to run the optimised lmd ghost algorithm. -pub struct OptimisedLMDGhost { +pub struct BitwiseLMDGhost { /// A cache of known ancestors at given heights for a specific block. //TODO: Consider FnvHashMap cache: HashMap, Hash256>, @@ -56,7 +56,7 @@ pub struct OptimisedLMDGhost { max_known_height: SlotHeight, } -impl OptimisedLMDGhost +impl BitwiseLMDGhost where T: ClientDB + Sized, { @@ -64,7 +64,7 @@ where block_store: Arc>, state_store: Arc>, ) -> Self { - OptimisedLMDGhost { + BitwiseLMDGhost { cache: HashMap::new(), ancestors: vec![HashMap::new(); 16], latest_attestation_targets: HashMap::new(), @@ -253,7 +253,7 @@ where } } -impl ForkChoice for OptimisedLMDGhost { +impl ForkChoice for BitwiseLMDGhost { fn add_block( &mut self, block: &BeaconBlock, diff --git a/eth2/fork_choice/src/lib.rs b/eth2/fork_choice/src/lib.rs index 90fb0e82b..4a58e3057 100644 --- a/eth2/fork_choice/src/lib.rs +++ b/eth2/fork_choice/src/lib.rs @@ -9,12 +9,12 @@ //! production**. //! - [`slow_lmd_ghost`]: This is a simple and very inefficient implementation given in the ethereum 2.0 //! specifications (https://github.com/ethereum/eth2.0-specs/blob/v0.1/specs/core/0_beacon-chain.md#get_block_root). -//! - [`optimised_lmd_ghost`]: This is an optimised version of bitwise LMD-GHOST as proposed +//! - [`bitwise_lmd_ghost`]: This is an optimised version of bitwise LMD-GHOST as proposed //! by Vitalik. The reference implementation can be found at: https://github.com/ethereum/research/blob/master/ghost/ghost.py //! //! [`longest-chain`]: struct.LongestChain.html //! [`slow_lmd_ghost`]: struct.SlowLmdGhost.html -//! [`optimised_lmd_ghost`]: struct.OptimisedLmdGhost.html +//! [`bitwise_lmd_ghost`]: struct.OptimisedLmdGhost.html extern crate db; extern crate ssz; @@ -22,16 +22,16 @@ extern crate types; #[macro_use] extern crate log; +pub mod bitwise_lmd_ghost; pub mod longest_chain; -pub mod optimised_lmd_ghost; pub mod slow_lmd_ghost; use db::stores::BeaconBlockAtSlotError; use db::DBError; use types::{BeaconBlock, ChainSpec, Hash256}; +pub use bitwise_lmd_ghost::BitwiseLMDGhost; pub use longest_chain::LongestChain; -pub use optimised_lmd_ghost::OptimisedLMDGhost; pub use slow_lmd_ghost::SlowLMDGhost; /// Defines the interface for Fork Choices. Each Fork choice will define their own data structures @@ -101,6 +101,6 @@ pub enum ForkChoiceAlgorithm { LongestChain, /// A simple and highly inefficient implementation of LMD ghost. SlowLMDGhost, - /// An optimised version of LMD-GHOST by Vitalik. - OptimisedLMDGhost, + /// An optimised version of bitwise LMD-GHOST by Vitalik. + BitwiseLMDGhost, } diff --git a/eth2/fork_choice/tests/optimised_lmd_ghost_test_vectors.yaml b/eth2/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml similarity index 100% rename from eth2/fork_choice/tests/optimised_lmd_ghost_test_vectors.yaml rename to eth2/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml diff --git a/eth2/fork_choice/tests/tests.rs b/eth2/fork_choice/tests/tests.rs index 0bebe6a77..1d93cd0db 100644 --- a/eth2/fork_choice/tests/tests.rs +++ b/eth2/fork_choice/tests/tests.rs @@ -16,7 +16,7 @@ use bls::{PublicKey, Signature}; use db::stores::{BeaconBlockStore, BeaconStateStore}; use db::MemoryDB; //use env_logger::{Builder, Env}; -use fork_choice::{ForkChoice, ForkChoiceAlgorithm, LongestChain, OptimisedLMDGhost, SlowLMDGhost}; +use fork_choice::{BitwiseLMDGhost, ForkChoice, ForkChoiceAlgorithm, LongestChain, SlowLMDGhost}; use ssz::ssz_encode; use std::collections::HashMap; use std::sync::Arc; @@ -29,13 +29,13 @@ use yaml_rust::yaml; // Note: We Assume the block Id's are hex-encoded. #[test] -fn test_optimised_lmd_ghost() { +fn test_bitwise_lmd_ghost() { // set up logging //Builder::from_env(Env::default().default_filter_or("trace")).init(); test_yaml_vectors( - ForkChoiceAlgorithm::OptimisedLMDGhost, - "tests/optimised_lmd_ghost_test_vectors.yaml", + ForkChoiceAlgorithm::BitwiseLMDGhost, + "tests/bitwise_lmd_ghost_test_vectors.yaml", 100, ); } @@ -214,7 +214,7 @@ fn setup_inital_state( // the fork choice instantiation let fork_choice: Box = match fork_choice_algo { - ForkChoiceAlgorithm::OptimisedLMDGhost => Box::new(OptimisedLMDGhost::new( + ForkChoiceAlgorithm::BitwiseLMDGhost => Box::new(BitwiseLMDGhost::new( block_store.clone(), state_store.clone(), )), From d8584cbed27e70ec15dacf39a17312988de5f61d Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 20 Feb 2019 12:15:41 +1100 Subject: [PATCH 36/39] Update to rust 2018 importing macros. --- eth2/fork_choice/src/bitwise_lmd_ghost.rs | 1 + eth2/fork_choice/src/lib.rs | 2 -- eth2/fork_choice/src/slow_lmd_ghost.rs | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eth2/fork_choice/src/bitwise_lmd_ghost.rs b/eth2/fork_choice/src/bitwise_lmd_ghost.rs index c09de1529..e057fa1ee 100644 --- a/eth2/fork_choice/src/bitwise_lmd_ghost.rs +++ b/eth2/fork_choice/src/bitwise_lmd_ghost.rs @@ -8,6 +8,7 @@ use db::{ ClientDB, }; use fast_math::log2_raw; +use log::{debug, trace}; use std::collections::HashMap; use std::sync::Arc; use types::{ diff --git a/eth2/fork_choice/src/lib.rs b/eth2/fork_choice/src/lib.rs index 4a58e3057..6062c19b1 100644 --- a/eth2/fork_choice/src/lib.rs +++ b/eth2/fork_choice/src/lib.rs @@ -19,8 +19,6 @@ extern crate db; extern crate ssz; extern crate types; -#[macro_use] -extern crate log; pub mod bitwise_lmd_ghost; pub mod longest_chain; diff --git a/eth2/fork_choice/src/slow_lmd_ghost.rs b/eth2/fork_choice/src/slow_lmd_ghost.rs index d7dc4d1ad..7cad73285 100644 --- a/eth2/fork_choice/src/slow_lmd_ghost.rs +++ b/eth2/fork_choice/src/slow_lmd_ghost.rs @@ -5,6 +5,7 @@ use db::{ stores::{BeaconBlockStore, BeaconStateStore}, ClientDB, }; +use log::{debug, trace}; use std::collections::HashMap; use std::sync::Arc; use types::{ From 2fbdc531471414bb11627a5912db3bf4cccdcbe4 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 20 Feb 2019 12:36:54 +1100 Subject: [PATCH 37/39] Add asserts to ensure block heights are not too large. --- eth2/fork_choice/src/bitwise_lmd_ghost.rs | 8 ++++++-- eth2/types/src/slot_epoch_macros.rs | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/eth2/fork_choice/src/bitwise_lmd_ghost.rs b/eth2/fork_choice/src/bitwise_lmd_ghost.rs index e057fa1ee..013c901b9 100644 --- a/eth2/fork_choice/src/bitwise_lmd_ghost.rs +++ b/eth2/fork_choice/src/bitwise_lmd_ghost.rs @@ -19,7 +19,7 @@ use types::{ //TODO: Pruning - Children //TODO: Handle Syncing -/// The optimised LMD-GHOST fork choice rule. +/// The optimised bitwise LMD-GHOST fork choice rule. /// NOTE: This uses u32 to represent difference between block heights. Thus this is only /// applicable for block height differences in the range of a u32. /// This can potentially be parallelized in some parts. @@ -30,6 +30,10 @@ fn log2_int(x: u32) -> u32 { if x == 0 { return 0; } + assert!( + x <= std::f32::MAX as u32, + "Height too large for fast log in bitwise fork choice" + ); log2_raw(x as f32) as u32 } @@ -37,7 +41,7 @@ fn power_of_2_below(x: u32) -> u32 { 2u32.pow(log2_int(x)) } -/// Stores the necessary data structures to run the optimised lmd ghost algorithm. +/// Stores the necessary data structures to run the optimised bitwise lmd ghost algorithm. pub struct BitwiseLMDGhost { /// A cache of known ancestors at given heights for a specific block. //TODO: Consider FnvHashMap diff --git a/eth2/types/src/slot_epoch_macros.rs b/eth2/types/src/slot_epoch_macros.rs index 54a8f2ce9..b0550f2f8 100644 --- a/eth2/types/src/slot_epoch_macros.rs +++ b/eth2/types/src/slot_epoch_macros.rs @@ -25,12 +25,14 @@ macro_rules! impl_into_u32 { ($main: ident) => { impl Into for $main { fn into(self) -> u32 { + assert!(self.0 < u64::from(std::u32::MAX), "Lossy conversion to u32"); self.0 as u32 } } impl $main { pub fn as_u32(&self) -> u32 { + assert!(self.0 < u64::from(std::u32::MAX), "Lossy conversion to u32"); self.0 as u32 } } From 0f7167992b8171fd1c9170630018b04e30872b00 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 20 Feb 2019 12:39:00 +1100 Subject: [PATCH 38/39] Removes topic from logs. --- eth2/fork_choice/src/bitwise_lmd_ghost.rs | 33 +++++++++++------------ eth2/fork_choice/src/slow_lmd_ghost.rs | 25 +++++++---------- 2 files changed, 24 insertions(+), 34 deletions(-) diff --git a/eth2/fork_choice/src/bitwise_lmd_ghost.rs b/eth2/fork_choice/src/bitwise_lmd_ghost.rs index 013c901b9..c467545ea 100644 --- a/eth2/fork_choice/src/bitwise_lmd_ghost.rs +++ b/eth2/fork_choice/src/bitwise_lmd_ghost.rs @@ -115,7 +115,7 @@ where } } } - trace!("FORKCHOICE: Latest votes: {:?}", latest_votes); + trace!("Latest votes: {:?}", latest_votes); Ok(latest_votes) } @@ -180,7 +180,7 @@ where let mut current_votes: HashMap = HashMap::new(); let mut total_vote_count = 0; - trace!("FORKCHOICE: Clear winner at block height: {}", block_height); + trace!("Clear winner at block height: {}", block_height); // loop through the latest votes and count all votes // these have already been weighted by balance for (hash, votes) in latest_votes.iter() { @@ -213,22 +213,22 @@ where let mut one_votes = 0; let mut single_candidate = (None, false); - trace!("FORKCHOICE: Child vote length: {}", votes.len()); + trace!("Child vote length: {}", votes.len()); for (candidate, votes) in votes.iter() { let candidate_bit: BitVec = BitVec::from_bytes(&candidate); /* trace!( - "FORKCHOICE: Child: {} in bits: {:?}", + "Child: {} in bits: {:?}", candidate, candidate_bit ); - trace!("FORKCHOICE: Current bitmask: {:?}", bitmask); + trace!("Current bitmask: {:?}", bitmask); */ // if the bitmasks don't match if !bitmask.iter().eq(candidate_bit.iter().take(bit)) { trace!( - "FORKCHOICE: Child: {} was removed in bit: {} with the bitmask: {:?}", + "Child: {} was removed in bit: {} with the bitmask: {:?}", candidate, bit, bitmask @@ -306,7 +306,7 @@ impl ForkChoice for BitwiseLMDGhost { // simply add the attestation to the latest_attestation_target if the block_height is // larger trace!( - "FORKCHOICE: Adding attestation of validator: {:?} for block: {}", + "Adding attestation of validator: {:?} for block: {}", validator_index, target_block_root ); @@ -316,10 +316,7 @@ impl ForkChoice for BitwiseLMDGhost { .or_insert_with(|| *target_block_root); // if we already have a value if attestation_target != target_block_root { - trace!( - "FORKCHOICE: Old attestation found: {:?}", - attestation_target - ); + trace!("Old attestation found: {:?}", attestation_target); // get the height of the target block let block_height = self .block_store @@ -337,7 +334,7 @@ impl ForkChoice for BitwiseLMDGhost { .height(spec.genesis_slot); // update the attestation only if the new target is higher if past_block_height < block_height { - trace!("FORKCHOICE: Updating old attestation"); + trace!("Updating old attestation"); *attestation_target = *target_block_root; } } @@ -374,7 +371,7 @@ impl ForkChoice for BitwiseLMDGhost { // begin searching for the head loop { debug!( - "FORKCHOICE: Iteration for block: {} with vote length: {}", + "Iteration for block: {} with vote length: {}", current_head, latest_votes.len() ); @@ -401,19 +398,19 @@ impl ForkChoice for BitwiseLMDGhost { step /= 2; } if step > 0 { - trace!("FORKCHOICE: Found clear winner in log lookup"); + trace!("Found clear winner in log lookup"); } // if our skip lookup failed and we only have one child, progress to that child else if children.len() == 1 { current_head = children[0]; trace!( - "FORKCHOICE: Lookup failed, only one child, proceeding to child: {}", + "Lookup failed, only one child, proceeding to child: {}", current_head ); } // we need to find the best child path to progress down. else { - trace!("FORKCHOICE: Searching for best child"); + trace!("Searching for best child"); let mut child_votes = HashMap::new(); for (voted_hash, vote) in latest_votes.iter() { // if the latest votes correspond to a child @@ -426,7 +423,7 @@ impl ForkChoice for BitwiseLMDGhost { current_head = self .choose_best_child(&child_votes) .ok_or(ForkChoiceError::CannotFindBestChild)?; - trace!("FORKCHOICE: Best child found: {}", current_head); + trace!("Best child found: {}", current_head); } // didn't find head yet, proceed to next iteration @@ -441,7 +438,7 @@ impl ForkChoice for BitwiseLMDGhost { // more specifically, only keep votes that have head as an ancestor for hash in latest_votes.keys() { trace!( - "FORKCHOICE: Ancestor for vote: {} at height: {} is: {:?}", + "Ancestor for vote: {} at height: {} is: {:?}", hash, block_height, self.get_ancestor(*hash, block_height, spec) diff --git a/eth2/fork_choice/src/slow_lmd_ghost.rs b/eth2/fork_choice/src/slow_lmd_ghost.rs index 7cad73285..3aafb3924 100644 --- a/eth2/fork_choice/src/slow_lmd_ghost.rs +++ b/eth2/fork_choice/src/slow_lmd_ghost.rs @@ -78,7 +78,7 @@ where } } } - trace!("FORKCHOICE: Latest votes: {:?}", latest_votes); + trace!("Latest votes: {:?}", latest_votes); Ok(latest_votes) } @@ -139,7 +139,7 @@ impl ForkChoice for SlowLMDGhost { // simply add the attestation to the latest_attestation_target if the block_height is // larger trace!( - "FORKCHOICE: Adding attestation of validator: {:?} for block: {}", + "Adding attestation of validator: {:?} for block: {}", validator_index, target_block_root ); @@ -149,10 +149,7 @@ impl ForkChoice for SlowLMDGhost { .or_insert_with(|| *target_block_root); // if we already have a value if attestation_target != target_block_root { - trace!( - "FORKCHOICE: Old attestation found: {:?}", - attestation_target - ); + trace!("Old attestation found: {:?}", attestation_target); // get the height of the target block let block_height = self .block_store @@ -170,7 +167,7 @@ impl ForkChoice for SlowLMDGhost { .height(spec.genesis_slot); // update the attestation only if the new target is higher if past_block_height < block_height { - trace!("FORKCHOICE: Updating old attestation"); + trace!("Updating old attestation"); *attestation_target = *target_block_root; } } @@ -183,7 +180,7 @@ impl ForkChoice for SlowLMDGhost { justified_block_start: &Hash256, spec: &ChainSpec, ) -> Result { - debug!("FORKCHOICE: Running LMD Ghost Fork-choice rule"); + debug!("Running LMD Ghost Fork-choice rule"); let start = self .block_store .get_deserialized(&justified_block_start)? @@ -196,7 +193,7 @@ impl ForkChoice for SlowLMDGhost { let mut head_hash = *justified_block_start; loop { - debug!("FORKCHOICE: Iteration for block: {}", head_hash); + debug!("Iteration for block: {}", head_hash); let children = match self.children.get(&head_hash) { Some(children) => children, @@ -206,20 +203,16 @@ impl ForkChoice for SlowLMDGhost { // if we only have one child, use it if children.len() == 1 { - trace!("FORKCHOICE: Single child found."); + trace!("Single child found."); head_hash = children[0]; continue; } - trace!("FORKCHOICE: Children found: {:?}", children); + trace!("Children found: {:?}", children); let mut head_vote_count = 0; for child_hash in children { let vote_count = self.get_vote_count(&latest_votes, &child_hash)?; - trace!( - "FORKCHOICE: Vote count for child: {} is: {}", - child_hash, - vote_count - ); + trace!("Vote count for child: {} is: {}", child_hash, vote_count); if vote_count > head_vote_count { head_hash = *child_hash; From efa8aa19e73f98cdd085cf00fff951bcbf5da3d6 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 20 Feb 2019 12:52:03 +1100 Subject: [PATCH 39/39] Corrects small comments. --- eth2/fork_choice/src/bitwise_lmd_ghost.rs | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/eth2/fork_choice/src/bitwise_lmd_ghost.rs b/eth2/fork_choice/src/bitwise_lmd_ghost.rs index c467545ea..e1d246e92 100644 --- a/eth2/fork_choice/src/bitwise_lmd_ghost.rs +++ b/eth2/fork_choice/src/bitwise_lmd_ghost.rs @@ -201,13 +201,14 @@ where None } - // Finds the best child, splitting children into a binary tree, based on their hashes + // Finds the best child, splitting children into a binary tree, based on their hashes (Bitwise + // LMD Ghost) fn choose_best_child(&self, votes: &HashMap) -> Option { if votes.is_empty() { return None; } let mut bitmask: BitVec = BitVec::new(); - // loop through bytes then bits + // loop through all bits for bit in 0..=256 { let mut zero_votes = 0; let mut one_votes = 0; @@ -216,16 +217,8 @@ where trace!("Child vote length: {}", votes.len()); for (candidate, votes) in votes.iter() { let candidate_bit: BitVec = BitVec::from_bytes(&candidate); - /* - trace!( - "Child: {} in bits: {:?}", - candidate, - candidate_bit - ); - trace!("Current bitmask: {:?}", bitmask); - */ - // if the bitmasks don't match + // if the bitmasks don't match, exclude candidate if !bitmask.iter().eq(candidate_bit.iter().take(bit)) { trace!( "Child: {} was removed in bit: {} with the bitmask: {:?}",