From 59d6b1fdd07925e6064a7ddf6809f75ae6a4a787 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 16:25:08 +1300 Subject: [PATCH 1/6] Move `BeaconStateBuilder` out of test mods This allows it to be accessed by other crates/tests. --- eth2/types/src/beacon_state.rs | 3 + eth2/types/src/beacon_state/builder.rs | 215 +++++++++++++++++++++++++ eth2/types/src/beacon_state/tests.rs | 67 +------- 3 files changed, 221 insertions(+), 64 deletions(-) create mode 100644 eth2/types/src/beacon_state/builder.rs diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 85b49bf01..956877a61 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -14,6 +14,9 @@ use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; use std::collections::HashMap; use swap_or_not_shuffle::get_permutated_index; +pub use builder::BeaconStateBuilder; + +mod builder; mod epoch_cache; mod tests; diff --git a/eth2/types/src/beacon_state/builder.rs b/eth2/types/src/beacon_state/builder.rs new file mode 100644 index 000000000..6258d819d --- /dev/null +++ b/eth2/types/src/beacon_state/builder.rs @@ -0,0 +1,215 @@ +use crate::*; +use bls::create_proof_of_possession; + +/// Builds a `BeaconState` for use in testing or benchmarking. +/// +/// Building the `BeaconState` is a three step processes: +/// +/// 1. Create a new `BeaconStateBuilder`. +/// 2. Run the `genesis` function to generate a new BeaconState. +/// 3. (Optional) Use builder functions to modify the `BeaconState`. +/// 4. Call `build()` to obtain a cloned `BeaconState`. +/// +/// Step (2) is necessary because step (3) requires an existing `BeaconState` object. (2) is not +/// included in (1) to allow for modifying params before generating the `BeaconState` (e.g., the +/// spec). +/// +/// Step (4) produces a clone of the BeaconState and doesn't consume the `BeaconStateBuilder` to +/// allow access to the `keypairs` and `spec`. +pub struct BeaconStateBuilder { + pub state: Option, + pub genesis_time: u64, + pub initial_validator_deposits: Vec, + pub latest_eth1_data: Eth1Data, + pub spec: ChainSpec, + pub keypairs: Vec, +} + +impl BeaconStateBuilder { + /// Create a new builder with the given number of validators. + pub fn with_random_validators(validator_count: usize) -> Self { + let genesis_time = 10_000_000; + let keypairs: Vec = (0..validator_count) + .collect::>() + .iter() + .map(|_| Keypair::random()) + .collect(); + let initial_validator_deposits = keypairs + .iter() + .map(|keypair| Deposit { + branch: vec![], // branch verification is not specified. + index: 0, // index verification is not specified. + deposit_data: DepositData { + amount: 32_000_000_000, // 32 ETH (in Gwei) + timestamp: genesis_time - 1, + deposit_input: DepositInput { + pubkey: keypair.pk.clone(), + withdrawal_credentials: Hash256::zero(), // Withdrawal not possible. + proof_of_possession: create_proof_of_possession(&keypair), + }, + }, + }) + .collect(); + let latest_eth1_data = Eth1Data { + deposit_root: Hash256::zero(), + block_hash: Hash256::zero(), + }; + let spec = ChainSpec::foundation(); + + Self { + state: None, + genesis_time, + initial_validator_deposits, + latest_eth1_data, + spec, + keypairs, + } + } + + /// Runs the `BeaconState::genesis` function and produces a `BeaconState`. + pub fn genesis(&mut self) -> Result<(), BeaconStateError> { + let state = BeaconState::genesis( + self.genesis_time, + self.initial_validator_deposits.clone(), + self.latest_eth1_data.clone(), + &self.spec, + )?; + + self.state = Some(state); + + Ok(()) + } + + /// Sets the `BeaconState` to be in the last slot of the given epoch. + /// + /// Sets all justification/finalization parameters to be be as "perfect" as possible (i.e., + /// highest justified and finalized slots, full justification bitfield, etc). + pub fn teleport_to_end_of_epoch(&mut self, epoch: Epoch) { + let state = self.state.as_mut().expect("Genesis required"); + + let slot = epoch.end_slot(self.spec.epoch_length); + + state.slot = slot; + state.validator_registry_update_epoch = epoch - 1; + + state.previous_calculation_epoch = epoch - 1; + state.current_calculation_epoch = epoch; + + state.previous_epoch_seed = Hash256::from(&b"previous_seed"[..]); + state.current_epoch_seed = Hash256::from(&b"current_seed"[..]); + + state.previous_justified_epoch = epoch - 2; + state.justified_epoch = epoch - 1; + state.justification_bitfield = u64::max_value(); + state.finalized_epoch = epoch - 1; + } + + /// Creates a full set of attestations for the `BeaconState`. Each attestation has full + /// participation from its committee and references the expected beacon_block hashes. + /// + /// These attestations should be fully conducive to justification and finalization. + pub fn insert_attestations(&mut self) { + let state = self.state.as_mut().expect("Genesis required"); + + state + .build_epoch_cache(RelativeEpoch::Previous, &self.spec) + .unwrap(); + state + .build_epoch_cache(RelativeEpoch::Current, &self.spec) + .unwrap(); + + let current_epoch = state.current_epoch(&self.spec); + let previous_epoch = state.previous_epoch(&self.spec); + let current_epoch_depth = + (state.slot - current_epoch.end_slot(self.spec.epoch_length)).as_usize(); + + let previous_epoch_slots = previous_epoch.slot_iter(self.spec.epoch_length); + let current_epoch_slots = current_epoch + .slot_iter(self.spec.epoch_length) + .take(current_epoch_depth); + + for slot in previous_epoch_slots.chain(current_epoch_slots) { + let committees = state + .get_crosslink_committees_at_slot(slot, &self.spec) + .unwrap() + .clone(); + + for (committee, shard) in committees { + state + .latest_attestations + .push(committee_to_pending_attestation( + state, &committee, shard, slot, &self.spec, + )) + } + } + } + + /// Returns a cloned `BeaconState`. + pub fn build(&self) -> Result { + match &self.state { + Some(state) => Ok(state.clone()), + None => panic!("Genesis required"), + } + } +} + +/// Builds a valid PendingAttestation with full participation for some committee. +fn committee_to_pending_attestation( + state: &BeaconState, + committee: &Vec, + shard: u64, + slot: Slot, + spec: &ChainSpec, +) -> PendingAttestation { + let current_epoch = state.current_epoch(spec); + let previous_epoch = state.previous_epoch(spec); + + let mut aggregation_bitfield = Bitfield::new(); + let mut custody_bitfield = Bitfield::new(); + + for (i, _) in committee.iter().enumerate() { + aggregation_bitfield.set(i, true); + custody_bitfield.set(i, true); + } + + let is_previous_epoch = state.slot.epoch(spec.epoch_length) != slot.epoch(spec.epoch_length); + + let justified_epoch = if is_previous_epoch { + state.previous_justified_epoch + } else { + state.justified_epoch + }; + + let epoch_boundary_root = if is_previous_epoch { + *state + .get_block_root(previous_epoch.start_slot(spec.epoch_length), spec) + .unwrap() + } else { + *state + .get_block_root(current_epoch.start_slot(spec.epoch_length), spec) + .unwrap() + }; + + let justified_block_root = *state + .get_block_root(justified_epoch.start_slot(spec.epoch_length), &spec) + .unwrap(); + + PendingAttestation { + aggregation_bitfield, + data: AttestationData { + slot, + shard, + beacon_block_root: *state.get_block_root(slot, spec).unwrap(), + epoch_boundary_root, + shard_block_root: Hash256::zero(), + latest_crosslink: Crosslink { + epoch: slot.epoch(spec.epoch_length), + shard_block_root: Hash256::zero(), + }, + justified_epoch, + justified_block_root, + }, + custody_bitfield, + inclusion_slot: slot, + } +} diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs index a4a43a8ed..7f43ae9a1 100644 --- a/eth2/types/src/beacon_state/tests.rs +++ b/eth2/types/src/beacon_state/tests.rs @@ -2,73 +2,12 @@ use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; -use crate::{ - BeaconState, BeaconStateError, ChainSpec, Deposit, DepositData, DepositInput, Eth1Data, - Hash256, Keypair, -}; -use bls::create_proof_of_possession; +use crate::{BeaconState, ChainSpec}; use ssz::{ssz_encode, Decodable}; -struct BeaconStateTestBuilder { - pub genesis_time: u64, - pub initial_validator_deposits: Vec, - pub latest_eth1_data: Eth1Data, - pub spec: ChainSpec, - pub keypairs: Vec, -} - -impl BeaconStateTestBuilder { - pub fn with_random_validators(validator_count: usize) -> Self { - let genesis_time = 10_000_000; - let keypairs: Vec = (0..validator_count) - .collect::>() - .iter() - .map(|_| Keypair::random()) - .collect(); - let initial_validator_deposits = keypairs - .iter() - .map(|keypair| Deposit { - branch: vec![], // branch verification is not specified. - index: 0, // index verification is not specified. - deposit_data: DepositData { - amount: 32_000_000_000, // 32 ETH (in Gwei) - timestamp: genesis_time - 1, - deposit_input: DepositInput { - pubkey: keypair.pk.clone(), - withdrawal_credentials: Hash256::zero(), // Withdrawal not possible. - proof_of_possession: create_proof_of_possession(&keypair), - }, - }, - }) - .collect(); - let latest_eth1_data = Eth1Data { - deposit_root: Hash256::zero(), - block_hash: Hash256::zero(), - }; - let spec = ChainSpec::foundation(); - - Self { - genesis_time, - initial_validator_deposits, - latest_eth1_data, - spec, - keypairs, - } - } - - pub fn build(&self) -> Result { - BeaconState::genesis( - self.genesis_time, - self.initial_validator_deposits.clone(), - self.latest_eth1_data.clone(), - &self.spec, - ) - } -} - #[test] pub fn can_produce_genesis_block() { - let builder = BeaconStateTestBuilder::with_random_validators(2); + let builder = BeaconStateBuilder::with_random_validators(2); builder.build().unwrap(); } @@ -79,7 +18,7 @@ pub fn can_produce_genesis_block() { pub fn get_attestation_participants_consistency() { let mut rng = XorShiftRng::from_seed([42; 16]); - let mut builder = BeaconStateTestBuilder::with_random_validators(8); + let mut builder = BeaconStateBuilder::with_random_validators(8); builder.spec = ChainSpec::few_validators(); let mut state = builder.build().unwrap(); From 53663e54b5d23622776fc294a21b4263ec5cf602 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 16:26:06 +1300 Subject: [PATCH 2/6] Fix error with get_permutated_index. Error types were modified. Error involved shuffling with the _value_of active validator indicies, not an _index_. --- eth2/types/src/beacon_state.rs | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 956877a61..f8d71c324 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -7,7 +7,7 @@ use crate::{ }; use bls::verify_proof_of_possession; use honey_badger_split::SplitExt; -use log::{debug, trace}; +use log::{debug, trace, error}; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; @@ -387,12 +387,13 @@ impl BeaconState { seed: Hash256, epoch: Epoch, spec: &ChainSpec, - ) -> Option>> { + ) -> Result>, Error> { let active_validator_indices = get_active_validator_indices(&self.validator_registry, epoch); if active_validator_indices.is_empty() { - return None; + error!("get_shuffling: no validators."); + return Err(Error::InsufficientValidators); } let committees_per_epoch = @@ -405,22 +406,21 @@ impl BeaconState { ); let mut shuffled_active_validator_indices = vec![0; active_validator_indices.len()]; - for &i in &active_validator_indices { + for (i, _) in active_validator_indices.iter().enumerate() { let shuffled_i = get_permutated_index( i, active_validator_indices.len(), &seed[..], spec.shuffle_round_count, - )?; + ) + .ok_or_else(|| Error::UnableToShuffle)?; shuffled_active_validator_indices[i] = active_validator_indices[shuffled_i] } - Some( - shuffled_active_validator_indices - .honey_badger_split(committees_per_epoch as usize) - .map(|slice: &[usize]| slice.to_vec()) - .collect(), - ) + Ok(shuffled_active_validator_indices + .honey_badger_split(committees_per_epoch as usize) + .map(|slice: &[usize]| slice.to_vec()) + .collect()) } /// Return the number of committees in the previous epoch. @@ -525,7 +525,6 @@ impl BeaconState { self.get_committee_params_at_slot(slot, registry_change, spec)?; self.get_shuffling(seed, shuffling_epoch, spec) - .ok_or_else(|| Error::UnableToShuffle) } /// Returns the following params for the given slot: From 906131f8826bac9f338075cb0c045d3739dc2948 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 16:27:42 +1300 Subject: [PATCH 3/6] Add tests and benches for epoch processing --- eth2/state_processing/Cargo.toml | 9 ++++++ eth2/state_processing/benches/benches.rs | 28 +++++++++++++++++++ .../state_processing/src/epoch_processable.rs | 10 ++----- .../src/epoch_processable/tests.rs | 21 ++++++++++++++ 4 files changed, 60 insertions(+), 8 deletions(-) create mode 100644 eth2/state_processing/benches/benches.rs create mode 100644 eth2/state_processing/src/epoch_processable/tests.rs diff --git a/eth2/state_processing/Cargo.toml b/eth2/state_processing/Cargo.toml index b6b0ea57c..85ac8b42a 100644 --- a/eth2/state_processing/Cargo.toml +++ b/eth2/state_processing/Cargo.toml @@ -4,6 +4,15 @@ version = "0.1.0" authors = ["Paul Hauner "] edition = "2018" +[[bench]] +name = "benches" +harness = false + +[dev-dependencies] +criterion = "0.2" +test_harness = { path = "../../beacon_node/beacon_chain/test_harness" } +env_logger = "0.6.0" + [dependencies] hashing = { path = "../utils/hashing" } int_to_bytes = { path = "../utils/int_to_bytes" } diff --git a/eth2/state_processing/benches/benches.rs b/eth2/state_processing/benches/benches.rs new file mode 100644 index 000000000..29f9a1e86 --- /dev/null +++ b/eth2/state_processing/benches/benches.rs @@ -0,0 +1,28 @@ +use criterion::Criterion; +use criterion::{black_box, criterion_group, criterion_main}; +// use env_logger::{Builder, Env}; +use state_processing::SlotProcessable; +use types::{beacon_state::BeaconStateBuilder, ChainSpec, Hash256}; + +fn epoch_processing(c: &mut Criterion) { + // Builder::from_env(Env::default().default_filter_or("debug")).init(); + + let mut builder = BeaconStateBuilder::with_random_validators(8); + builder.spec = ChainSpec::few_validators(); + + builder.genesis().unwrap(); + builder.teleport_to_end_of_epoch(builder.spec.genesis_epoch + 4); + + let state = builder.build().unwrap(); + + c.bench_function("epoch processing", move |b| { + let spec = &builder.spec; + b.iter_with_setup( + || state.clone(), + |mut state| black_box(state.per_slot_processing(Hash256::zero(), spec).unwrap()), + ) + }); +} + +criterion_group!(benches, epoch_processing,); +criterion_main!(benches); diff --git a/eth2/state_processing/src/epoch_processable.rs b/eth2/state_processing/src/epoch_processable.rs index 9b6c98c86..2cee455a3 100644 --- a/eth2/state_processing/src/epoch_processable.rs +++ b/eth2/state_processing/src/epoch_processable.rs @@ -9,6 +9,8 @@ use types::{ Crosslink, Epoch, Hash256, InclusionError, PendingAttestation, RelativeEpoch, }; +mod tests; + macro_rules! safe_add_assign { ($a: expr, $b: expr) => { $a = $a.saturating_add($b); @@ -725,11 +727,3 @@ impl From for WinningRootError { WinningRootError::BeaconStateError(e) } } - -#[cfg(test)] -mod tests { - #[test] - fn it_works() { - assert_eq!(2 + 2, 4); - } -} diff --git a/eth2/state_processing/src/epoch_processable/tests.rs b/eth2/state_processing/src/epoch_processable/tests.rs new file mode 100644 index 000000000..353672fd1 --- /dev/null +++ b/eth2/state_processing/src/epoch_processable/tests.rs @@ -0,0 +1,21 @@ +#![cfg(test)] +use crate::EpochProcessable; +use env_logger::{Builder, Env}; +use types::beacon_state::BeaconStateBuilder; +use types::*; + +#[test] +fn runs_without_error() { + Builder::from_env(Env::default().default_filter_or("error")).init(); + + let mut builder = BeaconStateBuilder::with_random_validators(8); + builder.spec = ChainSpec::few_validators(); + + builder.genesis().unwrap(); + builder.teleport_to_end_of_epoch(builder.spec.genesis_epoch + 4); + + let mut state = builder.build().unwrap(); + + let spec = &builder.spec; + state.per_epoch_processing(spec).unwrap(); +} From 9f846eda2664e2c0e91b90507e84a16ab4ee5585 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 16:49:17 +1300 Subject: [PATCH 4/6] Fix clippy lint --- eth2/types/src/beacon_state/builder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth2/types/src/beacon_state/builder.rs b/eth2/types/src/beacon_state/builder.rs index 6258d819d..7273f3658 100644 --- a/eth2/types/src/beacon_state/builder.rs +++ b/eth2/types/src/beacon_state/builder.rs @@ -156,7 +156,7 @@ impl BeaconStateBuilder { /// Builds a valid PendingAttestation with full participation for some committee. fn committee_to_pending_attestation( state: &BeaconState, - committee: &Vec, + committee: &[usize], shard: u64, slot: Slot, spec: &ChainSpec, From e97b554485d12698cd8daae9ae6bba752115f891 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 16:50:04 +1300 Subject: [PATCH 5/6] Run rustfmt --- eth2/types/src/beacon_state.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index f8d71c324..f7df3b0ab 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -7,7 +7,7 @@ use crate::{ }; use bls::verify_proof_of_possession; use honey_badger_split::SplitExt; -use log::{debug, trace, error}; +use log::{debug, error, trace}; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; From 65d303c50090c717542e5d9da43f65df472783f1 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 17:20:43 +1300 Subject: [PATCH 6/6] Fix failing tests --- eth2/types/src/beacon_state/tests.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs index 7f43ae9a1..01b4be803 100644 --- a/eth2/types/src/beacon_state/tests.rs +++ b/eth2/types/src/beacon_state/tests.rs @@ -7,7 +7,8 @@ use ssz::{ssz_encode, Decodable}; #[test] pub fn can_produce_genesis_block() { - let builder = BeaconStateBuilder::with_random_validators(2); + let mut builder = BeaconStateBuilder::with_random_validators(2); + builder.genesis().unwrap(); builder.build().unwrap(); } @@ -21,6 +22,8 @@ pub fn get_attestation_participants_consistency() { let mut builder = BeaconStateBuilder::with_random_validators(8); builder.spec = ChainSpec::few_validators(); + builder.genesis().unwrap(); + let mut state = builder.build().unwrap(); let spec = builder.spec.clone();