From f82c4268e29a8323f110c50ed0bfa0e7690f33f0 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 20:16:14 +1300 Subject: [PATCH 1/7] Split `BeaconState::genesis` into two functions This allows us to build a genesis state without inducting all the validators. This will be a signigicant speed up for tests as we can avoid generating keypairs and checking proof-of-possessions. --- eth2/types/src/beacon_state.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index f7df3b0ab..fad756708 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -117,19 +117,18 @@ pub struct BeaconState { impl BeaconState { /// Produce the first state of the Beacon Chain. - pub fn genesis( + pub fn genesis_without_validators( genesis_time: u64, - initial_validator_deposits: Vec, latest_eth1_data: Eth1Data, spec: &ChainSpec, ) -> Result { - debug!("Creating genesis state."); + debug!("Creating genesis state (without validator processing)."); let initial_crosslink = Crosslink { epoch: spec.genesis_epoch, shard_block_root: spec.zero_hash, }; - let mut genesis_state = BeaconState { + Ok(BeaconState { /* * Misc */ @@ -188,7 +187,19 @@ impl BeaconState { */ cache_index_offset: 0, caches: vec![EpochCache::empty(); CACHED_EPOCHS], - }; + }) + } + /// Produce the first state of the Beacon Chain. + pub fn genesis( + genesis_time: u64, + initial_validator_deposits: Vec, + latest_eth1_data: Eth1Data, + spec: &ChainSpec, + ) -> Result { + let mut genesis_state = + BeaconState::genesis_without_validators(genesis_time, latest_eth1_data, spec)?; + + trace!("Processing genesis deposits..."); let deposit_data = initial_validator_deposits .iter() From 931da13545d73ea02d4d43c687688f889d94e2ea Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 20:18:20 +1300 Subject: [PATCH 2/7] Add `drop_cache` fn to `BeaconState` Permits clearing an epoch cache. --- eth2/types/src/beacon_state.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index fad756708..3d94a8e3d 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -290,14 +290,18 @@ impl BeaconState { /// -- `Next` epoch is always _without_ a registry change. If you perform a registry update, /// you should rebuild the `Current` cache so it uses the new seed. pub fn advance_caches(&mut self) { - let previous_cache_index = self.cache_index(RelativeEpoch::Previous); - - self.caches[previous_cache_index] = EpochCache::empty(); + self.drop_cache(RelativeEpoch::Previous); self.cache_index_offset += 1; self.cache_index_offset %= CACHED_EPOCHS; } + /// Removes the specified cache and sets it to uninitialized. + pub fn drop_cache(&mut self, relative_epoch: RelativeEpoch) { + let previous_cache_index = self.cache_index(relative_epoch); + self.caches[previous_cache_index] = EpochCache::empty(); + } + /// Returns the index of `self.caches` for some `RelativeEpoch`. fn cache_index(&self, relative_epoch: RelativeEpoch) -> usize { let base_index = match relative_epoch { From 5cfc9cb21d166d4d145229240fd7488c3cea449b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 20:18:59 +1300 Subject: [PATCH 3/7] Test state processing with and w/o caches --- eth2/state_processing/benches/benches.rs | 36 ++++++++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/eth2/state_processing/benches/benches.rs b/eth2/state_processing/benches/benches.rs index 29f9a1e86..68a9735e1 100644 --- a/eth2/state_processing/benches/benches.rs +++ b/eth2/state_processing/benches/benches.rs @@ -2,7 +2,8 @@ use criterion::Criterion; use criterion::{black_box, criterion_group, criterion_main}; // use env_logger::{Builder, Env}; use state_processing::SlotProcessable; -use types::{beacon_state::BeaconStateBuilder, ChainSpec, Hash256}; +use types::*; +use types::beacon_state::BeaconStateBuilder; fn epoch_processing(c: &mut Criterion) { // Builder::from_env(Env::default().default_filter_or("debug")).init(); @@ -13,13 +14,36 @@ fn epoch_processing(c: &mut Criterion) { builder.genesis().unwrap(); builder.teleport_to_end_of_epoch(builder.spec.genesis_epoch + 4); - let state = builder.build().unwrap(); + let mut state = builder.build().unwrap(); - c.bench_function("epoch processing", move |b| { - let spec = &builder.spec; + // Build all the caches so the following state does _not_ include the cache-building time. + state.build_epoch_cache(RelativeEpoch::Previous, &builder.spec).unwrap(); + state.build_epoch_cache(RelativeEpoch::Current, &builder.spec).unwrap(); + state.build_epoch_cache(RelativeEpoch::Next, &builder.spec).unwrap(); + + let cached_state = state.clone(); + + // Drop all the caches so the following state includes the cache-building time. + state.drop_cache(RelativeEpoch::Previous); + state.drop_cache(RelativeEpoch::Current); + state.drop_cache(RelativeEpoch::Next); + + let cacheless_state = state; + + let spec_a = builder.spec.clone(); + let spec_b = builder.spec.clone(); + + c.bench_function("epoch processing with pre-built caches", move |b| { b.iter_with_setup( - || state.clone(), - |mut state| black_box(state.per_slot_processing(Hash256::zero(), spec).unwrap()), + || cached_state.clone(), + |mut state| black_box(state.per_slot_processing(Hash256::zero(), &spec_a).unwrap()), + ) + }); + + c.bench_function("epoch processing without pre-built caches", move |b| { + b.iter_with_setup( + || cacheless_state.clone(), + |mut state| black_box(state.per_slot_processing(Hash256::zero(), &spec_b).unwrap()), ) }); } From af17fb1d0301b96438c9028494bacaf4d7962965 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 22:00:07 +1300 Subject: [PATCH 4/7] Update `BeaconStateBuilder` API --- .../beacon_chain/test_harness/Cargo.toml | 1 + .../test_harness/benches/state_transition.rs | 1 + eth2/state_processing/benches/benches.rs | 20 ++-- .../src/epoch_processable/tests.rs | 6 +- eth2/types/src/beacon_state/builder.rs | 101 ++++++++++++------ eth2/types/src/beacon_state/tests.rs | 10 +- 6 files changed, 89 insertions(+), 50 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/Cargo.toml b/beacon_node/beacon_chain/test_harness/Cargo.toml index 77b52ccf6..657cc7955 100644 --- a/beacon_node/beacon_chain/test_harness/Cargo.toml +++ b/beacon_node/beacon_chain/test_harness/Cargo.toml @@ -10,6 +10,7 @@ harness = false [dev-dependencies] criterion = "0.2" +state_processing = { path = "../../../eth2/state_processing" } [dependencies] attester = { path = "../../../eth2/attester" } diff --git a/beacon_node/beacon_chain/test_harness/benches/state_transition.rs b/beacon_node/beacon_chain/test_harness/benches/state_transition.rs index 013ecfd1e..aa2a858fa 100644 --- a/beacon_node/beacon_chain/test_harness/benches/state_transition.rs +++ b/beacon_node/beacon_chain/test_harness/benches/state_transition.rs @@ -1,6 +1,7 @@ use criterion::Criterion; use criterion::{black_box, criterion_group, criterion_main, Benchmark}; // use env_logger::{Builder, Env}; +use state_processing::SlotProcessable; use test_harness::BeaconChainHarness; use types::{ChainSpec, Hash256}; diff --git a/eth2/state_processing/benches/benches.rs b/eth2/state_processing/benches/benches.rs index 68a9735e1..244e3c8a5 100644 --- a/eth2/state_processing/benches/benches.rs +++ b/eth2/state_processing/benches/benches.rs @@ -2,24 +2,30 @@ use criterion::Criterion; use criterion::{black_box, criterion_group, criterion_main}; // use env_logger::{Builder, Env}; use state_processing::SlotProcessable; -use types::*; use types::beacon_state::BeaconStateBuilder; +use types::*; fn epoch_processing(c: &mut Criterion) { // Builder::from_env(Env::default().default_filter_or("debug")).init(); - let mut builder = BeaconStateBuilder::with_random_validators(8); + let mut builder = BeaconStateBuilder::new(8); builder.spec = ChainSpec::few_validators(); - builder.genesis().unwrap(); + builder.build().unwrap(); builder.teleport_to_end_of_epoch(builder.spec.genesis_epoch + 4); - let mut state = builder.build().unwrap(); + let mut state = builder.cloned_state(); // Build all the caches so the following state does _not_ include the cache-building time. - state.build_epoch_cache(RelativeEpoch::Previous, &builder.spec).unwrap(); - state.build_epoch_cache(RelativeEpoch::Current, &builder.spec).unwrap(); - state.build_epoch_cache(RelativeEpoch::Next, &builder.spec).unwrap(); + state + .build_epoch_cache(RelativeEpoch::Previous, &builder.spec) + .unwrap(); + state + .build_epoch_cache(RelativeEpoch::Current, &builder.spec) + .unwrap(); + state + .build_epoch_cache(RelativeEpoch::Next, &builder.spec) + .unwrap(); let cached_state = state.clone(); diff --git a/eth2/state_processing/src/epoch_processable/tests.rs b/eth2/state_processing/src/epoch_processable/tests.rs index 353672fd1..d683d3971 100644 --- a/eth2/state_processing/src/epoch_processable/tests.rs +++ b/eth2/state_processing/src/epoch_processable/tests.rs @@ -8,13 +8,13 @@ use types::*; fn runs_without_error() { Builder::from_env(Env::default().default_filter_or("error")).init(); - let mut builder = BeaconStateBuilder::with_random_validators(8); + let mut builder = BeaconStateBuilder::new(8); builder.spec = ChainSpec::few_validators(); - builder.genesis().unwrap(); + builder.build().unwrap(); builder.teleport_to_end_of_epoch(builder.spec.genesis_epoch + 4); - let mut state = builder.build().unwrap(); + let mut state = builder.cloned_state(); let spec = &builder.spec; state.per_epoch_processing(spec).unwrap(); diff --git a/eth2/types/src/beacon_state/builder.rs b/eth2/types/src/beacon_state/builder.rs index 7273f3658..80e4d501f 100644 --- a/eth2/types/src/beacon_state/builder.rs +++ b/eth2/types/src/beacon_state/builder.rs @@ -6,20 +6,19 @@ use bls::create_proof_of_possession; /// Building the `BeaconState` is a three step processes: /// /// 1. Create a new `BeaconStateBuilder`. -/// 2. Run the `genesis` function to generate a new BeaconState. +/// 2. Call `Self::build()` or `Self::build_fast()` generate a `BeaconState`. /// 3. (Optional) Use builder functions to modify the `BeaconState`. -/// 4. Call `build()` to obtain a cloned `BeaconState`. +/// 4. Call `Self::cloned_state()` to obtain a `BeaconState` cloned from this struct. /// -/// Step (2) is necessary because step (3) requires an existing `BeaconState` object. (2) is not -/// included in (1) to allow for modifying params before generating the `BeaconState` (e.g., the -/// spec). +/// Step (2) happens prior to step (3) because some functionality requires an existing +/// `BeaconState`. /// /// Step (4) produces a clone of the BeaconState and doesn't consume the `BeaconStateBuilder` to -/// allow access to the `keypairs` and `spec`. +/// allow access to `self.keypairs` and `self.spec`. pub struct BeaconStateBuilder { + pub validator_count: usize, pub state: Option, pub genesis_time: u64, - pub initial_validator_deposits: Vec, pub latest_eth1_data: Eth1Data, pub spec: ChainSpec, pub keypairs: Vec, @@ -27,21 +26,41 @@ pub struct BeaconStateBuilder { impl BeaconStateBuilder { /// Create a new builder with the given number of validators. - pub fn with_random_validators(validator_count: usize) -> Self { + pub fn new(validator_count: usize) -> Self { let genesis_time = 10_000_000; - let keypairs: Vec = (0..validator_count) + + let latest_eth1_data = Eth1Data { + deposit_root: Hash256::zero(), + block_hash: Hash256::zero(), + }; + + let spec = ChainSpec::foundation(); + + Self { + validator_count, + state: None, + genesis_time, + latest_eth1_data, + spec, + keypairs: vec![], + } + } + + pub fn build(&mut self) -> Result<(), BeaconStateError> { + self.keypairs = (0..self.validator_count) .collect::>() .iter() .map(|_| Keypair::random()) .collect(); - let initial_validator_deposits = keypairs + + let initial_validator_deposits = self.keypairs .iter() .map(|keypair| Deposit { branch: vec![], // branch verification is not specified. index: 0, // index verification is not specified. deposit_data: DepositData { amount: 32_000_000_000, // 32 ETH (in Gwei) - timestamp: genesis_time - 1, + timestamp: self.genesis_time - 1, deposit_input: DepositInput { pubkey: keypair.pk.clone(), withdrawal_credentials: Hash256::zero(), // Withdrawal not possible. @@ -50,27 +69,10 @@ impl BeaconStateBuilder { }, }) .collect(); - let latest_eth1_data = Eth1Data { - deposit_root: Hash256::zero(), - block_hash: Hash256::zero(), - }; - let spec = ChainSpec::foundation(); - Self { - state: None, - genesis_time, - initial_validator_deposits, - latest_eth1_data, - spec, - keypairs, - } - } - - /// Runs the `BeaconState::genesis` function and produces a `BeaconState`. - pub fn genesis(&mut self) -> Result<(), BeaconStateError> { let state = BeaconState::genesis( self.genesis_time, - self.initial_validator_deposits.clone(), + initial_validator_deposits, self.latest_eth1_data.clone(), &self.spec, )?; @@ -80,6 +82,40 @@ impl BeaconStateBuilder { Ok(()) } + pub fn build_fast(&mut self) -> Result<(), BeaconStateError> { + let common_keypair = Keypair::random(); + + let mut validator_registry = Vec::with_capacity(self.validator_count); + let mut validator_balances = Vec::with_capacity(self.validator_count); + self.keypairs = Vec::with_capacity(self.validator_count); + + for _ in 0..self.validator_count { + self.keypairs.push(common_keypair.clone()); + validator_balances.push(32_000_000_000); + validator_registry.push(Validator { + pubkey: common_keypair.pk.clone(), + withdrawal_credentials: Hash256::zero(), + activation_epoch: self.spec.genesis_epoch, + ..Validator::default() + }) + } + + let state = BeaconState { + validator_registry, + validator_balances, + ..BeaconState::genesis( + self.genesis_time, + vec![], + self.latest_eth1_data.clone(), + &self.spec, + )? + }; + + self.state = Some(state); + + Ok(()) + } + /// Sets the `BeaconState` to be in the last slot of the given epoch. /// /// Sets all justification/finalization parameters to be be as "perfect" as possible (i.e., @@ -145,11 +181,8 @@ impl BeaconStateBuilder { } /// Returns a cloned `BeaconState`. - pub fn build(&self) -> Result { - match &self.state { - Some(state) => Ok(state.clone()), - None => panic!("Genesis required"), - } + pub fn cloned_state(&self) -> BeaconState { + self.state.as_ref().expect("Genesis required").clone() } } diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs index 01b4be803..bb8561511 100644 --- a/eth2/types/src/beacon_state/tests.rs +++ b/eth2/types/src/beacon_state/tests.rs @@ -7,9 +7,7 @@ use ssz::{ssz_encode, Decodable}; #[test] pub fn can_produce_genesis_block() { - let mut builder = BeaconStateBuilder::with_random_validators(2); - builder.genesis().unwrap(); - + let mut builder = BeaconStateBuilder::new(2); builder.build().unwrap(); } @@ -19,12 +17,12 @@ pub fn can_produce_genesis_block() { pub fn get_attestation_participants_consistency() { let mut rng = XorShiftRng::from_seed([42; 16]); - let mut builder = BeaconStateBuilder::with_random_validators(8); + let mut builder = BeaconStateBuilder::new(8); builder.spec = ChainSpec::few_validators(); - builder.genesis().unwrap(); + builder.build().unwrap(); - let mut state = builder.build().unwrap(); + let mut state = builder.cloned_state(); let spec = builder.spec.clone(); state From 3ff8f6ebb326cf429fe9a7a8038683c4b3b7cfad Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 22:25:29 +1300 Subject: [PATCH 5/7] Update epoch trans. tests to us 16,384 validators --- eth2/state_processing/benches/benches.rs | 39 ++++++++++++++---------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/eth2/state_processing/benches/benches.rs b/eth2/state_processing/benches/benches.rs index 244e3c8a5..682259eef 100644 --- a/eth2/state_processing/benches/benches.rs +++ b/eth2/state_processing/benches/benches.rs @@ -1,5 +1,5 @@ use criterion::Criterion; -use criterion::{black_box, criterion_group, criterion_main}; +use criterion::{black_box, criterion_group, criterion_main, Benchmark}; // use env_logger::{Builder, Env}; use state_processing::SlotProcessable; use types::beacon_state::BeaconStateBuilder; @@ -8,10 +8,9 @@ use types::*; fn epoch_processing(c: &mut Criterion) { // Builder::from_env(Env::default().default_filter_or("debug")).init(); - let mut builder = BeaconStateBuilder::new(8); - builder.spec = ChainSpec::few_validators(); + let mut builder = BeaconStateBuilder::new(16_384); - builder.build().unwrap(); + builder.build_fast().unwrap(); builder.teleport_to_end_of_epoch(builder.spec.genesis_epoch + 4); let mut state = builder.cloned_state(); @@ -39,19 +38,27 @@ fn epoch_processing(c: &mut Criterion) { let spec_a = builder.spec.clone(); let spec_b = builder.spec.clone(); - c.bench_function("epoch processing with pre-built caches", move |b| { - b.iter_with_setup( - || cached_state.clone(), - |mut state| black_box(state.per_slot_processing(Hash256::zero(), &spec_a).unwrap()), - ) - }); + c.bench( + "epoch processing", + Benchmark::new("with pre-built caches", move |b| { + b.iter_with_setup( + || cached_state.clone(), + |mut state| black_box(state.per_slot_processing(Hash256::zero(), &spec_a).unwrap()), + ) + }) + .sample_size(10), + ); - c.bench_function("epoch processing without pre-built caches", move |b| { - b.iter_with_setup( - || cacheless_state.clone(), - |mut state| black_box(state.per_slot_processing(Hash256::zero(), &spec_b).unwrap()), - ) - }); + c.bench( + "epoch processing", + Benchmark::new("without pre-built caches", move |b| { + b.iter_with_setup( + || cacheless_state.clone(), + |mut state| black_box(state.per_slot_processing(Hash256::zero(), &spec_b).unwrap()), + ) + }) + .sample_size(10), + ); } criterion_group!(benches, epoch_processing,); From c2fb09575947fab58289a2d0e6334da56f1b0458 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 22:34:41 +1300 Subject: [PATCH 6/7] Add docstrings to `BeaconStateBuilder` --- eth2/types/src/beacon_state/builder.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/eth2/types/src/beacon_state/builder.rs b/eth2/types/src/beacon_state/builder.rs index 80e4d501f..40bfe0d2c 100644 --- a/eth2/types/src/beacon_state/builder.rs +++ b/eth2/types/src/beacon_state/builder.rs @@ -46,6 +46,10 @@ impl BeaconStateBuilder { } } + /// Builds a `BeaconState` using the `BeaconState::genesis(..)` function. + /// + /// Each validator is assigned a unique, randomly-generated keypair and all + /// proof-of-possessions are verified during genesis. pub fn build(&mut self) -> Result<(), BeaconStateError> { self.keypairs = (0..self.validator_count) .collect::>() @@ -53,7 +57,8 @@ impl BeaconStateBuilder { .map(|_| Keypair::random()) .collect(); - let initial_validator_deposits = self.keypairs + let initial_validator_deposits = self + .keypairs .iter() .map(|keypair| Deposit { branch: vec![], // branch verification is not specified. @@ -82,6 +87,16 @@ impl BeaconStateBuilder { Ok(()) } + /// Builds a `BeaconState` using the `BeaconState::genesis(..)` function, without supplying any + /// validators. Instead validators are added to the state post-genesis. + /// + /// One keypair is randomly generated and all validators are assigned this same keypair. + /// Proof-of-possessions are not created (or validated). + /// + /// This function runs orders of magnitude faster than `Self::build()`, however it will be + /// erroneous for functions which use a validators public key as an identifier (e.g., + /// deposits). + /// proof-of-possessions are verified during genesis. pub fn build_fast(&mut self) -> Result<(), BeaconStateError> { let common_keypair = Keypair::random(); From 04f179243e4cd4bf0f6523efcf650b4f5d1125d5 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 22:35:51 +1300 Subject: [PATCH 7/7] Fix type in `BeaconStateBuilder` comments --- eth2/types/src/beacon_state/builder.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/eth2/types/src/beacon_state/builder.rs b/eth2/types/src/beacon_state/builder.rs index 40bfe0d2c..02886a86e 100644 --- a/eth2/types/src/beacon_state/builder.rs +++ b/eth2/types/src/beacon_state/builder.rs @@ -96,7 +96,6 @@ impl BeaconStateBuilder { /// This function runs orders of magnitude faster than `Self::build()`, however it will be /// erroneous for functions which use a validators public key as an identifier (e.g., /// deposits). - /// proof-of-possessions are verified during genesis. pub fn build_fast(&mut self) -> Result<(), BeaconStateError> { let common_keypair = Keypair::random();