diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index a4804e07e..b76bc3e82 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -19,6 +19,7 @@ slog = "^2.2.3" slot_clock = { path = "../eth2/utils/slot_clock" } slog-term = "^2.4.0" slog-async = "^2.3.0" +state_processing = { path = "../eth2/state_processing" } types = { path = "../eth2/types" } ssz = { path = "../eth2/utils/ssz" } tokio = "0.1" diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index 780a3d338..2436d4f7c 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -18,10 +18,8 @@ use slog::{error, info, o, Drain}; use slot_clock::SystemTimeSlotClock; use ssz::TreeHash; use std::sync::Arc; -use types::{ - beacon_state::BeaconStateBuilder, BeaconBlock, ChainSpec, Deposit, DepositData, DepositInput, - Eth1Data, Fork, Hash256, Keypair, -}; +use types::test_utils::TestingBeaconStateBuilder; +use types::*; fn main() { let decorator = slog_term::TermDecorator::new().build(); @@ -79,61 +77,18 @@ fn main() { let block_store = Arc::new(BeaconBlockStore::new(db.clone())); let state_store = Arc::new(BeaconStateStore::new(db.clone())); + let state_builder = TestingBeaconStateBuilder::from_deterministic_keypairs(8, &spec); + let (genesis_state, _keypairs) = state_builder.build(); + + let mut genesis_block = BeaconBlock::empty(&spec); + genesis_block.state_root = Hash256::from_slice(&genesis_state.hash_tree_root()); + // Slot clock - let genesis_time = 1_549_935_547; // 12th Feb 2018 (arbitrary value in the past). - let slot_clock = SystemTimeSlotClock::new(genesis_time, spec.seconds_per_slot) + let slot_clock = SystemTimeSlotClock::new(genesis_state.genesis_time, spec.seconds_per_slot) .expect("Unable to load SystemTimeSlotClock"); // Choose the fork choice let fork_choice = BitwiseLMDGhost::new(block_store.clone(), state_store.clone()); - /* - * Generate some random data to start a chain with. - * - * This is will need to be replace for production usage. - */ - let latest_eth1_data = Eth1Data { - deposit_root: Hash256::zero(), - block_hash: Hash256::zero(), - }; - let keypairs: Vec = (0..10) - .collect::>() - .iter() - .map(|_| Keypair::random()) - .collect(); - - let initial_validator_deposits: Vec = keypairs - .iter() - .map(|keypair| { - let mut deposit_input = DepositInput { - pubkey: keypair.pk.clone(), - withdrawal_credentials: Hash256::zero(), - proof_of_possession: spec.empty_signature.clone(), - }; - deposit_input.proof_of_possession = deposit_input.create_proof_of_possession( - &keypair.sk, - spec.genesis_epoch, - &Fork::genesis(&spec), - &spec, - ); - - Deposit { - proof: vec![], // branch verification is not specified. - index: 0, // index verification is not specified. - deposit_data: DepositData { - amount: 32_000_000_000, // 32 ETH (in Gwei) - timestamp: genesis_time - 1, - deposit_input, - }, - } - }) - .collect(); - - let mut state_builder = BeaconStateBuilder::new(genesis_time, latest_eth1_data, &spec); - state_builder.process_initial_deposits(&initial_validator_deposits, &spec); - let genesis_state = state_builder.build(&spec).unwrap(); - let mut genesis_block = BeaconBlock::empty(&spec); - genesis_block.state_root = Hash256::from_slice(&genesis_state.hash_tree_root()); - // Genesis chain let _chain_result = BeaconChain::from_genesis( state_store.clone(), diff --git a/eth2/state_processing/src/get_genesis_state.rs b/eth2/state_processing/src/get_genesis_state.rs new file mode 100644 index 000000000..3c6612349 --- /dev/null +++ b/eth2/state_processing/src/get_genesis_state.rs @@ -0,0 +1,59 @@ +use super::per_block_processing::{errors::BlockProcessingError, process_deposits}; +use ssz::TreeHash; +use types::*; + +pub enum GenesisError { + BlockProcessingError(BlockProcessingError), + BeaconStateError(BeaconStateError), +} + +/// Returns the genesis `BeaconState` +/// +/// Spec v0.5.0 +pub fn get_genesis_state( + genesis_validator_deposits: &[Deposit], + genesis_time: u64, + genesis_eth1_data: Eth1Data, + spec: &ChainSpec, +) -> Result<(), BlockProcessingError> { + // Get the genesis `BeaconState` + let mut state = BeaconState::genesis(genesis_time, genesis_eth1_data, spec); + + // Process genesis deposits. + process_deposits(&mut state, genesis_validator_deposits, spec)?; + + // Process genesis activations. + for i in 0..state.validator_registry.len() { + if state.get_effective_balance(i, spec)? >= spec.max_deposit_amount { + state.validator_registry[i].activation_epoch = spec.genesis_epoch; + } + } + + // Ensure the current epoch cache is built. + state.build_epoch_cache(RelativeEpoch::Current, spec)?; + + // Set all the active index roots to be the genesis active index root. + let active_validator_indices = state + .get_active_validator_indices(spec.genesis_epoch, spec)? + .to_vec(); + let genesis_active_index_root = Hash256::from_slice(&active_validator_indices.hash_tree_root()); + state.latest_active_index_roots = + vec![genesis_active_index_root; spec.latest_active_index_roots_length as usize]; + + // Generate the current shuffling seed. + state.current_shuffling_seed = state.generate_seed(spec.genesis_epoch, spec)?; + + Ok(()) +} + +impl From for GenesisError { + fn from(e: BlockProcessingError) -> GenesisError { + GenesisError::BlockProcessingError(e) + } +} + +impl From for GenesisError { + fn from(e: BeaconStateError) -> GenesisError { + GenesisError::BeaconStateError(e) + } +} diff --git a/eth2/state_processing/src/lib.rs b/eth2/state_processing/src/lib.rs index 2b30844cb..78dc7270d 100644 --- a/eth2/state_processing/src/lib.rs +++ b/eth2/state_processing/src/lib.rs @@ -1,10 +1,12 @@ #[macro_use] mod macros; +pub mod get_genesis_state; pub mod per_block_processing; pub mod per_epoch_processing; pub mod per_slot_processing; +pub use get_genesis_state::get_genesis_state; pub use per_block_processing::{ errors::{BlockInvalid, BlockProcessingError}, per_block_processing, per_block_processing_without_verifying_block_signature, diff --git a/eth2/state_processing/src/per_epoch_processing.rs b/eth2/state_processing/src/per_epoch_processing.rs index 2f1cc3551..d1bb4269a 100644 --- a/eth2/state_processing/src/per_epoch_processing.rs +++ b/eth2/state_processing/src/per_epoch_processing.rs @@ -45,7 +45,7 @@ pub fn per_epoch_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result process_rewards_and_penalities(state, &mut statuses, &winning_root_for_shards, spec)?; // Ejections - state.process_ejections(spec); + state.process_ejections(spec)?; // Validator Registry process_validator_registry(state, spec)?; @@ -53,7 +53,7 @@ pub fn per_epoch_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result // Final updates update_active_tree_index_roots(state, spec)?; update_latest_slashed_balances(state, spec); - clean_attestations(state, spec); + clean_attestations(state); // Rotate the epoch caches to suit the epoch transition. state.advance_caches(); @@ -472,6 +472,6 @@ pub fn update_latest_slashed_balances(state: &mut BeaconState, spec: &ChainSpec) /// Removes all pending attestations from the previous epoch. /// /// Spec v0.4.0 -pub fn clean_attestations(state: &mut BeaconState, spec: &ChainSpec) { +pub fn clean_attestations(state: &mut BeaconState) { state.previous_epoch_attestations = vec![]; } diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 8999d8be8..d7dbda782 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -10,9 +10,6 @@ use ssz_derive::{Decode, Encode, TreeHash}; use std::collections::HashMap; use test_random_derive::TestRandom; -pub use builder::BeaconStateBuilder; - -mod builder; mod epoch_cache; pub mod helpers; mod pubkey_cache; @@ -32,7 +29,8 @@ pub enum Error { InvalidBitfield, ValidatorIsWithdrawable, InsufficientRandaoMixes, - InsufficientValidators, + NoValidators, + UnableToDetermineProducer, InsufficientBlockRoots, InsufficientIndexRoots, InsufficientAttestations, @@ -534,7 +532,7 @@ impl BeaconState { /// /// If the state does not contain an index for a beacon proposer at the requested `slot`, then `None` is returned. /// - /// Spec v0.4.0 + /// Spec v0.5.0 pub fn get_beacon_proposer_index( &self, slot: Slot, @@ -547,14 +545,16 @@ impl BeaconState { .get_crosslink_committees_at_slot(slot, spec) .ok_or_else(|| Error::SlotOutOfBounds)?; + let epoch = slot.epoch(spec.slots_per_epoch); + committees .first() - .ok_or(Error::InsufficientValidators) + .ok_or(Error::UnableToDetermineProducer) .and_then(|first| { - let index = slot + let index = epoch .as_usize() .checked_rem(first.committee.len()) - .ok_or(Error::InsufficientValidators)?; + .ok_or(Error::UnableToDetermineProducer)?; Ok(first.committee[index]) }) } @@ -581,103 +581,9 @@ impl BeaconState { epoch + 1 + spec.activation_exit_delay } - /// Process multiple deposits in sequence. - /// - /// Builds a hashmap of validator pubkeys to validator index and passes it to each successive - /// call to `process_deposit(..)`. This requires much less computation than successive calls to - /// `process_deposits(..)` without the hashmap. - /// - /// Spec v0.4.0 - pub fn process_deposits( - &mut self, - deposits: Vec<&DepositData>, - spec: &ChainSpec, - ) -> Vec { - let mut added_indices = vec![]; - let mut pubkey_map: HashMap = HashMap::new(); - - for (i, validator) in self.validator_registry.iter().enumerate() { - pubkey_map.insert(validator.pubkey.clone(), i); - } - - for deposit_data in deposits { - let result = self.process_deposit( - deposit_data.deposit_input.clone(), - deposit_data.amount, - Some(&pubkey_map), - spec, - ); - if let Ok(index) = result { - added_indices.push(index); - } - } - added_indices - } - - /// Process a validator deposit, returning the validator index if the deposit is valid. - /// - /// Optionally accepts a hashmap of all validator pubkeys to their validator index. Without - /// this hashmap, each call to `process_deposits` requires an iteration though - /// `self.validator_registry`. This becomes highly inefficient at scale. - /// - /// TODO: this function also exists in a more optimal form in the `state_processing` crate as - /// `process_deposits`; unify these two functions. - /// - /// Spec v0.4.0 - pub fn process_deposit( - &mut self, - deposit_input: DepositInput, - amount: u64, - pubkey_map: Option<&HashMap>, - spec: &ChainSpec, - ) -> Result { - let proof_is_valid = deposit_input.proof_of_possession.verify( - &deposit_input.signed_root(), - spec.get_domain(self.current_epoch(&spec), Domain::Deposit, &self.fork), - &deposit_input.pubkey, - ); - - if !proof_is_valid { - return Err(()); - } - - let pubkey = deposit_input.pubkey.clone(); - let withdrawal_credentials = deposit_input.withdrawal_credentials.clone(); - - let validator_index = if let Some(pubkey_map) = pubkey_map { - pubkey_map.get(&pubkey).and_then(|i| Some(*i)) - } else { - self.validator_registry - .iter() - .position(|v| v.pubkey == pubkey) - }; - - if let Some(index) = validator_index { - if self.validator_registry[index].withdrawal_credentials == withdrawal_credentials { - safe_add_assign!(self.validator_balances[index], amount); - Ok(index) - } else { - Err(()) - } - } else { - let validator = Validator { - pubkey, - withdrawal_credentials, - activation_epoch: spec.far_future_epoch, - exit_epoch: spec.far_future_epoch, - withdrawable_epoch: spec.far_future_epoch, - initiated_exit: false, - slashed: false, - }; - self.validator_registry.push(validator); - self.validator_balances.push(amount); - Ok(self.validator_registry.len() - 1) - } - } - /// Activate the validator of the given ``index``. /// - /// Spec v0.4.0 + /// Spec v0.5.0 pub fn activate_validator( &mut self, validator_index: usize, diff --git a/eth2/types/src/beacon_state/builder.rs b/eth2/types/src/beacon_state/builder.rs deleted file mode 100644 index 780ec9b8b..000000000 --- a/eth2/types/src/beacon_state/builder.rs +++ /dev/null @@ -1,101 +0,0 @@ -use super::BeaconStateError; -use crate::validator_registry::get_active_validator_indices; -use crate::*; -use rayon::prelude::*; -use ssz::TreeHash; - -/// Builds a `BeaconState` for use in production. -/// -/// This struct should _not_ be modified for use in testing scenarios. Use `TestingBeaconStateBuilder` for that purpose. -/// -/// This struct should remain safe and sensible for production usage. -pub struct BeaconStateBuilder { - pub state: BeaconState, -} - -impl BeaconStateBuilder { - /// Create a new builder with the given number of validators. - /// - /// Spec v0.4.0 - pub fn new(genesis_time: u64, latest_eth1_data: Eth1Data, spec: &ChainSpec) -> Self { - Self { - state: BeaconState::genesis(genesis_time, latest_eth1_data, spec), - } - } - - /// Process deposit objects. - /// - /// Spec v0.4.0 - pub fn process_initial_deposits( - &mut self, - initial_validator_deposits: &[Deposit], - spec: &ChainSpec, - ) { - let deposit_data = initial_validator_deposits - .par_iter() - .map(|deposit| &deposit.deposit_data) - .collect(); - - self.state.process_deposits(deposit_data, spec); - - self.activate_genesis_validators(spec); - - self.state.deposit_index = initial_validator_deposits.len() as u64; - } - - fn activate_genesis_validators(&mut self, spec: &ChainSpec) -> Result<(), BeaconStateError> { - for validator_index in 0..self.state.validator_registry.len() { - if self.state.get_effective_balance(validator_index, spec)? >= spec.max_deposit_amount { - self.state.activate_validator(validator_index, true, spec); - } - } - - Ok(()) - } - - /// Instantiate the validator registry from a YAML file. - /// - /// This skips a lot of signing and verification, useful if signing and verification has been - /// completed previously. - /// - /// Spec v0.4.0 - pub fn import_existing_validators( - &mut self, - validators: Vec, - initial_balances: Vec, - deposit_index: u64, - spec: &ChainSpec, - ) { - self.state.validator_registry = validators; - - assert_eq!( - self.state.validator_registry.len(), - initial_balances.len(), - "Not enough balances for validators" - ); - - self.state.validator_balances = initial_balances; - - self.activate_genesis_validators(spec); - - self.state.deposit_index = deposit_index; - } - - /// Updates the final state variables and returns a fully built genesis state. - /// - /// Spec v0.4.0 - pub fn build(mut self, spec: &ChainSpec) -> Result { - let genesis_active_index_root = - get_active_validator_indices(&self.state.validator_registry, spec.genesis_epoch) - .hash_tree_root(); - - self.state.latest_active_index_roots = vec![ - Hash256::from_slice(&genesis_active_index_root); - spec.latest_active_index_roots_length - ]; - - self.state.current_shuffling_seed = self.state.generate_seed(spec.genesis_epoch, spec)?; - - Ok(self.state) - } -} diff --git a/eth2/types/src/beacon_state/epoch_cache.rs b/eth2/types/src/beacon_state/epoch_cache.rs index 4436972f1..0dbdf4054 100644 --- a/eth2/types/src/beacon_state/epoch_cache.rs +++ b/eth2/types/src/beacon_state/epoch_cache.rs @@ -4,6 +4,8 @@ use honey_badger_split::SplitExt; use serde_derive::{Deserialize, Serialize}; use swap_or_not_shuffle::shuffle_list; +mod tests; + #[derive(Debug, Default, PartialEq, Clone, Serialize, Deserialize)] pub struct EpochCache { /// `Some(epoch)` if the cache is initialized, where `epoch` is the cache it holds. @@ -247,7 +249,7 @@ impl EpochCrosslinkCommitteesBuilder { pub fn build(self, spec: &ChainSpec) -> Result { if self.active_validator_indices.is_empty() { - return Err(Error::InsufficientValidators); + return Err(Error::NoValidators); } let shuffled_active_validator_indices = shuffle_list( @@ -277,7 +279,7 @@ impl EpochCrosslinkCommitteesBuilder { let crosslink_committee = CrosslinkCommittee { slot, shard, - committee: committees.remove(j), + committee: committees[j].drain(..).collect(), }; epoch_crosslink_committees.crosslink_committees[i].push(crosslink_committee); diff --git a/eth2/types/src/beacon_state/epoch_cache/tests.rs b/eth2/types/src/beacon_state/epoch_cache/tests.rs new file mode 100644 index 000000000..10df635f2 --- /dev/null +++ b/eth2/types/src/beacon_state/epoch_cache/tests.rs @@ -0,0 +1,142 @@ +#![cfg(test)] + +use super::*; +use crate::test_utils::*; +use swap_or_not_shuffle::shuffle_list; + +fn do_sane_cache_test( + state: BeaconState, + epoch: Epoch, + validator_count: usize, + expected_seed: Hash256, + expected_shuffling_start: u64, + spec: &ChainSpec, +) { + let active_indices: Vec = (0..validator_count).collect(); + assert_eq!( + &active_indices[..], + state.get_active_validator_indices(epoch, &spec).unwrap(), + "Validator indices mismatch" + ); + + let shuffling = shuffle_list( + active_indices, + spec.shuffle_round_count, + &expected_seed[..], + true, + ) + .unwrap(); + + let committees_per_epoch = spec.get_epoch_committee_count(shuffling.len()); + let committees_per_slot = committees_per_epoch / spec.slots_per_epoch; + + let mut expected_indices_iter = shuffling.iter(); + let mut shard_counter = expected_shuffling_start; + + for (i, slot) in epoch.slot_iter(spec.slots_per_epoch).enumerate() { + let crosslink_committees_at_slot = + state.get_crosslink_committees_at_slot(slot, &spec).unwrap(); + + assert_eq!( + crosslink_committees_at_slot.len(), + committees_per_slot as usize, + "Bad committees per slot ({})", + i + ); + + for c in crosslink_committees_at_slot { + assert_eq!(c.shard, shard_counter, "Bad shard"); + shard_counter += 1; + shard_counter %= spec.shard_count; + + for &i in &c.committee { + assert_eq!( + i, + *expected_indices_iter.next().unwrap(), + "Non-sequential validators." + ); + } + } + } +} + +fn setup_sane_cache_test(validator_count: usize, spec: &ChainSpec) -> BeaconState { + let mut builder = + TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, spec); + + let epoch = spec.genesis_epoch + 4; + let slot = epoch.start_slot(spec.slots_per_epoch); + builder.teleport_to_slot(slot, spec); + + let (mut state, _keypairs) = builder.build(); + + state.current_shuffling_start_shard = 0; + state.current_shuffling_seed = Hash256::from_slice(&[1; 32]); + + state.previous_shuffling_start_shard = spec.shard_count - 1; + state.previous_shuffling_seed = Hash256::from_slice(&[2; 32]); + + state + .build_epoch_cache(RelativeEpoch::Previous, spec) + .unwrap(); + state + .build_epoch_cache(RelativeEpoch::Current, spec) + .unwrap(); + state + .build_epoch_cache(RelativeEpoch::NextWithRegistryChange, spec) + .unwrap(); + state + .build_epoch_cache(RelativeEpoch::NextWithoutRegistryChange, spec) + .unwrap(); + + state +} + +#[test] +fn builds_sane_current_epoch_cache() { + let mut spec = ChainSpec::few_validators(); + spec.shard_count = 4; + let validator_count = (spec.shard_count * spec.target_committee_size) + 1; + let state = setup_sane_cache_test(validator_count as usize, &spec); + do_sane_cache_test( + state.clone(), + state.current_epoch(&spec), + validator_count as usize, + state.current_shuffling_seed, + state.current_shuffling_start_shard, + &spec, + ); +} + +#[test] +fn builds_sane_previous_epoch_cache() { + let mut spec = ChainSpec::few_validators(); + spec.shard_count = 2; + let validator_count = (spec.shard_count * spec.target_committee_size) + 1; + let state = setup_sane_cache_test(validator_count as usize, &spec); + do_sane_cache_test( + state.clone(), + state.previous_epoch(&spec), + validator_count as usize, + state.previous_shuffling_seed, + state.previous_shuffling_start_shard, + &spec, + ); +} + +#[test] +fn builds_sane_next_without_update_epoch_cache() { + let mut spec = ChainSpec::few_validators(); + spec.shard_count = 2; + let validator_count = (spec.shard_count * spec.target_committee_size) + 1; + let mut state = setup_sane_cache_test(validator_count as usize, &spec); + state.validator_registry_update_epoch = state.slot.epoch(spec.slots_per_epoch); + do_sane_cache_test( + state.clone(), + state.next_epoch(&spec), + validator_count as usize, + state.current_shuffling_seed, + state.current_shuffling_start_shard, + &spec, + ); +} diff --git a/eth2/types/src/test_utils/testing_beacon_state_builder.rs b/eth2/types/src/test_utils/testing_beacon_state_builder.rs index 54e2fbe96..8180673d1 100644 --- a/eth2/types/src/test_utils/testing_beacon_state_builder.rs +++ b/eth2/types/src/test_utils/testing_beacon_state_builder.rs @@ -1,5 +1,4 @@ use super::{generate_deterministic_keypairs, KeypairsFile}; -use crate::beacon_state::BeaconStateBuilder; use crate::test_utils::TestingPendingAttestationBuilder; use crate::*; use bls::get_withdrawal_credentials; @@ -110,7 +109,8 @@ impl TestingBeaconStateBuilder { Validator { pubkey: keypair.pk.clone(), withdrawal_credentials, - activation_epoch: spec.far_future_epoch, + // All validators start active. + activation_epoch: spec.genesis_epoch, exit_epoch: spec.far_future_epoch, withdrawable_epoch: spec.far_future_epoch, initiated_exit: false, @@ -119,7 +119,7 @@ impl TestingBeaconStateBuilder { }) .collect(); - let mut state_builder = BeaconStateBuilder::new( + let mut state = BeaconState::genesis( 0, Eth1Data { deposit_root: Hash256::zero(), @@ -131,14 +131,8 @@ impl TestingBeaconStateBuilder { let balances = vec![32_000_000_000; validator_count]; debug!("Importing {} existing validators...", validator_count); - state_builder.import_existing_validators( - validators, - balances, - validator_count as u64, - spec, - ); - - let state = state_builder.build(spec).unwrap(); + state.validator_registry = validators; + state.validator_balances = balances; debug!("BeaconState built.");