From b211e39331de307fc9f15a908475de1eeebe4d2d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 21 Feb 2019 19:00:55 +1300 Subject: [PATCH 1/9] Add progress on FastBeaconState --- eth2/types/src/fast_beacon_state.rs | 1136 +++++++++++++++++ .../src/fast_beacon_state/committees_cache.rs | 128 ++ eth2/types/src/lib.rs | 2 + 3 files changed, 1266 insertions(+) create mode 100644 eth2/types/src/fast_beacon_state.rs create mode 100644 eth2/types/src/fast_beacon_state/committees_cache.rs diff --git a/eth2/types/src/fast_beacon_state.rs b/eth2/types/src/fast_beacon_state.rs new file mode 100644 index 000000000..7e56df8ae --- /dev/null +++ b/eth2/types/src/fast_beacon_state.rs @@ -0,0 +1,1136 @@ +use crate::test_utils::TestRandom; +use crate::{ + validator::StatusFlags, validator_registry::get_active_validator_indices, AttestationData, + Bitfield, ChainSpec, Crosslink, Deposit, Epoch, Eth1Data, Eth1DataVote, Fork, Hash256, + PendingAttestation, PublicKey, Signature, Slot, Validator, +}; +use bls::verify_proof_of_possession; +use committees_cache::CommitteesCache; +use honey_badger_split::SplitExt; +use log::trace; +use rand::RngCore; +use serde_derive::Serialize; +use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use swap_or_not_shuffle::get_permutated_index; + +mod committees_cache; + +#[derive(Debug, PartialEq)] +pub enum FastBeaconStateError { + EpochOutOfBounds, + UnableToShuffle, + InsufficientRandaoMixes, + InsufficientValidators, + InsufficientBlockRoots, + InsufficientIndexRoots, + InsufficientAttestations, + InsufficientCommittees, +} + +#[derive(Debug, PartialEq)] +pub enum InclusionError { + /// The validator did not participate in an attestation in this period. + NoAttestationsForValidator, + AttestationParticipantsError(AttestationParticipantsError), +} + +#[derive(Debug, PartialEq)] +pub enum AttestationParticipantsError { + /// There is no committee for the given shard in the given epoch. + NoCommitteeForShard, + FastBeaconStateError(FastBeaconStateError), +} + +macro_rules! safe_add_assign { + ($a: expr, $b: expr) => { + $a = $a.saturating_add($b); + }; +} +macro_rules! safe_sub_assign { + ($a: expr, $b: expr) => { + $a = $a.saturating_sub($b); + }; +} + +#[derive(Debug, PartialEq, Clone, Default, Serialize)] +pub struct FastBeaconState { + // Misc + pub slot: Slot, + pub genesis_time: u64, + pub fork: Fork, + + // Validator registry + pub validator_registry: Vec, + pub validator_balances: Vec, + pub validator_registry_update_epoch: Epoch, + + // Randomness and committees + pub latest_randao_mixes: Vec, + pub previous_epoch_start_shard: u64, + pub current_epoch_start_shard: u64, + pub previous_calculation_epoch: Epoch, + pub current_calculation_epoch: Epoch, + pub previous_epoch_seed: Hash256, + pub current_epoch_seed: Hash256, + + // Finality + pub previous_justified_epoch: Epoch, + pub justified_epoch: Epoch, + pub justification_bitfield: u64, + pub finalized_epoch: Epoch, + + // Recent state + pub latest_crosslinks: Vec, + pub latest_block_roots: Vec, + pub latest_index_roots: Vec, + pub latest_penalized_balances: Vec, + pub latest_attestations: Vec, + pub batched_block_roots: Vec, + + // Ethereum 1.0 chain data + pub latest_eth1_data: Eth1Data, + pub eth1_data_votes: Vec, + + // Cache + committees_cache: CommitteesCache, +} + +impl FastBeaconState { + /// Produce the first state of the Beacon Chain. + pub fn genesis( + genesis_time: u64, + initial_validator_deposits: Vec, + latest_eth1_data: Eth1Data, + spec: &ChainSpec, + ) -> Result { + let initial_crosslink = Crosslink { + epoch: spec.genesis_epoch, + shard_block_root: spec.zero_hash, + }; + + let mut genesis_state = FastBeaconState { + /* + * Misc + */ + slot: spec.genesis_slot, + genesis_time, + fork: Fork { + previous_version: spec.genesis_fork_version, + current_version: spec.genesis_fork_version, + epoch: spec.genesis_epoch, + }, + + /* + * Validator registry + */ + validator_registry: vec![], // Set later in the function. + validator_balances: vec![], // Set later in the function. + validator_registry_update_epoch: spec.genesis_epoch, + + /* + * Randomness and committees + */ + latest_randao_mixes: vec![spec.zero_hash; spec.latest_randao_mixes_length as usize], + previous_epoch_start_shard: spec.genesis_start_shard, + current_epoch_start_shard: spec.genesis_start_shard, + previous_calculation_epoch: spec.genesis_epoch, + current_calculation_epoch: spec.genesis_epoch, + previous_epoch_seed: spec.zero_hash, + current_epoch_seed: spec.zero_hash, + + /* + * Finality + */ + previous_justified_epoch: spec.genesis_epoch, + justified_epoch: spec.genesis_epoch, + justification_bitfield: 0, + finalized_epoch: spec.genesis_epoch, + + /* + * Recent state + */ + latest_crosslinks: vec![initial_crosslink; spec.shard_count as usize], + latest_block_roots: vec![spec.zero_hash; spec.latest_block_roots_length as usize], + latest_index_roots: vec![spec.zero_hash; spec.latest_index_roots_length as usize], + latest_penalized_balances: vec![0; spec.latest_penalized_exit_length as usize], + latest_attestations: vec![], + batched_block_roots: vec![], + + /* + * PoW receipt root + */ + latest_eth1_data, + eth1_data_votes: vec![], + + /* + * Caches (not in spec) + */ + committees_cache: CommitteesCache::new(spec.genesis_epoch, spec), + }; + + for deposit in initial_validator_deposits { + let _index = genesis_state.process_deposit( + deposit.deposit_data.deposit_input.pubkey, + deposit.deposit_data.amount, + deposit.deposit_data.deposit_input.proof_of_possession, + deposit.deposit_data.deposit_input.withdrawal_credentials, + spec, + ); + } + + for validator_index in 0..genesis_state.validator_registry.len() { + if genesis_state.get_effective_balance(validator_index, spec) >= spec.max_deposit_amount + { + genesis_state.activate_validator(validator_index, true, spec); + } + } + + let genesis_active_index_root = hash_tree_root(get_active_validator_indices( + &genesis_state.validator_registry, + spec.genesis_epoch, + )); + genesis_state.latest_index_roots = + vec![genesis_active_index_root; spec.latest_index_roots_length]; + genesis_state.current_epoch_seed = genesis_state.generate_seed(spec.genesis_epoch, spec)?; + + Ok(genesis_state) + } + + /// Return the tree hash root for this `FastBeaconState`. + /// + /// Spec v0.2.0 + pub fn canonical_root(&self) -> Hash256 { + Hash256::from(&self.hash_tree_root()[..]) + } + + /// The epoch corresponding to `self.slot`. + /// + /// Spec v0.2.0 + pub fn current_epoch(&self, spec: &ChainSpec) -> Epoch { + self.slot.epoch(spec.epoch_length) + } + + /// The epoch prior to `self.current_epoch()`. + /// + /// Spec v0.2.0 + pub fn previous_epoch(&self, spec: &ChainSpec) -> Epoch { + let current_epoch = self.current_epoch(&spec); + if current_epoch == spec.genesis_epoch { + current_epoch + } else { + current_epoch - 1 + } + } + + /// The epoch following `self.current_epoch()`. + /// + /// Spec v0.2.0 + pub fn next_epoch(&self, spec: &ChainSpec) -> Epoch { + self.current_epoch(spec).saturating_add(1_u64) + } + + /// The first slot of the epoch corresponding to `self.slot`. + /// + /// Spec v0.2.0 + pub fn current_epoch_start_slot(&self, spec: &ChainSpec) -> Slot { + self.current_epoch(spec).start_slot(spec.epoch_length) + } + + /// The first slot of the epoch preceeding the one corresponding to `self.slot`. + /// + /// Spec v0.2.0 + pub fn previous_epoch_start_slot(&self, spec: &ChainSpec) -> Slot { + self.previous_epoch(spec).start_slot(spec.epoch_length) + } + + /// Return the number of committees in one epoch. + /// + /// TODO: this should probably be a method on `ChainSpec`. + /// + /// Spec v0.2.0 + pub fn get_epoch_committee_count( + &self, + active_validator_count: usize, + spec: &ChainSpec, + ) -> u64 { + std::cmp::max( + 1, + std::cmp::min( + spec.shard_count / spec.epoch_length, + active_validator_count as u64 / spec.epoch_length / spec.target_committee_size, + ), + ) * spec.epoch_length + } + + /// Shuffle ``validators`` into crosslink committees seeded by ``seed`` and ``epoch``. + /// Return a list of ``committees_per_epoch`` committees where each + /// committee is itself a list of validator indices. + /// + /// Spec v0.1 + pub fn get_shuffling( + &self, + seed: Hash256, + epoch: Epoch, + spec: &ChainSpec, + ) -> Option>> { + let active_validator_indices = + get_active_validator_indices(&self.validator_registry, epoch); + + if active_validator_indices.is_empty() { + return None; + } + + trace!( + "get_shuffling: active_validator_indices.len() == {}", + active_validator_indices.len() + ); + + let committees_per_epoch = + self.get_epoch_committee_count(active_validator_indices.len(), spec); + + trace!( + "get_shuffling: active_validator_indices.len() == {}, committees_per_epoch: {}", + active_validator_indices.len(), + committees_per_epoch + ); + + let mut shuffled_active_validator_indices = vec![0; active_validator_indices.len()]; + for &i in &active_validator_indices { + let shuffled_i = get_permutated_index( + i, + active_validator_indices.len(), + &seed[..], + spec.shuffle_round_count, + )?; + shuffled_active_validator_indices[i] = active_validator_indices[shuffled_i] + } + + Some( + shuffled_active_validator_indices + .honey_badger_split(committees_per_epoch as usize) + .map(|slice: &[usize]| slice.to_vec()) + .collect(), + ) + } + + /// Return the number of committees in the previous epoch. + /// + /// Spec v0.2.0 + fn get_previous_epoch_committee_count(&self, spec: &ChainSpec) -> u64 { + let previous_active_validators = + get_active_validator_indices(&self.validator_registry, self.previous_calculation_epoch); + self.get_epoch_committee_count(previous_active_validators.len(), spec) + } + + /// Return the number of committees in the current epoch. + /// + /// Spec v0.2.0 + pub fn get_current_epoch_committee_count(&self, spec: &ChainSpec) -> u64 { + let current_active_validators = + get_active_validator_indices(&self.validator_registry, self.current_calculation_epoch); + self.get_epoch_committee_count(current_active_validators.len(), spec) + } + + /// Return the number of committees in the next epoch. + /// + /// Spec v0.2.0 + pub fn get_next_epoch_committee_count(&self, spec: &ChainSpec) -> u64 { + let current_active_validators = + get_active_validator_indices(&self.validator_registry, self.next_epoch(spec)); + self.get_epoch_committee_count(current_active_validators.len(), spec) + } + + pub fn get_active_index_root(&self, epoch: Epoch, spec: &ChainSpec) -> Option { + let current_epoch = self.current_epoch(spec); + + let earliest_index_root = current_epoch - Epoch::from(spec.latest_index_roots_length) + + Epoch::from(spec.entry_exit_delay) + + 1; + let latest_index_root = current_epoch + spec.entry_exit_delay; + + trace!( + "get_active_index_root: epoch: {}, earliest: {}, latest: {}", + epoch, + earliest_index_root, + latest_index_root + ); + + if (epoch >= earliest_index_root) & (epoch <= latest_index_root) { + Some(self.latest_index_roots[epoch.as_usize() % spec.latest_index_roots_length]) + } else { + trace!("get_active_index_root: epoch out of range."); + None + } + } + + /// Generate a seed for the given ``epoch``. + /// + /// Spec v0.2.0 + pub fn generate_seed( + &self, + epoch: Epoch, + spec: &ChainSpec, + ) -> Result { + let mut input = self + .get_randao_mix(epoch, spec) + .ok_or_else(|| FastBeaconStateError::InsufficientRandaoMixes)? + .to_vec(); + + input.append( + &mut self + .get_active_index_root(epoch, spec) + .ok_or_else(|| FastBeaconStateError::InsufficientIndexRoots)? + .to_vec(), + ); + + // TODO: ensure `Hash256::from(u64)` == `int_to_bytes32`. + input.append(&mut Hash256::from(epoch.as_u64()).to_vec()); + + Ok(Hash256::from(&hash(&input[..])[..])) + } + + /// Return the list of ``(committee, shard)`` tuples for the ``slot``. + /// + /// Note: There are two possible shufflings for crosslink committees for a + /// `slot` in the next epoch: with and without a `registry_change` + /// + /// Spec v0.2.0 + pub fn get_crosslink_committees_at_slot( + &self, + slot: Slot, + registry_change: bool, + spec: &ChainSpec, + ) -> Result, u64)>, FastBeaconStateError> { + let epoch = slot.epoch(spec.epoch_length); + let current_epoch = self.current_epoch(spec); + let previous_epoch = self.previous_epoch(spec); + let next_epoch = self.next_epoch(spec); + + let (committees_per_epoch, seed, shuffling_epoch, shuffling_start_shard) = + if epoch == current_epoch { + trace!("get_crosslink_committees_at_slot: current_epoch"); + ( + self.get_current_epoch_committee_count(spec), + self.current_epoch_seed, + self.current_calculation_epoch, + self.current_epoch_start_shard, + ) + } else if epoch == previous_epoch { + trace!("get_crosslink_committees_at_slot: previous_epoch"); + ( + self.get_previous_epoch_committee_count(spec), + self.previous_epoch_seed, + self.previous_calculation_epoch, + self.previous_epoch_start_shard, + ) + } else if epoch == next_epoch { + trace!("get_crosslink_committees_at_slot: next_epoch"); + let current_committees_per_epoch = self.get_current_epoch_committee_count(spec); + let epochs_since_last_registry_update = + current_epoch - self.validator_registry_update_epoch; + let (seed, shuffling_start_shard) = if registry_change { + let next_seed = self.generate_seed(next_epoch, spec)?; + ( + next_seed, + (self.current_epoch_start_shard + current_committees_per_epoch) + % spec.shard_count, + ) + } else if (epochs_since_last_registry_update > 1) + & epochs_since_last_registry_update.is_power_of_two() + { + let next_seed = self.generate_seed(next_epoch, spec)?; + (next_seed, self.current_epoch_start_shard) + } else { + (self.current_epoch_seed, self.current_epoch_start_shard) + }; + ( + self.get_next_epoch_committee_count(spec), + seed, + next_epoch, + shuffling_start_shard, + ) + } else { + return Err(FastBeaconStateError::EpochOutOfBounds); + }; + + let shuffling = self + .get_shuffling(seed, shuffling_epoch, spec) + .ok_or_else(|| FastBeaconStateError::UnableToShuffle)?; + let offset = slot.as_u64() % spec.epoch_length; + let committees_per_slot = committees_per_epoch / spec.epoch_length; + let slot_start_shard = + (shuffling_start_shard + committees_per_slot * offset) % spec.shard_count; + + trace!( + "get_crosslink_committees_at_slot: committees_per_slot: {}, slot_start_shard: {}, seed: {}", + committees_per_slot, + slot_start_shard, + seed + ); + + let mut crosslinks_at_slot = vec![]; + for i in 0..committees_per_slot { + let tuple = ( + shuffling[(committees_per_slot * offset + i) as usize].clone(), + (slot_start_shard + i) % spec.shard_count, + ); + crosslinks_at_slot.push(tuple) + } + Ok(crosslinks_at_slot) + } + + /// Returns the `slot`, `shard` and `committee_index` for which a validator must produce an + /// attestation. + /// + /// Spec v0.2.0 + pub fn attestation_slot_and_shard_for_validator( + &self, + validator_index: usize, + spec: &ChainSpec, + ) -> Result, FastBeaconStateError> { + let mut result = None; + for slot in self.current_epoch(spec).slot_iter(spec.epoch_length) { + for (committee, shard) in self.get_crosslink_committees_at_slot(slot, false, spec)? { + if let Some(committee_index) = committee.iter().position(|&i| i == validator_index) + { + result = Some((slot, shard, committee_index as u64)); + } + } + } + Ok(result) + } + + /// An entry or exit triggered in the ``epoch`` given by the input takes effect at + /// the epoch given by the output. + /// + /// Spec v0.2.0 + pub fn get_entry_exit_effect_epoch(&self, epoch: Epoch, spec: &ChainSpec) -> Epoch { + epoch + 1 + spec.entry_exit_delay + } + + /// Returns the beacon proposer index for the `slot`. + /// + /// If the state does not contain an index for a beacon proposer at the requested `slot`, then `None` is returned. + /// + /// Spec v0.2.0 + pub fn get_beacon_proposer_index( + &self, + slot: Slot, + spec: &ChainSpec, + ) -> Result { + let committees = self.get_crosslink_committees_at_slot(slot, false, spec)?; + trace!( + "get_beacon_proposer_index: slot: {}, committees_count: {}", + slot, + committees.len() + ); + committees + .first() + .ok_or(FastBeaconStateError::InsufficientValidators) + .and_then(|(first_committee, _)| { + let index = (slot.as_usize()) + .checked_rem(first_committee.len()) + .ok_or(FastBeaconStateError::InsufficientValidators)?; + Ok(first_committee[index]) + }) + } + + /// Process the penalties and prepare the validators who are eligible to withdrawal. + /// + /// Spec v0.2.0 + pub fn process_penalties_and_exits(&mut self, spec: &ChainSpec) { + let current_epoch = self.current_epoch(spec); + let active_validator_indices = + get_active_validator_indices(&self.validator_registry, current_epoch); + let total_balance = self.get_total_balance(&active_validator_indices[..], spec); + + for index in 0..self.validator_balances.len() { + let validator = &self.validator_registry[index]; + + if current_epoch + == validator.penalized_epoch + Epoch::from(spec.latest_penalized_exit_length / 2) + { + let epoch_index: usize = + current_epoch.as_usize() % spec.latest_penalized_exit_length; + + let total_at_start = self.latest_penalized_balances + [(epoch_index + 1) % spec.latest_penalized_exit_length]; + let total_at_end = self.latest_penalized_balances[epoch_index]; + let total_penalities = total_at_end.saturating_sub(total_at_start); + let penalty = self.get_effective_balance(index, spec) + * std::cmp::min(total_penalities * 3, total_balance) + / total_balance; + safe_sub_assign!(self.validator_balances[index], penalty); + } + } + + let eligible = |index: usize| { + let validator = &self.validator_registry[index]; + + if validator.penalized_epoch <= current_epoch { + let penalized_withdrawal_epochs = spec.latest_penalized_exit_length / 2; + current_epoch >= validator.penalized_epoch + penalized_withdrawal_epochs as u64 + } else { + current_epoch >= validator.exit_epoch + spec.min_validator_withdrawal_epochs + } + }; + + let mut eligable_indices: Vec = (0..self.validator_registry.len()) + .filter(|i| eligible(*i)) + .collect(); + eligable_indices.sort_by_key(|i| self.validator_registry[*i].exit_epoch); + for (withdrawn_so_far, index) in eligable_indices.iter().enumerate() { + self.prepare_validator_for_withdrawal(*index); + if withdrawn_so_far as u64 >= spec.max_withdrawals_per_epoch { + break; + } + } + } + + /// Return the randao mix at a recent ``epoch``. + /// + /// Returns `None` if the epoch is out-of-bounds of `self.latest_randao_mixes`. + /// + /// Spec v0.2.0 + pub fn get_randao_mix(&self, epoch: Epoch, spec: &ChainSpec) -> Option<&Hash256> { + self.latest_randao_mixes + .get(epoch.as_usize() % spec.latest_randao_mixes_length) + } + + /// Update validator registry, activating/exiting validators if possible. + /// + /// Spec v0.2.0 + pub fn update_validator_registry(&mut self, spec: &ChainSpec) { + let current_epoch = self.current_epoch(spec); + let active_validator_indices = + get_active_validator_indices(&self.validator_registry, current_epoch); + let total_balance = self.get_total_balance(&active_validator_indices[..], spec); + + let max_balance_churn = std::cmp::max( + spec.max_deposit_amount, + total_balance / (2 * spec.max_balance_churn_quotient), + ); + + let mut balance_churn = 0; + for index in 0..self.validator_registry.len() { + let validator = &self.validator_registry[index]; + + if (validator.activation_epoch > self.get_entry_exit_effect_epoch(current_epoch, spec)) + && self.validator_balances[index] >= spec.max_deposit_amount + { + balance_churn += self.get_effective_balance(index, spec); + if balance_churn > max_balance_churn { + break; + } + self.activate_validator(index, false, spec); + } + } + + let mut balance_churn = 0; + for index in 0..self.validator_registry.len() { + let validator = &self.validator_registry[index]; + + if (validator.exit_epoch > self.get_entry_exit_effect_epoch(current_epoch, spec)) + && validator.status_flags == Some(StatusFlags::InitiatedExit) + { + balance_churn += self.get_effective_balance(index, spec); + if balance_churn > max_balance_churn { + break; + } + + self.exit_validator(index, spec); + } + } + + self.validator_registry_update_epoch = current_epoch; + } + /// Process a validator deposit, returning the validator index if the deposit is valid. + /// + /// Spec v0.2.0 + pub fn process_deposit( + &mut self, + pubkey: PublicKey, + amount: u64, + proof_of_possession: Signature, + withdrawal_credentials: Hash256, + spec: &ChainSpec, + ) -> Result { + // TODO: ensure verify proof-of-possession represents the spec accurately. + if !verify_proof_of_possession(&proof_of_possession, &pubkey) { + return Err(()); + } + + if let Some(index) = self + .validator_registry + .iter() + .position(|v| v.pubkey == pubkey) + { + if self.validator_registry[index].withdrawal_credentials == withdrawal_credentials { + safe_add_assign!(self.validator_balances[index], amount); + Ok(index) + } else { + Err(()) + } + } else { + let validator = Validator { + pubkey, + withdrawal_credentials, + activation_epoch: spec.far_future_epoch, + exit_epoch: spec.far_future_epoch, + withdrawal_epoch: spec.far_future_epoch, + penalized_epoch: spec.far_future_epoch, + status_flags: None, + }; + self.validator_registry.push(validator); + self.validator_balances.push(amount); + Ok(self.validator_registry.len() - 1) + } + } + + /// Activate the validator of the given ``index``. + /// + /// Spec v0.2.0 + pub fn activate_validator( + &mut self, + validator_index: usize, + is_genesis: bool, + spec: &ChainSpec, + ) { + let current_epoch = self.current_epoch(spec); + + self.validator_registry[validator_index].activation_epoch = if is_genesis { + spec.genesis_epoch + } else { + self.get_entry_exit_effect_epoch(current_epoch, spec) + } + } + + /// Initiate an exit for the validator of the given `index`. + /// + /// Spec v0.2.0 + pub fn initiate_validator_exit(&mut self, validator_index: usize) { + // TODO: the spec does an `|=` here, ensure this isn't buggy. + self.validator_registry[validator_index].status_flags = Some(StatusFlags::InitiatedExit); + } + + /// Exit the validator of the given `index`. + /// + /// Spec v0.2.0 + fn exit_validator(&mut self, validator_index: usize, spec: &ChainSpec) { + let current_epoch = self.current_epoch(spec); + + if self.validator_registry[validator_index].exit_epoch + <= self.get_entry_exit_effect_epoch(current_epoch, spec) + { + return; + } + + self.validator_registry[validator_index].exit_epoch = + self.get_entry_exit_effect_epoch(current_epoch, spec); + } + + /// Penalize the validator of the given ``index``. + /// + /// Exits the validator and assigns its effective balance to the block producer for this + /// state. + /// + /// Spec v0.2.0 + pub fn penalize_validator( + &mut self, + validator_index: usize, + spec: &ChainSpec, + ) -> Result<(), FastBeaconStateError> { + self.exit_validator(validator_index, spec); + let current_epoch = self.current_epoch(spec); + + self.latest_penalized_balances + [current_epoch.as_usize() % spec.latest_penalized_exit_length] += + self.get_effective_balance(validator_index, spec); + + let whistleblower_index = self.get_beacon_proposer_index(self.slot, spec)?; + let whistleblower_reward = self.get_effective_balance(validator_index, spec); + safe_add_assign!( + self.validator_balances[whistleblower_index as usize], + whistleblower_reward + ); + safe_sub_assign!( + self.validator_balances[validator_index], + whistleblower_reward + ); + self.validator_registry[validator_index].penalized_epoch = current_epoch; + Ok(()) + } + + /// Initiate an exit for the validator of the given `index`. + /// + /// Spec v0.2.0 + pub fn prepare_validator_for_withdrawal(&mut self, validator_index: usize) { + //TODO: we're not ANDing here, we're setting. Potentially wrong. + self.validator_registry[validator_index].status_flags = Some(StatusFlags::Withdrawable); + } + + /// Iterate through the validator registry and eject active validators with balance below + /// ``EJECTION_BALANCE``. + /// + /// Spec v0.2.0 + pub fn process_ejections(&mut self, spec: &ChainSpec) { + for validator_index in + get_active_validator_indices(&self.validator_registry, self.current_epoch(spec)) + { + if self.validator_balances[validator_index] < spec.ejection_balance { + self.exit_validator(validator_index, spec) + } + } + } + + /// Returns the penality that should be applied to some validator for inactivity. + /// + /// Note: this is defined "inline" in the spec, not as a helper function. + /// + /// Spec v0.2.0 + pub fn inactivity_penalty( + &self, + validator_index: usize, + epochs_since_finality: Epoch, + base_reward_quotient: u64, + spec: &ChainSpec, + ) -> u64 { + let effective_balance = self.get_effective_balance(validator_index, spec); + self.base_reward(validator_index, base_reward_quotient, spec) + + effective_balance * epochs_since_finality.as_u64() + / spec.inactivity_penalty_quotient + / 2 + } + + /// Returns the distance between the first included attestation for some validator and this + /// slot. + /// + /// Note: In the spec this is defined "inline", not as a helper function. + /// + /// Spec v0.2.0 + pub fn inclusion_distance( + &self, + attestations: &[&PendingAttestation], + validator_index: usize, + spec: &ChainSpec, + ) -> Result { + let attestation = + self.earliest_included_attestation(attestations, validator_index, spec)?; + Ok((attestation.inclusion_slot - attestation.data.slot).as_u64()) + } + + /// Returns the slot of the earliest included attestation for some validator. + /// + /// Note: In the spec this is defined "inline", not as a helper function. + /// + /// Spec v0.2.0 + pub fn inclusion_slot( + &self, + attestations: &[&PendingAttestation], + validator_index: usize, + spec: &ChainSpec, + ) -> Result { + let attestation = + self.earliest_included_attestation(attestations, validator_index, spec)?; + Ok(attestation.inclusion_slot) + } + + /// Finds the earliest included attestation for some validator. + /// + /// Note: In the spec this is defined "inline", not as a helper function. + /// + /// Spec v0.2.0 + fn earliest_included_attestation( + &self, + attestations: &[&PendingAttestation], + validator_index: usize, + spec: &ChainSpec, + ) -> Result { + let mut included_attestations = vec![]; + + for (i, a) in attestations.iter().enumerate() { + let participants = + self.get_attestation_participants(&a.data, &a.aggregation_bitfield, spec)?; + if participants.iter().any(|i| *i == validator_index) { + included_attestations.push(i); + } + } + + let earliest_attestation_index = included_attestations + .iter() + .min_by_key(|i| attestations[**i].inclusion_slot) + .ok_or_else(|| InclusionError::NoAttestationsForValidator)?; + Ok(attestations[*earliest_attestation_index].clone()) + } + + /// Returns the base reward for some validator. + /// + /// Note: In the spec this is defined "inline", not as a helper function. + /// + /// Spec v0.2.0 + pub fn base_reward( + &self, + validator_index: usize, + base_reward_quotient: u64, + spec: &ChainSpec, + ) -> u64 { + self.get_effective_balance(validator_index, spec) / base_reward_quotient / 5 + } + + /// Return the combined effective balance of an array of validators. + /// + /// Spec v0.2.0 + pub fn get_total_balance(&self, validator_indices: &[usize], spec: &ChainSpec) -> u64 { + validator_indices + .iter() + .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)) + } + + /// Return the effective balance (also known as "balance at stake") for a validator with the given ``index``. + /// + /// Spec v0.2.0 + pub fn get_effective_balance(&self, validator_index: usize, spec: &ChainSpec) -> u64 { + std::cmp::min( + self.validator_balances[validator_index], + spec.max_deposit_amount, + ) + } + + /// Return the block root at a recent `slot`. + /// + /// Spec v0.2.0 + pub fn get_block_root(&self, slot: Slot, spec: &ChainSpec) -> Option<&Hash256> { + self.latest_block_roots + .get(slot.as_usize() % spec.latest_block_roots_length) + } + + pub fn get_attestation_participants_union( + &self, + attestations: &[&PendingAttestation], + spec: &ChainSpec, + ) -> Result, AttestationParticipantsError> { + let mut all_participants = attestations.iter().try_fold::<_, _, Result< + Vec, + AttestationParticipantsError, + >>(vec![], |mut acc, a| { + acc.append(&mut self.get_attestation_participants( + &a.data, + &a.aggregation_bitfield, + spec, + )?); + Ok(acc) + })?; + all_participants.sort_unstable(); + all_participants.dedup(); + Ok(all_participants) + } + + /// Return the participant indices at for the ``attestation_data`` and ``bitfield``. + /// + /// In effect, this converts the "committee indices" on the bitfield into "validator indices" + /// for self.validator_registy. + /// + /// Spec v0.2.0 + pub fn get_attestation_participants( + &self, + attestation_data: &AttestationData, + bitfield: &Bitfield, + spec: &ChainSpec, + ) -> Result, AttestationParticipantsError> { + let crosslink_committees = + self.get_crosslink_committees_at_slot(attestation_data.slot, false, spec)?; + + let committee_index: usize = crosslink_committees + .iter() + .position(|(_committee, shard)| *shard == attestation_data.shard) + .ok_or_else(|| AttestationParticipantsError::NoCommitteeForShard)?; + let (crosslink_committee, _shard) = &crosslink_committees[committee_index]; + + /* + * TODO: verify bitfield length is valid. + */ + + let mut participants = vec![]; + for (i, validator_index) in crosslink_committee.iter().enumerate() { + if bitfield.get(i).unwrap() { + participants.push(*validator_index); + } + } + Ok(participants) + } +} + +fn hash_tree_root(input: Vec) -> Hash256 { + Hash256::from(&input.hash_tree_root()[..]) +} + +impl From for AttestationParticipantsError { + fn from(e: FastBeaconStateError) -> AttestationParticipantsError { + AttestationParticipantsError::FastBeaconStateError(e) + } +} + +impl From for InclusionError { + fn from(e: AttestationParticipantsError) -> InclusionError { + InclusionError::AttestationParticipantsError(e) + } +} + +impl Encodable for FastBeaconState { + fn ssz_append(&self, s: &mut SszStream) { + s.append(&self.slot); + s.append(&self.genesis_time); + s.append(&self.fork); + s.append(&self.validator_registry); + s.append(&self.validator_balances); + s.append(&self.validator_registry_update_epoch); + s.append(&self.latest_randao_mixes); + s.append(&self.previous_epoch_start_shard); + s.append(&self.current_epoch_start_shard); + s.append(&self.previous_calculation_epoch); + s.append(&self.current_calculation_epoch); + s.append(&self.previous_epoch_seed); + s.append(&self.current_epoch_seed); + s.append(&self.previous_justified_epoch); + s.append(&self.justified_epoch); + s.append(&self.justification_bitfield); + s.append(&self.finalized_epoch); + s.append(&self.latest_crosslinks); + s.append(&self.latest_block_roots); + s.append(&self.latest_index_roots); + s.append(&self.latest_penalized_balances); + s.append(&self.latest_attestations); + s.append(&self.batched_block_roots); + s.append(&self.latest_eth1_data); + s.append(&self.eth1_data_votes); + } +} + +impl Decodable for FastBeaconState { + fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { + let (slot, i) = <_>::ssz_decode(bytes, i)?; + let (genesis_time, i) = <_>::ssz_decode(bytes, i)?; + let (fork, i) = <_>::ssz_decode(bytes, i)?; + let (validator_registry, i) = <_>::ssz_decode(bytes, i)?; + let (validator_balances, i) = <_>::ssz_decode(bytes, i)?; + let (validator_registry_update_epoch, i) = <_>::ssz_decode(bytes, i)?; + let (latest_randao_mixes, i) = <_>::ssz_decode(bytes, i)?; + let (previous_epoch_start_shard, i) = <_>::ssz_decode(bytes, i)?; + let (current_epoch_start_shard, i) = <_>::ssz_decode(bytes, i)?; + let (previous_calculation_epoch, i) = <_>::ssz_decode(bytes, i)?; + let (current_calculation_epoch, i) = <_>::ssz_decode(bytes, i)?; + let (previous_epoch_seed, i) = <_>::ssz_decode(bytes, i)?; + let (current_epoch_seed, i) = <_>::ssz_decode(bytes, i)?; + let (previous_justified_epoch, i) = <_>::ssz_decode(bytes, i)?; + let (justified_epoch, i) = <_>::ssz_decode(bytes, i)?; + let (justification_bitfield, i) = <_>::ssz_decode(bytes, i)?; + let (finalized_epoch, i) = <_>::ssz_decode(bytes, i)?; + let (latest_crosslinks, i) = <_>::ssz_decode(bytes, i)?; + let (latest_block_roots, i) = <_>::ssz_decode(bytes, i)?; + let (latest_index_roots, i) = <_>::ssz_decode(bytes, i)?; + let (latest_penalized_balances, i) = <_>::ssz_decode(bytes, i)?; + let (latest_attestations, i) = <_>::ssz_decode(bytes, i)?; + let (batched_block_roots, i) = <_>::ssz_decode(bytes, i)?; + let (latest_eth1_data, i) = <_>::ssz_decode(bytes, i)?; + let (eth1_data_votes, i) = <_>::ssz_decode(bytes, i)?; + + Ok(( + Self { + slot, + genesis_time, + fork, + validator_registry, + validator_balances, + validator_registry_update_epoch, + latest_randao_mixes, + previous_epoch_start_shard, + current_epoch_start_shard, + previous_calculation_epoch, + current_calculation_epoch, + previous_epoch_seed, + current_epoch_seed, + previous_justified_epoch, + justified_epoch, + justification_bitfield, + finalized_epoch, + latest_crosslinks, + latest_block_roots, + latest_index_roots, + latest_penalized_balances, + latest_attestations, + batched_block_roots, + latest_eth1_data, + eth1_data_votes, + }, + i, + )) + } +} + +impl TreeHash for FastBeaconState { + fn hash_tree_root_internal(&self) -> Vec { + let mut result: Vec = vec![]; + result.append(&mut self.slot.hash_tree_root_internal()); + result.append(&mut self.genesis_time.hash_tree_root_internal()); + result.append(&mut self.fork.hash_tree_root_internal()); + result.append(&mut self.validator_registry.hash_tree_root_internal()); + result.append(&mut self.validator_balances.hash_tree_root_internal()); + result.append( + &mut self + .validator_registry_update_epoch + .hash_tree_root_internal(), + ); + result.append(&mut self.latest_randao_mixes.hash_tree_root_internal()); + result.append(&mut self.previous_epoch_start_shard.hash_tree_root_internal()); + result.append(&mut self.current_epoch_start_shard.hash_tree_root_internal()); + result.append(&mut self.previous_calculation_epoch.hash_tree_root_internal()); + result.append(&mut self.current_calculation_epoch.hash_tree_root_internal()); + result.append(&mut self.previous_epoch_seed.hash_tree_root_internal()); + result.append(&mut self.current_epoch_seed.hash_tree_root_internal()); + result.append(&mut self.previous_justified_epoch.hash_tree_root_internal()); + result.append(&mut self.justified_epoch.hash_tree_root_internal()); + result.append(&mut self.justification_bitfield.hash_tree_root_internal()); + result.append(&mut self.finalized_epoch.hash_tree_root_internal()); + result.append(&mut self.latest_crosslinks.hash_tree_root_internal()); + result.append(&mut self.latest_block_roots.hash_tree_root_internal()); + result.append(&mut self.latest_index_roots.hash_tree_root_internal()); + result.append(&mut self.latest_penalized_balances.hash_tree_root_internal()); + result.append(&mut self.latest_attestations.hash_tree_root_internal()); + result.append(&mut self.batched_block_roots.hash_tree_root_internal()); + result.append(&mut self.latest_eth1_data.hash_tree_root_internal()); + result.append(&mut self.eth1_data_votes.hash_tree_root_internal()); + hash(&result) + } +} + +impl TestRandom for FastBeaconState { + fn random_for_test(rng: &mut T) -> Self { + Self { + slot: <_>::random_for_test(rng), + genesis_time: <_>::random_for_test(rng), + fork: <_>::random_for_test(rng), + validator_registry: <_>::random_for_test(rng), + validator_balances: <_>::random_for_test(rng), + validator_registry_update_epoch: <_>::random_for_test(rng), + latest_randao_mixes: <_>::random_for_test(rng), + previous_epoch_start_shard: <_>::random_for_test(rng), + current_epoch_start_shard: <_>::random_for_test(rng), + previous_calculation_epoch: <_>::random_for_test(rng), + current_calculation_epoch: <_>::random_for_test(rng), + previous_epoch_seed: <_>::random_for_test(rng), + current_epoch_seed: <_>::random_for_test(rng), + previous_justified_epoch: <_>::random_for_test(rng), + justified_epoch: <_>::random_for_test(rng), + justification_bitfield: <_>::random_for_test(rng), + finalized_epoch: <_>::random_for_test(rng), + latest_crosslinks: <_>::random_for_test(rng), + latest_block_roots: <_>::random_for_test(rng), + latest_index_roots: <_>::random_for_test(rng), + latest_penalized_balances: <_>::random_for_test(rng), + latest_attestations: <_>::random_for_test(rng), + batched_block_roots: <_>::random_for_test(rng), + latest_eth1_data: <_>::random_for_test(rng), + eth1_data_votes: <_>::random_for_test(rng), + } + } +} diff --git a/eth2/types/src/fast_beacon_state/committees_cache.rs b/eth2/types/src/fast_beacon_state/committees_cache.rs new file mode 100644 index 000000000..42dd101ba --- /dev/null +++ b/eth2/types/src/fast_beacon_state/committees_cache.rs @@ -0,0 +1,128 @@ +use crate::{ + validator_registry::get_active_validator_indices, BeaconState, ChainSpec, Epoch, Hash256, +}; +use honey_badger_split::SplitExt; +use serde_derive::Serialize; +use swap_or_not_shuffle::get_permutated_index; + +pub const CACHED_EPOCHS: usize = 3; + +#[derive(Debug, PartialEq, Clone, Default, Serialize)] +pub struct CommitteesCache { + cache_index_offset: usize, + active_validator_indices_cache: Vec>>, + shuffling_cache: Vec>>>>, + previous_epoch: Epoch, + current_epoch: Epoch, + next_epoch: Epoch, +} + +impl CommitteesCache { + pub fn new(current_epoch: Epoch, spec: &ChainSpec) -> Self { + let previous_epoch = if current_epoch == spec.genesis_epoch { + current_epoch + } else { + current_epoch - 1 + }; + let next_epoch = current_epoch + 1; + + Self { + cache_index_offset: 0, + active_validator_indices_cache: vec![None; CACHED_EPOCHS], + shuffling_cache: vec![None; CACHED_EPOCHS], + previous_epoch, + current_epoch, + next_epoch, + } + } + + pub fn advance_epoch(&mut self) { + let previous_cache_index = self.cache_index(self.previous_epoch); + + self.active_validator_indices_cache[previous_cache_index] = None; + self.shuffling_cache[previous_cache_index] = None; + + self.cache_index_offset += 1; + self.cache_index_offset %= CACHED_EPOCHS; + } + + pub fn cache_index(&self, epoch: Epoch) -> usize { + let base_index = match epoch { + e if e == self.previous_epoch => 0, + e if e == self.current_epoch => 1, + e if e == self.next_epoch => 2, + _ => panic!("Bad cache index."), + }; + + (base_index + self.cache_index_offset) % CACHED_EPOCHS + } + + pub fn get_active_validator_indices( + &mut self, + state: &BeaconState, + epoch: Epoch, + ) -> &Vec { + let i = self.cache_index(epoch); + + if self.active_validator_indices_cache[i] == None { + self.active_validator_indices_cache[i] = Some(get_active_validator_indices( + &state.validator_registry, + epoch, + )); + } + + self.active_validator_indices_cache[i] + .as_ref() + .expect("Cache cannot be None") + } + + pub fn get_shuffling( + &mut self, + state: &BeaconState, + seed: Hash256, + epoch: Epoch, + spec: &ChainSpec, + ) -> Option<&Vec>> { + let cache_index = self.cache_index(epoch); + + if self.shuffling_cache[cache_index] == None { + let active_validator_indices = self.get_active_validator_indices(&state, epoch); + + if active_validator_indices.is_empty() { + return None; + } + + let committees_per_epoch = + state.get_epoch_committee_count(active_validator_indices.len(), spec); + + let mut shuffled_active_validator_indices = vec![0; active_validator_indices.len()]; + for &i in active_validator_indices { + let shuffled_i = get_permutated_index( + i, + active_validator_indices.len(), + &seed[..], + spec.shuffle_round_count, + )?; + shuffled_active_validator_indices[i] = active_validator_indices[shuffled_i] + } + + let committees: Vec> = shuffled_active_validator_indices + .honey_badger_split(committees_per_epoch as usize) + .map(|slice: &[usize]| slice.to_vec()) + .collect(); + + self.shuffling_cache[cache_index] = Some(Some(committees)); + } + + match self.shuffling_cache[cache_index] { + Some(_) => Some( + self.shuffling_cache[cache_index] + .as_ref() + .expect("Cache cannot be None") + .as_ref() + .expect("Shuffling cannot be None."), + ), + None => None, + } + } +} diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index f2c128440..820c26ad6 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -16,6 +16,7 @@ pub mod deposit_input; pub mod eth1_data; pub mod eth1_data_vote; pub mod exit; +pub mod fast_beacon_state; pub mod fork; pub mod free_attestation; pub mod pending_attestation; @@ -52,6 +53,7 @@ pub use crate::deposit_input::DepositInput; pub use crate::eth1_data::Eth1Data; pub use crate::eth1_data_vote::Eth1DataVote; pub use crate::exit::Exit; +pub use crate::fast_beacon_state::FastBeaconState; pub use crate::fork::Fork; pub use crate::free_attestation::FreeAttestation; pub use crate::pending_attestation::PendingAttestation; From cdc03f1749e5f27a1516269232bd15b0bca440f8 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 22 Feb 2019 17:55:36 +1300 Subject: [PATCH 2/9] Remove FastBeaconState --- eth2/types/src/fast_beacon_state.rs | 1136 ----------------- .../src/fast_beacon_state/committees_cache.rs | 128 -- eth2/types/src/lib.rs | 2 - 3 files changed, 1266 deletions(-) delete mode 100644 eth2/types/src/fast_beacon_state.rs delete mode 100644 eth2/types/src/fast_beacon_state/committees_cache.rs diff --git a/eth2/types/src/fast_beacon_state.rs b/eth2/types/src/fast_beacon_state.rs deleted file mode 100644 index 7e56df8ae..000000000 --- a/eth2/types/src/fast_beacon_state.rs +++ /dev/null @@ -1,1136 +0,0 @@ -use crate::test_utils::TestRandom; -use crate::{ - validator::StatusFlags, validator_registry::get_active_validator_indices, AttestationData, - Bitfield, ChainSpec, Crosslink, Deposit, Epoch, Eth1Data, Eth1DataVote, Fork, Hash256, - PendingAttestation, PublicKey, Signature, Slot, Validator, -}; -use bls::verify_proof_of_possession; -use committees_cache::CommitteesCache; -use honey_badger_split::SplitExt; -use log::trace; -use rand::RngCore; -use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; -use swap_or_not_shuffle::get_permutated_index; - -mod committees_cache; - -#[derive(Debug, PartialEq)] -pub enum FastBeaconStateError { - EpochOutOfBounds, - UnableToShuffle, - InsufficientRandaoMixes, - InsufficientValidators, - InsufficientBlockRoots, - InsufficientIndexRoots, - InsufficientAttestations, - InsufficientCommittees, -} - -#[derive(Debug, PartialEq)] -pub enum InclusionError { - /// The validator did not participate in an attestation in this period. - NoAttestationsForValidator, - AttestationParticipantsError(AttestationParticipantsError), -} - -#[derive(Debug, PartialEq)] -pub enum AttestationParticipantsError { - /// There is no committee for the given shard in the given epoch. - NoCommitteeForShard, - FastBeaconStateError(FastBeaconStateError), -} - -macro_rules! safe_add_assign { - ($a: expr, $b: expr) => { - $a = $a.saturating_add($b); - }; -} -macro_rules! safe_sub_assign { - ($a: expr, $b: expr) => { - $a = $a.saturating_sub($b); - }; -} - -#[derive(Debug, PartialEq, Clone, Default, Serialize)] -pub struct FastBeaconState { - // Misc - pub slot: Slot, - pub genesis_time: u64, - pub fork: Fork, - - // Validator registry - pub validator_registry: Vec, - pub validator_balances: Vec, - pub validator_registry_update_epoch: Epoch, - - // Randomness and committees - pub latest_randao_mixes: Vec, - pub previous_epoch_start_shard: u64, - pub current_epoch_start_shard: u64, - pub previous_calculation_epoch: Epoch, - pub current_calculation_epoch: Epoch, - pub previous_epoch_seed: Hash256, - pub current_epoch_seed: Hash256, - - // Finality - pub previous_justified_epoch: Epoch, - pub justified_epoch: Epoch, - pub justification_bitfield: u64, - pub finalized_epoch: Epoch, - - // Recent state - pub latest_crosslinks: Vec, - pub latest_block_roots: Vec, - pub latest_index_roots: Vec, - pub latest_penalized_balances: Vec, - pub latest_attestations: Vec, - pub batched_block_roots: Vec, - - // Ethereum 1.0 chain data - pub latest_eth1_data: Eth1Data, - pub eth1_data_votes: Vec, - - // Cache - committees_cache: CommitteesCache, -} - -impl FastBeaconState { - /// Produce the first state of the Beacon Chain. - pub fn genesis( - genesis_time: u64, - initial_validator_deposits: Vec, - latest_eth1_data: Eth1Data, - spec: &ChainSpec, - ) -> Result { - let initial_crosslink = Crosslink { - epoch: spec.genesis_epoch, - shard_block_root: spec.zero_hash, - }; - - let mut genesis_state = FastBeaconState { - /* - * Misc - */ - slot: spec.genesis_slot, - genesis_time, - fork: Fork { - previous_version: spec.genesis_fork_version, - current_version: spec.genesis_fork_version, - epoch: spec.genesis_epoch, - }, - - /* - * Validator registry - */ - validator_registry: vec![], // Set later in the function. - validator_balances: vec![], // Set later in the function. - validator_registry_update_epoch: spec.genesis_epoch, - - /* - * Randomness and committees - */ - latest_randao_mixes: vec![spec.zero_hash; spec.latest_randao_mixes_length as usize], - previous_epoch_start_shard: spec.genesis_start_shard, - current_epoch_start_shard: spec.genesis_start_shard, - previous_calculation_epoch: spec.genesis_epoch, - current_calculation_epoch: spec.genesis_epoch, - previous_epoch_seed: spec.zero_hash, - current_epoch_seed: spec.zero_hash, - - /* - * Finality - */ - previous_justified_epoch: spec.genesis_epoch, - justified_epoch: spec.genesis_epoch, - justification_bitfield: 0, - finalized_epoch: spec.genesis_epoch, - - /* - * Recent state - */ - latest_crosslinks: vec![initial_crosslink; spec.shard_count as usize], - latest_block_roots: vec![spec.zero_hash; spec.latest_block_roots_length as usize], - latest_index_roots: vec![spec.zero_hash; spec.latest_index_roots_length as usize], - latest_penalized_balances: vec![0; spec.latest_penalized_exit_length as usize], - latest_attestations: vec![], - batched_block_roots: vec![], - - /* - * PoW receipt root - */ - latest_eth1_data, - eth1_data_votes: vec![], - - /* - * Caches (not in spec) - */ - committees_cache: CommitteesCache::new(spec.genesis_epoch, spec), - }; - - for deposit in initial_validator_deposits { - let _index = genesis_state.process_deposit( - deposit.deposit_data.deposit_input.pubkey, - deposit.deposit_data.amount, - deposit.deposit_data.deposit_input.proof_of_possession, - deposit.deposit_data.deposit_input.withdrawal_credentials, - spec, - ); - } - - for validator_index in 0..genesis_state.validator_registry.len() { - if genesis_state.get_effective_balance(validator_index, spec) >= spec.max_deposit_amount - { - genesis_state.activate_validator(validator_index, true, spec); - } - } - - let genesis_active_index_root = hash_tree_root(get_active_validator_indices( - &genesis_state.validator_registry, - spec.genesis_epoch, - )); - genesis_state.latest_index_roots = - vec![genesis_active_index_root; spec.latest_index_roots_length]; - genesis_state.current_epoch_seed = genesis_state.generate_seed(spec.genesis_epoch, spec)?; - - Ok(genesis_state) - } - - /// Return the tree hash root for this `FastBeaconState`. - /// - /// Spec v0.2.0 - pub fn canonical_root(&self) -> Hash256 { - Hash256::from(&self.hash_tree_root()[..]) - } - - /// The epoch corresponding to `self.slot`. - /// - /// Spec v0.2.0 - pub fn current_epoch(&self, spec: &ChainSpec) -> Epoch { - self.slot.epoch(spec.epoch_length) - } - - /// The epoch prior to `self.current_epoch()`. - /// - /// Spec v0.2.0 - pub fn previous_epoch(&self, spec: &ChainSpec) -> Epoch { - let current_epoch = self.current_epoch(&spec); - if current_epoch == spec.genesis_epoch { - current_epoch - } else { - current_epoch - 1 - } - } - - /// The epoch following `self.current_epoch()`. - /// - /// Spec v0.2.0 - pub fn next_epoch(&self, spec: &ChainSpec) -> Epoch { - self.current_epoch(spec).saturating_add(1_u64) - } - - /// The first slot of the epoch corresponding to `self.slot`. - /// - /// Spec v0.2.0 - pub fn current_epoch_start_slot(&self, spec: &ChainSpec) -> Slot { - self.current_epoch(spec).start_slot(spec.epoch_length) - } - - /// The first slot of the epoch preceeding the one corresponding to `self.slot`. - /// - /// Spec v0.2.0 - pub fn previous_epoch_start_slot(&self, spec: &ChainSpec) -> Slot { - self.previous_epoch(spec).start_slot(spec.epoch_length) - } - - /// Return the number of committees in one epoch. - /// - /// TODO: this should probably be a method on `ChainSpec`. - /// - /// Spec v0.2.0 - pub fn get_epoch_committee_count( - &self, - active_validator_count: usize, - spec: &ChainSpec, - ) -> u64 { - std::cmp::max( - 1, - std::cmp::min( - spec.shard_count / spec.epoch_length, - active_validator_count as u64 / spec.epoch_length / spec.target_committee_size, - ), - ) * spec.epoch_length - } - - /// Shuffle ``validators`` into crosslink committees seeded by ``seed`` and ``epoch``. - /// Return a list of ``committees_per_epoch`` committees where each - /// committee is itself a list of validator indices. - /// - /// Spec v0.1 - pub fn get_shuffling( - &self, - seed: Hash256, - epoch: Epoch, - spec: &ChainSpec, - ) -> Option>> { - let active_validator_indices = - get_active_validator_indices(&self.validator_registry, epoch); - - if active_validator_indices.is_empty() { - return None; - } - - trace!( - "get_shuffling: active_validator_indices.len() == {}", - active_validator_indices.len() - ); - - let committees_per_epoch = - self.get_epoch_committee_count(active_validator_indices.len(), spec); - - trace!( - "get_shuffling: active_validator_indices.len() == {}, committees_per_epoch: {}", - active_validator_indices.len(), - committees_per_epoch - ); - - let mut shuffled_active_validator_indices = vec![0; active_validator_indices.len()]; - for &i in &active_validator_indices { - let shuffled_i = get_permutated_index( - i, - active_validator_indices.len(), - &seed[..], - spec.shuffle_round_count, - )?; - shuffled_active_validator_indices[i] = active_validator_indices[shuffled_i] - } - - Some( - shuffled_active_validator_indices - .honey_badger_split(committees_per_epoch as usize) - .map(|slice: &[usize]| slice.to_vec()) - .collect(), - ) - } - - /// Return the number of committees in the previous epoch. - /// - /// Spec v0.2.0 - fn get_previous_epoch_committee_count(&self, spec: &ChainSpec) -> u64 { - let previous_active_validators = - get_active_validator_indices(&self.validator_registry, self.previous_calculation_epoch); - self.get_epoch_committee_count(previous_active_validators.len(), spec) - } - - /// Return the number of committees in the current epoch. - /// - /// Spec v0.2.0 - pub fn get_current_epoch_committee_count(&self, spec: &ChainSpec) -> u64 { - let current_active_validators = - get_active_validator_indices(&self.validator_registry, self.current_calculation_epoch); - self.get_epoch_committee_count(current_active_validators.len(), spec) - } - - /// Return the number of committees in the next epoch. - /// - /// Spec v0.2.0 - pub fn get_next_epoch_committee_count(&self, spec: &ChainSpec) -> u64 { - let current_active_validators = - get_active_validator_indices(&self.validator_registry, self.next_epoch(spec)); - self.get_epoch_committee_count(current_active_validators.len(), spec) - } - - pub fn get_active_index_root(&self, epoch: Epoch, spec: &ChainSpec) -> Option { - let current_epoch = self.current_epoch(spec); - - let earliest_index_root = current_epoch - Epoch::from(spec.latest_index_roots_length) - + Epoch::from(spec.entry_exit_delay) - + 1; - let latest_index_root = current_epoch + spec.entry_exit_delay; - - trace!( - "get_active_index_root: epoch: {}, earliest: {}, latest: {}", - epoch, - earliest_index_root, - latest_index_root - ); - - if (epoch >= earliest_index_root) & (epoch <= latest_index_root) { - Some(self.latest_index_roots[epoch.as_usize() % spec.latest_index_roots_length]) - } else { - trace!("get_active_index_root: epoch out of range."); - None - } - } - - /// Generate a seed for the given ``epoch``. - /// - /// Spec v0.2.0 - pub fn generate_seed( - &self, - epoch: Epoch, - spec: &ChainSpec, - ) -> Result { - let mut input = self - .get_randao_mix(epoch, spec) - .ok_or_else(|| FastBeaconStateError::InsufficientRandaoMixes)? - .to_vec(); - - input.append( - &mut self - .get_active_index_root(epoch, spec) - .ok_or_else(|| FastBeaconStateError::InsufficientIndexRoots)? - .to_vec(), - ); - - // TODO: ensure `Hash256::from(u64)` == `int_to_bytes32`. - input.append(&mut Hash256::from(epoch.as_u64()).to_vec()); - - Ok(Hash256::from(&hash(&input[..])[..])) - } - - /// Return the list of ``(committee, shard)`` tuples for the ``slot``. - /// - /// Note: There are two possible shufflings for crosslink committees for a - /// `slot` in the next epoch: with and without a `registry_change` - /// - /// Spec v0.2.0 - pub fn get_crosslink_committees_at_slot( - &self, - slot: Slot, - registry_change: bool, - spec: &ChainSpec, - ) -> Result, u64)>, FastBeaconStateError> { - let epoch = slot.epoch(spec.epoch_length); - let current_epoch = self.current_epoch(spec); - let previous_epoch = self.previous_epoch(spec); - let next_epoch = self.next_epoch(spec); - - let (committees_per_epoch, seed, shuffling_epoch, shuffling_start_shard) = - if epoch == current_epoch { - trace!("get_crosslink_committees_at_slot: current_epoch"); - ( - self.get_current_epoch_committee_count(spec), - self.current_epoch_seed, - self.current_calculation_epoch, - self.current_epoch_start_shard, - ) - } else if epoch == previous_epoch { - trace!("get_crosslink_committees_at_slot: previous_epoch"); - ( - self.get_previous_epoch_committee_count(spec), - self.previous_epoch_seed, - self.previous_calculation_epoch, - self.previous_epoch_start_shard, - ) - } else if epoch == next_epoch { - trace!("get_crosslink_committees_at_slot: next_epoch"); - let current_committees_per_epoch = self.get_current_epoch_committee_count(spec); - let epochs_since_last_registry_update = - current_epoch - self.validator_registry_update_epoch; - let (seed, shuffling_start_shard) = if registry_change { - let next_seed = self.generate_seed(next_epoch, spec)?; - ( - next_seed, - (self.current_epoch_start_shard + current_committees_per_epoch) - % spec.shard_count, - ) - } else if (epochs_since_last_registry_update > 1) - & epochs_since_last_registry_update.is_power_of_two() - { - let next_seed = self.generate_seed(next_epoch, spec)?; - (next_seed, self.current_epoch_start_shard) - } else { - (self.current_epoch_seed, self.current_epoch_start_shard) - }; - ( - self.get_next_epoch_committee_count(spec), - seed, - next_epoch, - shuffling_start_shard, - ) - } else { - return Err(FastBeaconStateError::EpochOutOfBounds); - }; - - let shuffling = self - .get_shuffling(seed, shuffling_epoch, spec) - .ok_or_else(|| FastBeaconStateError::UnableToShuffle)?; - let offset = slot.as_u64() % spec.epoch_length; - let committees_per_slot = committees_per_epoch / spec.epoch_length; - let slot_start_shard = - (shuffling_start_shard + committees_per_slot * offset) % spec.shard_count; - - trace!( - "get_crosslink_committees_at_slot: committees_per_slot: {}, slot_start_shard: {}, seed: {}", - committees_per_slot, - slot_start_shard, - seed - ); - - let mut crosslinks_at_slot = vec![]; - for i in 0..committees_per_slot { - let tuple = ( - shuffling[(committees_per_slot * offset + i) as usize].clone(), - (slot_start_shard + i) % spec.shard_count, - ); - crosslinks_at_slot.push(tuple) - } - Ok(crosslinks_at_slot) - } - - /// Returns the `slot`, `shard` and `committee_index` for which a validator must produce an - /// attestation. - /// - /// Spec v0.2.0 - pub fn attestation_slot_and_shard_for_validator( - &self, - validator_index: usize, - spec: &ChainSpec, - ) -> Result, FastBeaconStateError> { - let mut result = None; - for slot in self.current_epoch(spec).slot_iter(spec.epoch_length) { - for (committee, shard) in self.get_crosslink_committees_at_slot(slot, false, spec)? { - if let Some(committee_index) = committee.iter().position(|&i| i == validator_index) - { - result = Some((slot, shard, committee_index as u64)); - } - } - } - Ok(result) - } - - /// An entry or exit triggered in the ``epoch`` given by the input takes effect at - /// the epoch given by the output. - /// - /// Spec v0.2.0 - pub fn get_entry_exit_effect_epoch(&self, epoch: Epoch, spec: &ChainSpec) -> Epoch { - epoch + 1 + spec.entry_exit_delay - } - - /// Returns the beacon proposer index for the `slot`. - /// - /// If the state does not contain an index for a beacon proposer at the requested `slot`, then `None` is returned. - /// - /// Spec v0.2.0 - pub fn get_beacon_proposer_index( - &self, - slot: Slot, - spec: &ChainSpec, - ) -> Result { - let committees = self.get_crosslink_committees_at_slot(slot, false, spec)?; - trace!( - "get_beacon_proposer_index: slot: {}, committees_count: {}", - slot, - committees.len() - ); - committees - .first() - .ok_or(FastBeaconStateError::InsufficientValidators) - .and_then(|(first_committee, _)| { - let index = (slot.as_usize()) - .checked_rem(first_committee.len()) - .ok_or(FastBeaconStateError::InsufficientValidators)?; - Ok(first_committee[index]) - }) - } - - /// Process the penalties and prepare the validators who are eligible to withdrawal. - /// - /// Spec v0.2.0 - pub fn process_penalties_and_exits(&mut self, spec: &ChainSpec) { - let current_epoch = self.current_epoch(spec); - let active_validator_indices = - get_active_validator_indices(&self.validator_registry, current_epoch); - let total_balance = self.get_total_balance(&active_validator_indices[..], spec); - - for index in 0..self.validator_balances.len() { - let validator = &self.validator_registry[index]; - - if current_epoch - == validator.penalized_epoch + Epoch::from(spec.latest_penalized_exit_length / 2) - { - let epoch_index: usize = - current_epoch.as_usize() % spec.latest_penalized_exit_length; - - let total_at_start = self.latest_penalized_balances - [(epoch_index + 1) % spec.latest_penalized_exit_length]; - let total_at_end = self.latest_penalized_balances[epoch_index]; - let total_penalities = total_at_end.saturating_sub(total_at_start); - let penalty = self.get_effective_balance(index, spec) - * std::cmp::min(total_penalities * 3, total_balance) - / total_balance; - safe_sub_assign!(self.validator_balances[index], penalty); - } - } - - let eligible = |index: usize| { - let validator = &self.validator_registry[index]; - - if validator.penalized_epoch <= current_epoch { - let penalized_withdrawal_epochs = spec.latest_penalized_exit_length / 2; - current_epoch >= validator.penalized_epoch + penalized_withdrawal_epochs as u64 - } else { - current_epoch >= validator.exit_epoch + spec.min_validator_withdrawal_epochs - } - }; - - let mut eligable_indices: Vec = (0..self.validator_registry.len()) - .filter(|i| eligible(*i)) - .collect(); - eligable_indices.sort_by_key(|i| self.validator_registry[*i].exit_epoch); - for (withdrawn_so_far, index) in eligable_indices.iter().enumerate() { - self.prepare_validator_for_withdrawal(*index); - if withdrawn_so_far as u64 >= spec.max_withdrawals_per_epoch { - break; - } - } - } - - /// Return the randao mix at a recent ``epoch``. - /// - /// Returns `None` if the epoch is out-of-bounds of `self.latest_randao_mixes`. - /// - /// Spec v0.2.0 - pub fn get_randao_mix(&self, epoch: Epoch, spec: &ChainSpec) -> Option<&Hash256> { - self.latest_randao_mixes - .get(epoch.as_usize() % spec.latest_randao_mixes_length) - } - - /// Update validator registry, activating/exiting validators if possible. - /// - /// Spec v0.2.0 - pub fn update_validator_registry(&mut self, spec: &ChainSpec) { - let current_epoch = self.current_epoch(spec); - let active_validator_indices = - get_active_validator_indices(&self.validator_registry, current_epoch); - let total_balance = self.get_total_balance(&active_validator_indices[..], spec); - - let max_balance_churn = std::cmp::max( - spec.max_deposit_amount, - total_balance / (2 * spec.max_balance_churn_quotient), - ); - - let mut balance_churn = 0; - for index in 0..self.validator_registry.len() { - let validator = &self.validator_registry[index]; - - if (validator.activation_epoch > self.get_entry_exit_effect_epoch(current_epoch, spec)) - && self.validator_balances[index] >= spec.max_deposit_amount - { - balance_churn += self.get_effective_balance(index, spec); - if balance_churn > max_balance_churn { - break; - } - self.activate_validator(index, false, spec); - } - } - - let mut balance_churn = 0; - for index in 0..self.validator_registry.len() { - let validator = &self.validator_registry[index]; - - if (validator.exit_epoch > self.get_entry_exit_effect_epoch(current_epoch, spec)) - && validator.status_flags == Some(StatusFlags::InitiatedExit) - { - balance_churn += self.get_effective_balance(index, spec); - if balance_churn > max_balance_churn { - break; - } - - self.exit_validator(index, spec); - } - } - - self.validator_registry_update_epoch = current_epoch; - } - /// Process a validator deposit, returning the validator index if the deposit is valid. - /// - /// Spec v0.2.0 - pub fn process_deposit( - &mut self, - pubkey: PublicKey, - amount: u64, - proof_of_possession: Signature, - withdrawal_credentials: Hash256, - spec: &ChainSpec, - ) -> Result { - // TODO: ensure verify proof-of-possession represents the spec accurately. - if !verify_proof_of_possession(&proof_of_possession, &pubkey) { - return Err(()); - } - - if let Some(index) = self - .validator_registry - .iter() - .position(|v| v.pubkey == pubkey) - { - if self.validator_registry[index].withdrawal_credentials == withdrawal_credentials { - safe_add_assign!(self.validator_balances[index], amount); - Ok(index) - } else { - Err(()) - } - } else { - let validator = Validator { - pubkey, - withdrawal_credentials, - activation_epoch: spec.far_future_epoch, - exit_epoch: spec.far_future_epoch, - withdrawal_epoch: spec.far_future_epoch, - penalized_epoch: spec.far_future_epoch, - status_flags: None, - }; - self.validator_registry.push(validator); - self.validator_balances.push(amount); - Ok(self.validator_registry.len() - 1) - } - } - - /// Activate the validator of the given ``index``. - /// - /// Spec v0.2.0 - pub fn activate_validator( - &mut self, - validator_index: usize, - is_genesis: bool, - spec: &ChainSpec, - ) { - let current_epoch = self.current_epoch(spec); - - self.validator_registry[validator_index].activation_epoch = if is_genesis { - spec.genesis_epoch - } else { - self.get_entry_exit_effect_epoch(current_epoch, spec) - } - } - - /// Initiate an exit for the validator of the given `index`. - /// - /// Spec v0.2.0 - pub fn initiate_validator_exit(&mut self, validator_index: usize) { - // TODO: the spec does an `|=` here, ensure this isn't buggy. - self.validator_registry[validator_index].status_flags = Some(StatusFlags::InitiatedExit); - } - - /// Exit the validator of the given `index`. - /// - /// Spec v0.2.0 - fn exit_validator(&mut self, validator_index: usize, spec: &ChainSpec) { - let current_epoch = self.current_epoch(spec); - - if self.validator_registry[validator_index].exit_epoch - <= self.get_entry_exit_effect_epoch(current_epoch, spec) - { - return; - } - - self.validator_registry[validator_index].exit_epoch = - self.get_entry_exit_effect_epoch(current_epoch, spec); - } - - /// Penalize the validator of the given ``index``. - /// - /// Exits the validator and assigns its effective balance to the block producer for this - /// state. - /// - /// Spec v0.2.0 - pub fn penalize_validator( - &mut self, - validator_index: usize, - spec: &ChainSpec, - ) -> Result<(), FastBeaconStateError> { - self.exit_validator(validator_index, spec); - let current_epoch = self.current_epoch(spec); - - self.latest_penalized_balances - [current_epoch.as_usize() % spec.latest_penalized_exit_length] += - self.get_effective_balance(validator_index, spec); - - let whistleblower_index = self.get_beacon_proposer_index(self.slot, spec)?; - let whistleblower_reward = self.get_effective_balance(validator_index, spec); - safe_add_assign!( - self.validator_balances[whistleblower_index as usize], - whistleblower_reward - ); - safe_sub_assign!( - self.validator_balances[validator_index], - whistleblower_reward - ); - self.validator_registry[validator_index].penalized_epoch = current_epoch; - Ok(()) - } - - /// Initiate an exit for the validator of the given `index`. - /// - /// Spec v0.2.0 - pub fn prepare_validator_for_withdrawal(&mut self, validator_index: usize) { - //TODO: we're not ANDing here, we're setting. Potentially wrong. - self.validator_registry[validator_index].status_flags = Some(StatusFlags::Withdrawable); - } - - /// Iterate through the validator registry and eject active validators with balance below - /// ``EJECTION_BALANCE``. - /// - /// Spec v0.2.0 - pub fn process_ejections(&mut self, spec: &ChainSpec) { - for validator_index in - get_active_validator_indices(&self.validator_registry, self.current_epoch(spec)) - { - if self.validator_balances[validator_index] < spec.ejection_balance { - self.exit_validator(validator_index, spec) - } - } - } - - /// Returns the penality that should be applied to some validator for inactivity. - /// - /// Note: this is defined "inline" in the spec, not as a helper function. - /// - /// Spec v0.2.0 - pub fn inactivity_penalty( - &self, - validator_index: usize, - epochs_since_finality: Epoch, - base_reward_quotient: u64, - spec: &ChainSpec, - ) -> u64 { - let effective_balance = self.get_effective_balance(validator_index, spec); - self.base_reward(validator_index, base_reward_quotient, spec) - + effective_balance * epochs_since_finality.as_u64() - / spec.inactivity_penalty_quotient - / 2 - } - - /// Returns the distance between the first included attestation for some validator and this - /// slot. - /// - /// Note: In the spec this is defined "inline", not as a helper function. - /// - /// Spec v0.2.0 - pub fn inclusion_distance( - &self, - attestations: &[&PendingAttestation], - validator_index: usize, - spec: &ChainSpec, - ) -> Result { - let attestation = - self.earliest_included_attestation(attestations, validator_index, spec)?; - Ok((attestation.inclusion_slot - attestation.data.slot).as_u64()) - } - - /// Returns the slot of the earliest included attestation for some validator. - /// - /// Note: In the spec this is defined "inline", not as a helper function. - /// - /// Spec v0.2.0 - pub fn inclusion_slot( - &self, - attestations: &[&PendingAttestation], - validator_index: usize, - spec: &ChainSpec, - ) -> Result { - let attestation = - self.earliest_included_attestation(attestations, validator_index, spec)?; - Ok(attestation.inclusion_slot) - } - - /// Finds the earliest included attestation for some validator. - /// - /// Note: In the spec this is defined "inline", not as a helper function. - /// - /// Spec v0.2.0 - fn earliest_included_attestation( - &self, - attestations: &[&PendingAttestation], - validator_index: usize, - spec: &ChainSpec, - ) -> Result { - let mut included_attestations = vec![]; - - for (i, a) in attestations.iter().enumerate() { - let participants = - self.get_attestation_participants(&a.data, &a.aggregation_bitfield, spec)?; - if participants.iter().any(|i| *i == validator_index) { - included_attestations.push(i); - } - } - - let earliest_attestation_index = included_attestations - .iter() - .min_by_key(|i| attestations[**i].inclusion_slot) - .ok_or_else(|| InclusionError::NoAttestationsForValidator)?; - Ok(attestations[*earliest_attestation_index].clone()) - } - - /// Returns the base reward for some validator. - /// - /// Note: In the spec this is defined "inline", not as a helper function. - /// - /// Spec v0.2.0 - pub fn base_reward( - &self, - validator_index: usize, - base_reward_quotient: u64, - spec: &ChainSpec, - ) -> u64 { - self.get_effective_balance(validator_index, spec) / base_reward_quotient / 5 - } - - /// Return the combined effective balance of an array of validators. - /// - /// Spec v0.2.0 - pub fn get_total_balance(&self, validator_indices: &[usize], spec: &ChainSpec) -> u64 { - validator_indices - .iter() - .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)) - } - - /// Return the effective balance (also known as "balance at stake") for a validator with the given ``index``. - /// - /// Spec v0.2.0 - pub fn get_effective_balance(&self, validator_index: usize, spec: &ChainSpec) -> u64 { - std::cmp::min( - self.validator_balances[validator_index], - spec.max_deposit_amount, - ) - } - - /// Return the block root at a recent `slot`. - /// - /// Spec v0.2.0 - pub fn get_block_root(&self, slot: Slot, spec: &ChainSpec) -> Option<&Hash256> { - self.latest_block_roots - .get(slot.as_usize() % spec.latest_block_roots_length) - } - - pub fn get_attestation_participants_union( - &self, - attestations: &[&PendingAttestation], - spec: &ChainSpec, - ) -> Result, AttestationParticipantsError> { - let mut all_participants = attestations.iter().try_fold::<_, _, Result< - Vec, - AttestationParticipantsError, - >>(vec![], |mut acc, a| { - acc.append(&mut self.get_attestation_participants( - &a.data, - &a.aggregation_bitfield, - spec, - )?); - Ok(acc) - })?; - all_participants.sort_unstable(); - all_participants.dedup(); - Ok(all_participants) - } - - /// Return the participant indices at for the ``attestation_data`` and ``bitfield``. - /// - /// In effect, this converts the "committee indices" on the bitfield into "validator indices" - /// for self.validator_registy. - /// - /// Spec v0.2.0 - pub fn get_attestation_participants( - &self, - attestation_data: &AttestationData, - bitfield: &Bitfield, - spec: &ChainSpec, - ) -> Result, AttestationParticipantsError> { - let crosslink_committees = - self.get_crosslink_committees_at_slot(attestation_data.slot, false, spec)?; - - let committee_index: usize = crosslink_committees - .iter() - .position(|(_committee, shard)| *shard == attestation_data.shard) - .ok_or_else(|| AttestationParticipantsError::NoCommitteeForShard)?; - let (crosslink_committee, _shard) = &crosslink_committees[committee_index]; - - /* - * TODO: verify bitfield length is valid. - */ - - let mut participants = vec![]; - for (i, validator_index) in crosslink_committee.iter().enumerate() { - if bitfield.get(i).unwrap() { - participants.push(*validator_index); - } - } - Ok(participants) - } -} - -fn hash_tree_root(input: Vec) -> Hash256 { - Hash256::from(&input.hash_tree_root()[..]) -} - -impl From for AttestationParticipantsError { - fn from(e: FastBeaconStateError) -> AttestationParticipantsError { - AttestationParticipantsError::FastBeaconStateError(e) - } -} - -impl From for InclusionError { - fn from(e: AttestationParticipantsError) -> InclusionError { - InclusionError::AttestationParticipantsError(e) - } -} - -impl Encodable for FastBeaconState { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.slot); - s.append(&self.genesis_time); - s.append(&self.fork); - s.append(&self.validator_registry); - s.append(&self.validator_balances); - s.append(&self.validator_registry_update_epoch); - s.append(&self.latest_randao_mixes); - s.append(&self.previous_epoch_start_shard); - s.append(&self.current_epoch_start_shard); - s.append(&self.previous_calculation_epoch); - s.append(&self.current_calculation_epoch); - s.append(&self.previous_epoch_seed); - s.append(&self.current_epoch_seed); - s.append(&self.previous_justified_epoch); - s.append(&self.justified_epoch); - s.append(&self.justification_bitfield); - s.append(&self.finalized_epoch); - s.append(&self.latest_crosslinks); - s.append(&self.latest_block_roots); - s.append(&self.latest_index_roots); - s.append(&self.latest_penalized_balances); - s.append(&self.latest_attestations); - s.append(&self.batched_block_roots); - s.append(&self.latest_eth1_data); - s.append(&self.eth1_data_votes); - } -} - -impl Decodable for FastBeaconState { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (slot, i) = <_>::ssz_decode(bytes, i)?; - let (genesis_time, i) = <_>::ssz_decode(bytes, i)?; - let (fork, i) = <_>::ssz_decode(bytes, i)?; - let (validator_registry, i) = <_>::ssz_decode(bytes, i)?; - let (validator_balances, i) = <_>::ssz_decode(bytes, i)?; - let (validator_registry_update_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (latest_randao_mixes, i) = <_>::ssz_decode(bytes, i)?; - let (previous_epoch_start_shard, i) = <_>::ssz_decode(bytes, i)?; - let (current_epoch_start_shard, i) = <_>::ssz_decode(bytes, i)?; - let (previous_calculation_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (current_calculation_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (previous_epoch_seed, i) = <_>::ssz_decode(bytes, i)?; - let (current_epoch_seed, i) = <_>::ssz_decode(bytes, i)?; - let (previous_justified_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (justified_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (justification_bitfield, i) = <_>::ssz_decode(bytes, i)?; - let (finalized_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (latest_crosslinks, i) = <_>::ssz_decode(bytes, i)?; - let (latest_block_roots, i) = <_>::ssz_decode(bytes, i)?; - let (latest_index_roots, i) = <_>::ssz_decode(bytes, i)?; - let (latest_penalized_balances, i) = <_>::ssz_decode(bytes, i)?; - let (latest_attestations, i) = <_>::ssz_decode(bytes, i)?; - let (batched_block_roots, i) = <_>::ssz_decode(bytes, i)?; - let (latest_eth1_data, i) = <_>::ssz_decode(bytes, i)?; - let (eth1_data_votes, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - Self { - slot, - genesis_time, - fork, - validator_registry, - validator_balances, - validator_registry_update_epoch, - latest_randao_mixes, - previous_epoch_start_shard, - current_epoch_start_shard, - previous_calculation_epoch, - current_calculation_epoch, - previous_epoch_seed, - current_epoch_seed, - previous_justified_epoch, - justified_epoch, - justification_bitfield, - finalized_epoch, - latest_crosslinks, - latest_block_roots, - latest_index_roots, - latest_penalized_balances, - latest_attestations, - batched_block_roots, - latest_eth1_data, - eth1_data_votes, - }, - i, - )) - } -} - -impl TreeHash for FastBeaconState { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.slot.hash_tree_root_internal()); - result.append(&mut self.genesis_time.hash_tree_root_internal()); - result.append(&mut self.fork.hash_tree_root_internal()); - result.append(&mut self.validator_registry.hash_tree_root_internal()); - result.append(&mut self.validator_balances.hash_tree_root_internal()); - result.append( - &mut self - .validator_registry_update_epoch - .hash_tree_root_internal(), - ); - result.append(&mut self.latest_randao_mixes.hash_tree_root_internal()); - result.append(&mut self.previous_epoch_start_shard.hash_tree_root_internal()); - result.append(&mut self.current_epoch_start_shard.hash_tree_root_internal()); - result.append(&mut self.previous_calculation_epoch.hash_tree_root_internal()); - result.append(&mut self.current_calculation_epoch.hash_tree_root_internal()); - result.append(&mut self.previous_epoch_seed.hash_tree_root_internal()); - result.append(&mut self.current_epoch_seed.hash_tree_root_internal()); - result.append(&mut self.previous_justified_epoch.hash_tree_root_internal()); - result.append(&mut self.justified_epoch.hash_tree_root_internal()); - result.append(&mut self.justification_bitfield.hash_tree_root_internal()); - result.append(&mut self.finalized_epoch.hash_tree_root_internal()); - result.append(&mut self.latest_crosslinks.hash_tree_root_internal()); - result.append(&mut self.latest_block_roots.hash_tree_root_internal()); - result.append(&mut self.latest_index_roots.hash_tree_root_internal()); - result.append(&mut self.latest_penalized_balances.hash_tree_root_internal()); - result.append(&mut self.latest_attestations.hash_tree_root_internal()); - result.append(&mut self.batched_block_roots.hash_tree_root_internal()); - result.append(&mut self.latest_eth1_data.hash_tree_root_internal()); - result.append(&mut self.eth1_data_votes.hash_tree_root_internal()); - hash(&result) - } -} - -impl TestRandom for FastBeaconState { - fn random_for_test(rng: &mut T) -> Self { - Self { - slot: <_>::random_for_test(rng), - genesis_time: <_>::random_for_test(rng), - fork: <_>::random_for_test(rng), - validator_registry: <_>::random_for_test(rng), - validator_balances: <_>::random_for_test(rng), - validator_registry_update_epoch: <_>::random_for_test(rng), - latest_randao_mixes: <_>::random_for_test(rng), - previous_epoch_start_shard: <_>::random_for_test(rng), - current_epoch_start_shard: <_>::random_for_test(rng), - previous_calculation_epoch: <_>::random_for_test(rng), - current_calculation_epoch: <_>::random_for_test(rng), - previous_epoch_seed: <_>::random_for_test(rng), - current_epoch_seed: <_>::random_for_test(rng), - previous_justified_epoch: <_>::random_for_test(rng), - justified_epoch: <_>::random_for_test(rng), - justification_bitfield: <_>::random_for_test(rng), - finalized_epoch: <_>::random_for_test(rng), - latest_crosslinks: <_>::random_for_test(rng), - latest_block_roots: <_>::random_for_test(rng), - latest_index_roots: <_>::random_for_test(rng), - latest_penalized_balances: <_>::random_for_test(rng), - latest_attestations: <_>::random_for_test(rng), - batched_block_roots: <_>::random_for_test(rng), - latest_eth1_data: <_>::random_for_test(rng), - eth1_data_votes: <_>::random_for_test(rng), - } - } -} diff --git a/eth2/types/src/fast_beacon_state/committees_cache.rs b/eth2/types/src/fast_beacon_state/committees_cache.rs deleted file mode 100644 index 42dd101ba..000000000 --- a/eth2/types/src/fast_beacon_state/committees_cache.rs +++ /dev/null @@ -1,128 +0,0 @@ -use crate::{ - validator_registry::get_active_validator_indices, BeaconState, ChainSpec, Epoch, Hash256, -}; -use honey_badger_split::SplitExt; -use serde_derive::Serialize; -use swap_or_not_shuffle::get_permutated_index; - -pub const CACHED_EPOCHS: usize = 3; - -#[derive(Debug, PartialEq, Clone, Default, Serialize)] -pub struct CommitteesCache { - cache_index_offset: usize, - active_validator_indices_cache: Vec>>, - shuffling_cache: Vec>>>>, - previous_epoch: Epoch, - current_epoch: Epoch, - next_epoch: Epoch, -} - -impl CommitteesCache { - pub fn new(current_epoch: Epoch, spec: &ChainSpec) -> Self { - let previous_epoch = if current_epoch == spec.genesis_epoch { - current_epoch - } else { - current_epoch - 1 - }; - let next_epoch = current_epoch + 1; - - Self { - cache_index_offset: 0, - active_validator_indices_cache: vec![None; CACHED_EPOCHS], - shuffling_cache: vec![None; CACHED_EPOCHS], - previous_epoch, - current_epoch, - next_epoch, - } - } - - pub fn advance_epoch(&mut self) { - let previous_cache_index = self.cache_index(self.previous_epoch); - - self.active_validator_indices_cache[previous_cache_index] = None; - self.shuffling_cache[previous_cache_index] = None; - - self.cache_index_offset += 1; - self.cache_index_offset %= CACHED_EPOCHS; - } - - pub fn cache_index(&self, epoch: Epoch) -> usize { - let base_index = match epoch { - e if e == self.previous_epoch => 0, - e if e == self.current_epoch => 1, - e if e == self.next_epoch => 2, - _ => panic!("Bad cache index."), - }; - - (base_index + self.cache_index_offset) % CACHED_EPOCHS - } - - pub fn get_active_validator_indices( - &mut self, - state: &BeaconState, - epoch: Epoch, - ) -> &Vec { - let i = self.cache_index(epoch); - - if self.active_validator_indices_cache[i] == None { - self.active_validator_indices_cache[i] = Some(get_active_validator_indices( - &state.validator_registry, - epoch, - )); - } - - self.active_validator_indices_cache[i] - .as_ref() - .expect("Cache cannot be None") - } - - pub fn get_shuffling( - &mut self, - state: &BeaconState, - seed: Hash256, - epoch: Epoch, - spec: &ChainSpec, - ) -> Option<&Vec>> { - let cache_index = self.cache_index(epoch); - - if self.shuffling_cache[cache_index] == None { - let active_validator_indices = self.get_active_validator_indices(&state, epoch); - - if active_validator_indices.is_empty() { - return None; - } - - let committees_per_epoch = - state.get_epoch_committee_count(active_validator_indices.len(), spec); - - let mut shuffled_active_validator_indices = vec![0; active_validator_indices.len()]; - for &i in active_validator_indices { - let shuffled_i = get_permutated_index( - i, - active_validator_indices.len(), - &seed[..], - spec.shuffle_round_count, - )?; - shuffled_active_validator_indices[i] = active_validator_indices[shuffled_i] - } - - let committees: Vec> = shuffled_active_validator_indices - .honey_badger_split(committees_per_epoch as usize) - .map(|slice: &[usize]| slice.to_vec()) - .collect(); - - self.shuffling_cache[cache_index] = Some(Some(committees)); - } - - match self.shuffling_cache[cache_index] { - Some(_) => Some( - self.shuffling_cache[cache_index] - .as_ref() - .expect("Cache cannot be None") - .as_ref() - .expect("Shuffling cannot be None."), - ), - None => None, - } - } -} diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index 820c26ad6..f2c128440 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -16,7 +16,6 @@ pub mod deposit_input; pub mod eth1_data; pub mod eth1_data_vote; pub mod exit; -pub mod fast_beacon_state; pub mod fork; pub mod free_attestation; pub mod pending_attestation; @@ -53,7 +52,6 @@ pub use crate::deposit_input::DepositInput; pub use crate::eth1_data::Eth1Data; pub use crate::eth1_data_vote::Eth1DataVote; pub use crate::exit::Exit; -pub use crate::fast_beacon_state::FastBeaconState; pub use crate::fork::Fork; pub use crate::free_attestation::FreeAttestation; pub use crate::pending_attestation::PendingAttestation; From a5de6a1915b3d9b8afd24d22f0c1acb2987abf40 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 22 Feb 2019 18:14:16 +1300 Subject: [PATCH 3/9] Add caching to BeaconState. Removes CachingBeaconState --- .../src/attestation_aggregator.rs | 111 +++---- beacon_node/beacon_chain/src/beacon_chain.rs | 35 ++- .../beacon_chain/src/cached_beacon_state.rs | 150 --------- beacon_node/beacon_chain/src/checkpoint.rs | 2 +- beacon_node/beacon_chain/src/lib.rs | 5 +- .../test_harness/src/beacon_chain_harness.rs | 13 +- .../beacon_chain/test_harness/tests/chain.rs | 3 + .../state_processing/src/block_processable.rs | 29 +- .../state_processing/src/epoch_processable.rs | 65 ++-- eth2/state_processing/src/slot_processable.rs | 2 +- eth2/types/src/beacon_state.rs | 286 +++++++++++------- eth2/types/src/beacon_state/epoch_cache.rs | 77 +++++ eth2/types/src/beacon_state/tests.rs | 51 +++- eth2/types/src/lib.rs | 4 +- 14 files changed, 459 insertions(+), 374 deletions(-) delete mode 100644 beacon_node/beacon_chain/src/cached_beacon_state.rs create mode 100644 eth2/types/src/beacon_state/epoch_cache.rs diff --git a/beacon_node/beacon_chain/src/attestation_aggregator.rs b/beacon_node/beacon_chain/src/attestation_aggregator.rs index fa2ec87ab..70348dc94 100644 --- a/beacon_node/beacon_chain/src/attestation_aggregator.rs +++ b/beacon_node/beacon_chain/src/attestation_aggregator.rs @@ -1,9 +1,9 @@ -use crate::cached_beacon_state::CachedBeaconState; +use log::trace; use state_processing::validate_attestation_without_signature; use std::collections::{HashMap, HashSet}; use types::{ - beacon_state::BeaconStateError, AggregateSignature, Attestation, AttestationData, BeaconState, - Bitfield, ChainSpec, FreeAttestation, Signature, + AggregateSignature, Attestation, AttestationData, BeaconState, BeaconStateError, Bitfield, + ChainSpec, FreeAttestation, Signature, }; const PHASE_0_CUSTODY_BIT: bool = false; @@ -42,21 +42,28 @@ pub enum Message { BadSignature, /// The given `slot` does not match the validators committee assignment. BadSlot, - /// The given `shard` does not match the validators committee assignment. + /// The given `shard` does not match the validators committee assignment, or is not included in + /// a committee for the given slot. BadShard, + /// Attestation is from the epoch prior to this, ignoring. + TooOld, } -macro_rules! some_or_invalid { - ($expression: expr, $error: expr) => { - match $expression { - Some(x) => x, - None => { - return Ok(Outcome { - valid: false, - message: $error, - }); - } - } +macro_rules! valid_outcome { + ($error: expr) => { + return Ok(Outcome { + valid: true, + message: $error, + }); + }; +} + +macro_rules! invalid_outcome { + ($error: expr) => { + return Ok(Outcome { + valid: false, + message: $error, + }); }; } @@ -77,49 +84,56 @@ impl AttestationAggregator { /// - The signature is verified against that of the validator at `validator_index`. pub fn process_free_attestation( &mut self, - cached_state: &CachedBeaconState, + cached_state: &BeaconState, free_attestation: &FreeAttestation, spec: &ChainSpec, ) -> Result { - let (slot, shard, committee_index) = some_or_invalid!( - cached_state.attestation_slot_and_shard_for_validator( - free_attestation.validator_index as usize, - spec, - )?, - Message::BadValidatorIndex + let attestation_duties = match cached_state.attestation_slot_and_shard_for_validator( + free_attestation.validator_index as usize, + spec, + ) { + Err(BeaconStateError::EpochCacheUninitialized(e)) => { + panic!("Attempted to access unbuilt cache {:?}.", e) + } + Err(BeaconStateError::EpochOutOfBounds) => invalid_outcome!(Message::TooOld), + Err(BeaconStateError::ShardOutOfBounds) => invalid_outcome!(Message::BadShard), + Err(e) => return Err(e), + Ok(None) => invalid_outcome!(Message::BadValidatorIndex), + Ok(Some(attestation_duties)) => attestation_duties, + }; + + let (slot, shard, committee_index) = attestation_duties; + + trace!( + "slot: {}, shard: {}, committee_index: {}, val_index: {}", + slot, + shard, + committee_index, + free_attestation.validator_index ); if free_attestation.data.slot != slot { - return Ok(Outcome { - valid: false, - message: Message::BadSlot, - }); + invalid_outcome!(Message::BadSlot); } if free_attestation.data.shard != shard { - return Ok(Outcome { - valid: false, - message: Message::BadShard, - }); + invalid_outcome!(Message::BadShard); } let signable_message = free_attestation.data.signable_message(PHASE_0_CUSTODY_BIT); - let validator_record = some_or_invalid!( - cached_state - .state - .validator_registry - .get(free_attestation.validator_index as usize), - Message::BadValidatorIndex - ); + let validator_record = match cached_state + .validator_registry + .get(free_attestation.validator_index as usize) + { + None => invalid_outcome!(Message::BadValidatorIndex), + Some(validator_record) => validator_record, + }; if !free_attestation .signature .verify(&signable_message, &validator_record.pubkey) { - return Ok(Outcome { - valid: false, - message: Message::BadSignature, - }); + invalid_outcome!(Message::BadSignature); } if let Some(existing_attestation) = self.store.get(&signable_message) { @@ -129,15 +143,9 @@ impl AttestationAggregator { committee_index as usize, ) { self.store.insert(signable_message, updated_attestation); - Ok(Outcome { - valid: true, - message: Message::Aggregated, - }) + valid_outcome!(Message::Aggregated); } else { - Ok(Outcome { - valid: true, - message: Message::AggregationNotRequired, - }) + valid_outcome!(Message::AggregationNotRequired); } } else { let mut aggregate_signature = AggregateSignature::new(); @@ -151,10 +159,7 @@ impl AttestationAggregator { aggregate_signature, }; self.store.insert(signable_message, new_attestation); - Ok(Outcome { - valid: true, - message: Message::NewAttestationCreated, - }) + valid_outcome!(Message::NewAttestationCreated); } } diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index b2d041654..9ee55e5a3 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1,5 +1,4 @@ use crate::attestation_aggregator::{AttestationAggregator, Outcome as AggregationOutcome}; -use crate::cached_beacon_state::CachedBeaconState; use crate::checkpoint::CheckPoint; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, @@ -15,10 +14,10 @@ use state_processing::{ }; use std::sync::Arc; use types::{ - beacon_state::BeaconStateError, readers::{BeaconBlockReader, BeaconStateReader}, - AttestationData, BeaconBlock, BeaconBlockBody, BeaconState, ChainSpec, Crosslink, Deposit, - Epoch, Eth1Data, FreeAttestation, Hash256, PublicKey, Signature, Slot, + AttestationData, BeaconBlock, BeaconBlockBody, BeaconState, BeaconStateError, ChainSpec, + Crosslink, Deposit, Epoch, Eth1Data, FreeAttestation, Hash256, PublicKey, RelativeEpoch, + Signature, Slot, }; #[derive(Debug, PartialEq)] @@ -70,7 +69,6 @@ pub struct BeaconChain { canonical_head: RwLock, finalized_head: RwLock, pub state: RwLock, - pub cached_state: RwLock, pub spec: ChainSpec, pub fork_choice: RwLock, } @@ -96,7 +94,7 @@ where return Err(Error::InsufficientValidators); } - let genesis_state = BeaconState::genesis( + let mut genesis_state = BeaconState::genesis( genesis_time, initial_validator_deposits, latest_eth1_data, @@ -109,32 +107,32 @@ where let block_root = genesis_block.canonical_root(); block_store.put(&block_root, &ssz_encode(&genesis_block)[..])?; - let cached_state = RwLock::new(CachedBeaconState::from_beacon_state( - genesis_state.clone(), - spec.clone(), - )?); - let finalized_head = RwLock::new(CheckPoint::new( genesis_block.clone(), block_root, + // TODO: this is a memory waste; remove full clone. genesis_state.clone(), state_root, )); let canonical_head = RwLock::new(CheckPoint::new( genesis_block.clone(), block_root, + // TODO: this is a memory waste; remove full clone. genesis_state.clone(), state_root, )); let attestation_aggregator = RwLock::new(AttestationAggregator::new()); + genesis_state.build_epoch_cache(RelativeEpoch::Previous, &spec)?; + genesis_state.build_epoch_cache(RelativeEpoch::Current, &spec)?; + genesis_state.build_epoch_cache(RelativeEpoch::Next, &spec)?; + Ok(Self { block_store, state_store, slot_clock, attestation_aggregator, - state: RwLock::new(genesis_state.clone()), - cached_state, + state: RwLock::new(genesis_state), finalized_head, canonical_head, spec, @@ -150,6 +148,10 @@ where new_beacon_state: BeaconState, new_beacon_state_root: Hash256, ) { + debug!( + "Updating canonical head with block at slot: {}", + new_beacon_block.slot + ); let mut head = self.canonical_head.write(); head.update( new_beacon_block, @@ -288,7 +290,7 @@ where validator_index ); if let Some((slot, shard, _committee)) = self - .cached_state + .state .read() .attestation_slot_and_shard_for_validator(validator_index, &self.spec)? { @@ -346,7 +348,7 @@ where let aggregation_outcome = self .attestation_aggregator .write() - .process_free_attestation(&self.cached_state.read(), &free_attestation, &self.spec)?; + .process_free_attestation(&self.state.read(), &free_attestation, &self.spec)?; // return if the attestation is invalid if !aggregation_outcome.valid { @@ -501,9 +503,6 @@ where ); // Update the local state variable. *self.state.write() = state.clone(); - // Update the cached state variable. - *self.cached_state.write() = - CachedBeaconState::from_beacon_state(state.clone(), self.spec.clone())?; } Ok(BlockProcessingOutcome::ValidBlock(ValidBlock::Processed)) diff --git a/beacon_node/beacon_chain/src/cached_beacon_state.rs b/beacon_node/beacon_chain/src/cached_beacon_state.rs deleted file mode 100644 index e14e9fe99..000000000 --- a/beacon_node/beacon_chain/src/cached_beacon_state.rs +++ /dev/null @@ -1,150 +0,0 @@ -use log::{debug, trace}; -use std::collections::HashMap; -use types::{beacon_state::BeaconStateError, BeaconState, ChainSpec, Epoch, Slot}; - -pub const CACHE_PREVIOUS: bool = false; -pub const CACHE_CURRENT: bool = true; -pub const CACHE_NEXT: bool = false; - -pub type CrosslinkCommittees = Vec<(Vec, u64)>; -pub type Shard = u64; -pub type CommitteeIndex = u64; -pub type AttestationDuty = (Slot, Shard, CommitteeIndex); -pub type AttestationDutyMap = HashMap; - -// TODO: CachedBeaconState is presently duplicating `BeaconState` and `ChainSpec`. This is a -// massive memory waste, switch them to references. - -pub struct CachedBeaconState { - pub state: BeaconState, - committees: Vec>, - attestation_duties: Vec, - next_epoch: Epoch, - current_epoch: Epoch, - previous_epoch: Epoch, - spec: ChainSpec, -} - -impl CachedBeaconState { - pub fn from_beacon_state( - state: BeaconState, - spec: ChainSpec, - ) -> Result { - let current_epoch = state.current_epoch(&spec); - let previous_epoch = if current_epoch == spec.genesis_epoch { - current_epoch - } else { - current_epoch.saturating_sub(1_u64) - }; - let next_epoch = state.next_epoch(&spec); - - let mut committees: Vec> = Vec::with_capacity(3); - let mut attestation_duties: Vec = Vec::with_capacity(3); - - if CACHE_PREVIOUS { - debug!("from_beacon_state: building previous epoch cache."); - let cache = build_epoch_cache(&state, previous_epoch, &spec)?; - committees.push(cache.committees); - attestation_duties.push(cache.attestation_duty_map); - } else { - committees.push(vec![]); - attestation_duties.push(HashMap::new()); - } - if CACHE_CURRENT { - debug!("from_beacon_state: building current epoch cache."); - let cache = build_epoch_cache(&state, current_epoch, &spec)?; - committees.push(cache.committees); - attestation_duties.push(cache.attestation_duty_map); - } else { - committees.push(vec![]); - attestation_duties.push(HashMap::new()); - } - if CACHE_NEXT { - debug!("from_beacon_state: building next epoch cache."); - let cache = build_epoch_cache(&state, next_epoch, &spec)?; - committees.push(cache.committees); - attestation_duties.push(cache.attestation_duty_map); - } else { - committees.push(vec![]); - attestation_duties.push(HashMap::new()); - } - - Ok(Self { - state, - committees, - attestation_duties, - next_epoch, - current_epoch, - previous_epoch, - spec, - }) - } - - fn slot_to_cache_index(&self, slot: Slot) -> Option { - trace!("slot_to_cache_index: cache lookup"); - match slot.epoch(self.spec.epoch_length) { - epoch if (epoch == self.previous_epoch) & CACHE_PREVIOUS => Some(0), - epoch if (epoch == self.current_epoch) & CACHE_CURRENT => Some(1), - epoch if (epoch == self.next_epoch) & CACHE_NEXT => Some(2), - _ => None, - } - } - - /// Returns the `slot`, `shard` and `committee_index` for which a validator must produce an - /// attestation. - /// - /// Cached method. - /// - /// Spec v0.2.0 - pub fn attestation_slot_and_shard_for_validator( - &self, - validator_index: usize, - _spec: &ChainSpec, - ) -> Result, BeaconStateError> { - // Get the result for this epoch. - let cache_index = self - .slot_to_cache_index(self.state.slot) - .expect("Current epoch should always have a cache index."); - - let duties = self.attestation_duties[cache_index] - .get(&(validator_index as u64)) - .and_then(|tuple| Some(*tuple)); - - Ok(duties) - } -} - -struct EpochCacheResult { - committees: Vec, - attestation_duty_map: AttestationDutyMap, -} - -fn build_epoch_cache( - state: &BeaconState, - epoch: Epoch, - spec: &ChainSpec, -) -> Result { - let mut epoch_committees: Vec = - Vec::with_capacity(spec.epoch_length as usize); - let mut attestation_duty_map: AttestationDutyMap = HashMap::new(); - - for slot in epoch.slot_iter(spec.epoch_length) { - let slot_committees = state.get_crosslink_committees_at_slot(slot, false, spec)?; - - for (committee, shard) in slot_committees { - for (committee_index, validator_index) in committee.iter().enumerate() { - attestation_duty_map.insert( - *validator_index as u64, - (slot, shard, committee_index as u64), - ); - } - } - - epoch_committees.push(state.get_crosslink_committees_at_slot(slot, false, spec)?) - } - - Ok(EpochCacheResult { - committees: epoch_committees, - attestation_duty_map, - }) -} diff --git a/beacon_node/beacon_chain/src/checkpoint.rs b/beacon_node/beacon_chain/src/checkpoint.rs index bef97d2ed..828e462de 100644 --- a/beacon_node/beacon_chain/src/checkpoint.rs +++ b/beacon_node/beacon_chain/src/checkpoint.rs @@ -3,7 +3,7 @@ use types::{BeaconBlock, BeaconState, Hash256}; /// Represents some block and it's associated state. Generally, this will be used for tracking the /// head, justified head and finalized head. -#[derive(PartialEq, Clone, Serialize)] +#[derive(Clone, Serialize)] pub struct CheckPoint { pub beacon_block: BeaconBlock, pub beacon_block_root: Hash256, diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index bc9085fbe..d7d1d9664 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -1,8 +1,9 @@ mod attestation_aggregator; mod beacon_chain; -mod cached_beacon_state; mod checkpoint; -pub use self::beacon_chain::{BeaconChain, Error}; +pub use self::beacon_chain::{ + BeaconChain, BlockProcessingOutcome, Error, InvalidBlock, ValidBlock, +}; pub use self::checkpoint::CheckPoint; pub use fork_choice::{ForkChoice, ForkChoiceAlgorithms, ForkChoiceError}; diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index 9d61952f0..a15e82aa2 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -1,5 +1,5 @@ use super::ValidatorHarness; -use beacon_chain::BeaconChain; +use beacon_chain::{BeaconChain, BlockProcessingOutcome}; pub use beacon_chain::{CheckPoint, Error as BeaconChainError}; use bls::create_proof_of_possession; use db::{ @@ -157,7 +157,7 @@ impl BeaconChainHarness { .beacon_chain .state .read() - .get_crosslink_committees_at_slot(present_slot, false, &self.spec) + .get_crosslink_committees_at_slot(present_slot, &self.spec) .unwrap() .iter() .fold(vec![], |mut acc, (committee, _slot)| { @@ -223,7 +223,10 @@ impl BeaconChainHarness { debug!("Producing block..."); let block = self.produce_block(); debug!("Submitting block for processing..."); - self.beacon_chain.process_block(block).unwrap(); + match self.beacon_chain.process_block(block) { + Ok(BlockProcessingOutcome::ValidBlock(_)) => {} + other => panic!("block processing failed with {:?}", other), + }; debug!("...block processed by BeaconChain."); debug!("Producing free attestations..."); @@ -242,6 +245,10 @@ impl BeaconChainHarness { debug!("Free attestations processed."); } + pub fn run_fork_choice(&mut self) { + self.beacon_chain.fork_choice().unwrap() + } + /// Dump all blocks and states from the canonical beacon chain. pub fn chain_dump(&self) -> Result, BeaconChainError> { self.beacon_chain.chain_dump() diff --git a/beacon_node/beacon_chain/test_harness/tests/chain.rs b/beacon_node/beacon_chain/test_harness/tests/chain.rs index 1a08ffcf1..1b29a412f 100644 --- a/beacon_node/beacon_chain/test_harness/tests/chain.rs +++ b/beacon_node/beacon_chain/test_harness/tests/chain.rs @@ -35,6 +35,9 @@ fn it_can_produce_past_first_epoch_boundary() { harness.advance_chain_with_block(); debug!("Produced block {}/{}.", i + 1, blocks); } + + harness.run_fork_choice(); + let dump = harness.chain_dump().expect("Chain dump failed."); assert_eq!(dump.len() as u64, blocks + 1); // + 1 for genesis block. diff --git a/eth2/state_processing/src/block_processable.rs b/eth2/state_processing/src/block_processable.rs index 539711c69..0e6b57cf0 100644 --- a/eth2/state_processing/src/block_processable.rs +++ b/eth2/state_processing/src/block_processable.rs @@ -4,9 +4,8 @@ use int_to_bytes::int_to_bytes32; use log::{debug, trace}; use ssz::{ssz_encode, TreeHash}; use types::{ - beacon_state::{AttestationParticipantsError, BeaconStateError}, - AggregatePublicKey, Attestation, BeaconBlock, BeaconState, ChainSpec, Crosslink, Epoch, Exit, - Fork, Hash256, PendingAttestation, PublicKey, Signature, + AggregatePublicKey, Attestation, BeaconBlock, BeaconState, BeaconStateError, ChainSpec, + Crosslink, Epoch, Exit, Fork, Hash256, PendingAttestation, PublicKey, RelativeEpoch, Signature, }; // TODO: define elsehwere. @@ -27,7 +26,6 @@ pub enum Error { MissingBeaconBlock(Hash256), InvalidBeaconBlock(Hash256), MissingParentBlock(Hash256), - NoBlockProducer, StateSlotMismatch, BadBlockSignature, BadRandaoSignature, @@ -56,7 +54,7 @@ pub enum AttestationValidationError { BadSignature, ShardBlockRootNotZero, NoBlockRoot, - AttestationParticipantsError(AttestationParticipantsError), + BeaconStateError(BeaconStateError), } macro_rules! ensure { @@ -98,12 +96,15 @@ fn per_block_processing_signature_optional( ) -> Result<(), Error> { ensure!(block.slot == state.slot, Error::StateSlotMismatch); + // Building the previous epoch could be delayed until an attestation from a previous epoch is + // included. This is left for future optimisation. + state.build_epoch_cache(RelativeEpoch::Previous, spec)?; + state.build_epoch_cache(RelativeEpoch::Current, spec)?; + /* * Proposer Signature */ - let block_proposer_index = state - .get_beacon_proposer_index(block.slot, spec) - .map_err(|_| Error::NoBlockProducer)?; + let block_proposer_index = state.get_beacon_proposer_index(block.slot, spec)?; let block_proposer = &state.validator_registry[block_proposer_index]; if verify_block_signature { @@ -361,6 +362,12 @@ fn validate_attestation_signature_optional( &attestation.aggregation_bitfield, spec, )?; + trace!( + "slot: {}, shard: {}, participants: {:?}", + attestation.data.slot, + attestation.data.shard, + participants + ); let mut group_public_key = AggregatePublicKey::new(); for participant in participants { group_public_key.add( @@ -417,8 +424,8 @@ impl From for Error { } } -impl From for AttestationValidationError { - fn from(e: AttestationParticipantsError) -> AttestationValidationError { - AttestationValidationError::AttestationParticipantsError(e) +impl From for AttestationValidationError { + fn from(e: BeaconStateError) -> AttestationValidationError { + AttestationValidationError::BeaconStateError(e) } } diff --git a/eth2/state_processing/src/epoch_processable.rs b/eth2/state_processing/src/epoch_processable.rs index 11b2b224d..409d40a2c 100644 --- a/eth2/state_processing/src/epoch_processable.rs +++ b/eth2/state_processing/src/epoch_processable.rs @@ -5,9 +5,8 @@ use ssz::TreeHash; use std::collections::{HashMap, HashSet}; use std::iter::FromIterator; use types::{ - beacon_state::{AttestationParticipantsError, BeaconStateError, InclusionError}, - validator_registry::get_active_validator_indices, - BeaconState, ChainSpec, Crosslink, Epoch, Hash256, PendingAttestation, + validator_registry::get_active_validator_indices, BeaconState, BeaconStateError, ChainSpec, + Crosslink, Epoch, Hash256, InclusionError, PendingAttestation, RelativeEpoch, }; macro_rules! safe_add_assign { @@ -28,7 +27,6 @@ pub enum Error { BaseRewardQuotientIsZero, NoRandaoSeed, BeaconStateError(BeaconStateError), - AttestationParticipantsError(AttestationParticipantsError), InclusionError(InclusionError), WinningRootError(WinningRootError), } @@ -36,7 +34,7 @@ pub enum Error { #[derive(Debug, PartialEq)] pub enum WinningRootError { NoWinningRoot, - AttestationParticipantsError(AttestationParticipantsError), + BeaconStateError(BeaconStateError), } #[derive(Clone)] @@ -66,6 +64,11 @@ impl EpochProcessable for BeaconState { self.current_epoch(spec) ); + // Ensure all of the caches are built. + self.build_epoch_cache(RelativeEpoch::Previous, spec)?; + self.build_epoch_cache(RelativeEpoch::Current, spec)?; + self.build_epoch_cache(RelativeEpoch::Next, spec)?; + /* * Validators attesting during the current epoch. */ @@ -322,8 +325,11 @@ impl EpochProcessable for BeaconState { slot, slot.epoch(spec.epoch_length) ); + + // Clone is used to remove the borrow. It becomes an issue later when trying to mutate + // `self.balances`. let crosslink_committees_at_slot = - self.get_crosslink_committees_at_slot(slot, false, spec)?; + self.get_crosslink_committees_at_slot(slot, spec)?.clone(); for (crosslink_committee, shard) in crosslink_committees_at_slot { let shard = shard as u64; @@ -499,8 +505,10 @@ impl EpochProcessable for BeaconState { * Crosslinks */ for slot in self.previous_epoch(spec).slot_iter(spec.epoch_length) { + // Clone is used to remove the borrow. It becomes an issue later when trying to mutate + // `self.balances`. let crosslink_committees_at_slot = - self.get_crosslink_committees_at_slot(slot, false, spec)?; + self.get_crosslink_committees_at_slot(slot, spec)?.clone(); for (_crosslink_committee, shard) in crosslink_committees_at_slot { let shard = shard as u64; @@ -609,6 +617,12 @@ impl EpochProcessable for BeaconState { .cloned() .collect(); + /* + * Manage the beacon state caches + */ + self.advance_caches(); + self.build_epoch_cache(RelativeEpoch::Next, spec)?; + debug!("Epoch transition complete."); Ok(()) @@ -645,19 +659,18 @@ fn winning_root( } // TODO: `cargo fmt` makes this rather ugly; tidy up. - let attesting_validator_indices = attestations.iter().try_fold::<_, _, Result< - _, - AttestationParticipantsError, - >>(vec![], |mut acc, a| { - if (a.data.shard == shard) && (a.data.shard_block_root == *shard_block_root) { - acc.append(&mut state.get_attestation_participants( - &a.data, - &a.aggregation_bitfield, - spec, - )?); - } - Ok(acc) - })?; + let attesting_validator_indices = attestations + .iter() + .try_fold::<_, _, Result<_, BeaconStateError>>(vec![], |mut acc, a| { + if (a.data.shard == shard) && (a.data.shard_block_root == *shard_block_root) { + acc.append(&mut state.get_attestation_participants( + &a.data, + &a.aggregation_bitfield, + spec, + )?); + } + Ok(acc) + })?; let total_balance: u64 = attesting_validator_indices .iter() @@ -708,15 +721,9 @@ impl From for Error { } } -impl From for Error { - fn from(e: AttestationParticipantsError) -> Error { - Error::AttestationParticipantsError(e) - } -} - -impl From for WinningRootError { - fn from(e: AttestationParticipantsError) -> WinningRootError { - WinningRootError::AttestationParticipantsError(e) +impl From for WinningRootError { + fn from(e: BeaconStateError) -> WinningRootError { + WinningRootError::BeaconStateError(e) } } diff --git a/eth2/state_processing/src/slot_processable.rs b/eth2/state_processing/src/slot_processable.rs index 9e3b611fd..0bbc79ab0 100644 --- a/eth2/state_processing/src/slot_processable.rs +++ b/eth2/state_processing/src/slot_processable.rs @@ -1,5 +1,5 @@ use crate::{EpochProcessable, EpochProcessingError}; -use types::{beacon_state::BeaconStateError, BeaconState, ChainSpec, Hash256}; +use types::{BeaconState, BeaconStateError, ChainSpec, Hash256}; #[derive(Debug, PartialEq)] pub enum Error { diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 2216e9516..47805da80 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -1,3 +1,4 @@ +use self::epoch_cache::EpochCache; use crate::test_utils::TestRandom; use crate::{ validator::StatusFlags, validator_registry::get_active_validator_indices, AttestationData, @@ -10,13 +11,35 @@ use log::trace; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use std::collections::HashMap; use swap_or_not_shuffle::get_permutated_index; +mod epoch_cache; mod tests; +pub type Committee = Vec; +pub type CrosslinkCommittees = Vec<(Committee, u64)>; +pub type Shard = u64; +pub type CommitteeIndex = u64; +pub type AttestationDuty = (Slot, Shard, CommitteeIndex); +pub type AttestationDutyMap = HashMap; +pub type ShardCommitteeIndexMap = HashMap; + +pub const CACHED_EPOCHS: usize = 3; + +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum RelativeEpoch { + Previous, + Current, + Next, +} + #[derive(Debug, PartialEq)] -pub enum BeaconStateError { +pub enum Error { EpochOutOfBounds, + /// The supplied shard is unknown. It may be larger than the maximum shard count, or not in a + /// committee for the given slot. + ShardOutOfBounds, UnableToShuffle, InsufficientRandaoMixes, InsufficientValidators, @@ -24,20 +47,14 @@ pub enum BeaconStateError { InsufficientIndexRoots, InsufficientAttestations, InsufficientCommittees, + EpochCacheUninitialized(RelativeEpoch), } #[derive(Debug, PartialEq)] pub enum InclusionError { /// The validator did not participate in an attestation in this period. NoAttestationsForValidator, - AttestationParticipantsError(AttestationParticipantsError), -} - -#[derive(Debug, PartialEq)] -pub enum AttestationParticipantsError { - /// There is no committee for the given shard in the given epoch. - NoCommitteeForShard, - BeaconStateError(BeaconStateError), + Error(Error), } macro_rules! safe_add_assign { @@ -89,6 +106,10 @@ pub struct BeaconState { // Ethereum 1.0 chain data pub latest_eth1_data: Eth1Data, pub eth1_data_votes: Vec, + + // Caching + pub cache_index_offset: usize, + pub caches: Vec, } impl BeaconState { @@ -98,7 +119,7 @@ impl BeaconState { initial_validator_deposits: Vec, latest_eth1_data: Eth1Data, spec: &ChainSpec, - ) -> Result { + ) -> Result { let initial_crosslink = Crosslink { epoch: spec.genesis_epoch, shard_block_root: spec.zero_hash, @@ -157,6 +178,12 @@ impl BeaconState { */ latest_eth1_data, eth1_data_votes: vec![], + + /* + * Caching (not in spec) + */ + cache_index_offset: 0, + caches: vec![EpochCache::empty(); CACHED_EPOCHS], }; for deposit in initial_validator_deposits { @@ -187,6 +214,81 @@ impl BeaconState { Ok(genesis_state) } + /// Build an epoch cache, unless it is has already been built. + pub fn build_epoch_cache( + &mut self, + relative_epoch: RelativeEpoch, + spec: &ChainSpec, + ) -> Result<(), Error> { + let cache_index = self.cache_index(relative_epoch); + + if self.caches[cache_index].initialized == false { + self.force_build_epoch_cache(relative_epoch, spec) + } else { + Ok(()) + } + } + + /// Always builds an epoch cache, even if it is alread initialized. + pub fn force_build_epoch_cache( + &mut self, + relative_epoch: RelativeEpoch, + spec: &ChainSpec, + ) -> Result<(), Error> { + let epoch = self.absolute_epoch(relative_epoch, spec); + let cache_index = self.cache_index(relative_epoch); + + self.caches[cache_index] = EpochCache::initialized(&self, epoch, spec)?; + + Ok(()) + } + + fn absolute_epoch(&self, relative_epoch: RelativeEpoch, spec: &ChainSpec) -> Epoch { + match relative_epoch { + RelativeEpoch::Previous => self.previous_epoch(spec), + RelativeEpoch::Current => self.current_epoch(spec), + RelativeEpoch::Next => self.next_epoch(spec), + } + } + + fn relative_epoch(&self, epoch: Epoch, spec: &ChainSpec) -> Result { + match epoch { + e if e == self.current_epoch(spec) => Ok(RelativeEpoch::Current), + e if e == self.previous_epoch(spec) => Ok(RelativeEpoch::Previous), + e if e == self.next_epoch(spec) => Ok(RelativeEpoch::Next), + _ => Err(Error::EpochOutOfBounds), + } + } + + pub fn advance_caches(&mut self) { + let previous_cache_index = self.cache_index(RelativeEpoch::Previous); + + self.caches[previous_cache_index] = EpochCache::empty(); + + self.cache_index_offset += 1; + self.cache_index_offset %= CACHED_EPOCHS; + } + + fn cache_index(&self, relative_epoch: RelativeEpoch) -> usize { + let base_index = match relative_epoch { + RelativeEpoch::Current => 1, + RelativeEpoch::Previous => 0, + RelativeEpoch::Next => 2, + }; + + (base_index + self.cache_index_offset) % CACHED_EPOCHS + } + + fn cache<'a>(&'a self, relative_epoch: RelativeEpoch) -> Result<&'a EpochCache, Error> { + let cache = &self.caches[self.cache_index(relative_epoch)]; + + if cache.initialized == false { + Err(Error::EpochCacheUninitialized(relative_epoch)) + } else { + Ok(cache) + } + } + /// Return the tree hash root for this `BeaconState`. /// /// Spec v0.2.0 @@ -258,7 +360,7 @@ impl BeaconState { /// committee is itself a list of validator indices. /// /// Spec v0.1 - pub fn get_shuffling( + pub(crate) fn get_shuffling( &self, seed: Hash256, epoch: Epoch, @@ -271,11 +373,6 @@ impl BeaconState { return None; } - trace!( - "get_shuffling: active_validator_indices.len() == {}", - active_validator_indices.len() - ); - let committees_per_epoch = self.get_epoch_committee_count(active_validator_indices.len(), spec); @@ -339,17 +436,9 @@ impl BeaconState { + 1; let latest_index_root = current_epoch + spec.entry_exit_delay; - trace!( - "get_active_index_root: epoch: {}, earliest: {}, latest: {}", - epoch, - earliest_index_root, - latest_index_root - ); - if (epoch >= earliest_index_root) & (epoch <= latest_index_root) { Some(self.latest_index_roots[epoch.as_usize() % spec.latest_index_roots_length]) } else { - trace!("get_active_index_root: epoch out of range."); None } } @@ -357,20 +446,16 @@ impl BeaconState { /// Generate a seed for the given ``epoch``. /// /// Spec v0.2.0 - pub fn generate_seed( - &self, - epoch: Epoch, - spec: &ChainSpec, - ) -> Result { + pub fn generate_seed(&self, epoch: Epoch, spec: &ChainSpec) -> Result { let mut input = self .get_randao_mix(epoch, spec) - .ok_or_else(|| BeaconStateError::InsufficientRandaoMixes)? + .ok_or_else(|| Error::InsufficientRandaoMixes)? .to_vec(); input.append( &mut self .get_active_index_root(epoch, spec) - .ok_or_else(|| BeaconStateError::InsufficientIndexRoots)? + .ok_or_else(|| Error::InsufficientIndexRoots)? .to_vec(), ); @@ -380,18 +465,34 @@ impl BeaconState { Ok(Hash256::from(&hash(&input[..])[..])) } + pub fn get_crosslink_committees_at_slot( + &self, + slot: Slot, + spec: &ChainSpec, + ) -> Result<&CrosslinkCommittees, Error> { + let epoch = slot.epoch(spec.epoch_length); + let relative_epoch = self.relative_epoch(epoch, spec)?; + let cache = self.cache(relative_epoch)?; + + let slot_offset = slot - epoch.start_slot(spec.epoch_length); + + Ok(&cache.committees[slot_offset.as_usize()]) + } + /// Return the list of ``(committee, shard)`` tuples for the ``slot``. /// /// Note: There are two possible shufflings for crosslink committees for a /// `slot` in the next epoch: with and without a `registry_change` /// + /// Note: this is equivalent to the `get_crosslink_committees_at_slot` function in the spec. + /// /// Spec v0.2.0 - pub fn get_crosslink_committees_at_slot( + pub(crate) fn calculate_crosslink_committees_at_slot( &self, slot: Slot, registry_change: bool, spec: &ChainSpec, - ) -> Result, u64)>, BeaconStateError> { + ) -> Result, u64)>, Error> { let epoch = slot.epoch(spec.epoch_length); let current_epoch = self.current_epoch(spec); let previous_epoch = self.previous_epoch(spec); @@ -441,24 +542,17 @@ impl BeaconState { shuffling_start_shard, ) } else { - return Err(BeaconStateError::EpochOutOfBounds); + return Err(Error::EpochOutOfBounds); }; let shuffling = self .get_shuffling(seed, shuffling_epoch, spec) - .ok_or_else(|| BeaconStateError::UnableToShuffle)?; + .ok_or_else(|| Error::UnableToShuffle)?; let offset = slot.as_u64() % spec.epoch_length; let committees_per_slot = committees_per_epoch / spec.epoch_length; let slot_start_shard = (shuffling_start_shard + committees_per_slot * offset) % spec.shard_count; - trace!( - "get_crosslink_committees_at_slot: committees_per_slot: {}, slot_start_shard: {}, seed: {}", - committees_per_slot, - slot_start_shard, - seed - ); - let mut crosslinks_at_slot = vec![]; for i in 0..committees_per_slot { let tuple = ( @@ -473,22 +567,20 @@ impl BeaconState { /// Returns the `slot`, `shard` and `committee_index` for which a validator must produce an /// attestation. /// + /// Only reads the current epoch. + /// /// Spec v0.2.0 pub fn attestation_slot_and_shard_for_validator( &self, validator_index: usize, - spec: &ChainSpec, - ) -> Result, BeaconStateError> { - let mut result = None; - for slot in self.current_epoch(spec).slot_iter(spec.epoch_length) { - for (committee, shard) in self.get_crosslink_committees_at_slot(slot, false, spec)? { - if let Some(committee_index) = committee.iter().position(|&i| i == validator_index) - { - result = Some((slot, shard, committee_index as u64)); - } - } - } - Ok(result) + _spec: &ChainSpec, + ) -> Result, Error> { + let cache = self.cache(RelativeEpoch::Current)?; + + Ok(cache + .attestation_duty_map + .get(&(validator_index as u64)) + .and_then(|tuple| Some(*tuple))) } /// An entry or exit triggered in the ``epoch`` given by the input takes effect at @@ -504,12 +596,8 @@ impl BeaconState { /// If the state does not contain an index for a beacon proposer at the requested `slot`, then `None` is returned. /// /// Spec v0.2.0 - pub fn get_beacon_proposer_index( - &self, - slot: Slot, - spec: &ChainSpec, - ) -> Result { - let committees = self.get_crosslink_committees_at_slot(slot, false, spec)?; + pub fn get_beacon_proposer_index(&self, slot: Slot, spec: &ChainSpec) -> Result { + let committees = self.get_crosslink_committees_at_slot(slot, spec)?; trace!( "get_beacon_proposer_index: slot: {}, committees_count: {}", slot, @@ -517,11 +605,12 @@ impl BeaconState { ); committees .first() - .ok_or(BeaconStateError::InsufficientValidators) + .ok_or(Error::InsufficientValidators) .and_then(|(first_committee, _)| { - let index = (slot.as_usize()) + let index = slot + .as_usize() .checked_rem(first_committee.len()) - .ok_or(BeaconStateError::InsufficientValidators)?; + .ok_or(Error::InsufficientValidators)?; Ok(first_committee[index]) }) } @@ -730,7 +819,7 @@ impl BeaconState { &mut self, validator_index: usize, spec: &ChainSpec, - ) -> Result<(), BeaconStateError> { + ) -> Result<(), Error> { self.exit_validator(validator_index, spec); let current_epoch = self.current_epoch(spec); @@ -899,54 +988,47 @@ impl BeaconState { &self, attestations: &[&PendingAttestation], spec: &ChainSpec, - ) -> Result, AttestationParticipantsError> { - let mut all_participants = attestations.iter().try_fold::<_, _, Result< - Vec, - AttestationParticipantsError, - >>(vec![], |mut acc, a| { - acc.append(&mut self.get_attestation_participants( - &a.data, - &a.aggregation_bitfield, - spec, - )?); - Ok(acc) - })?; + ) -> Result, Error> { + let mut all_participants = attestations + .iter() + .try_fold::<_, _, Result, Error>>(vec![], |mut acc, a| { + acc.append(&mut self.get_attestation_participants( + &a.data, + &a.aggregation_bitfield, + spec, + )?); + Ok(acc) + })?; all_participants.sort_unstable(); all_participants.dedup(); Ok(all_participants) } - /// Return the participant indices at for the ``attestation_data`` and ``bitfield``. - /// - /// In effect, this converts the "committee indices" on the bitfield into "validator indices" - /// for self.validator_registy. - /// - /// Spec v0.2.0 pub fn get_attestation_participants( &self, attestation_data: &AttestationData, bitfield: &Bitfield, spec: &ChainSpec, - ) -> Result, AttestationParticipantsError> { - let crosslink_committees = - self.get_crosslink_committees_at_slot(attestation_data.slot, false, spec)?; + ) -> Result, Error> { + let epoch = attestation_data.slot.epoch(spec.epoch_length); + let relative_epoch = self.relative_epoch(epoch, spec)?; + let cache = self.cache(relative_epoch)?; - let committee_index: usize = crosslink_committees - .iter() - .position(|(_committee, shard)| *shard == attestation_data.shard) - .ok_or_else(|| AttestationParticipantsError::NoCommitteeForShard)?; - let (crosslink_committee, _shard) = &crosslink_committees[committee_index]; + let (committee_slot_index, committee_index) = cache + .shard_committee_index_map + .get(&attestation_data.shard) + .ok_or_else(|| Error::ShardOutOfBounds)?; + let (committee, shard) = &cache.committees[*committee_slot_index][*committee_index]; - /* - * TODO: verify bitfield length is valid. - */ + assert_eq!(*shard, attestation_data.shard, "Bad epoch cache build."); let mut participants = vec![]; - for (i, validator_index) in crosslink_committee.iter().enumerate() { + for (i, validator_index) in committee.iter().enumerate() { if bitfield.get(i).unwrap() { participants.push(*validator_index); } } + Ok(participants) } } @@ -955,15 +1037,9 @@ fn hash_tree_root(input: Vec) -> Hash256 { Hash256::from(&input.hash_tree_root()[..]) } -impl From for AttestationParticipantsError { - fn from(e: BeaconStateError) -> AttestationParticipantsError { - AttestationParticipantsError::BeaconStateError(e) - } -} - -impl From for InclusionError { - fn from(e: AttestationParticipantsError) -> InclusionError { - InclusionError::AttestationParticipantsError(e) +impl From for InclusionError { + fn from(e: Error) -> InclusionError { + InclusionError::Error(e) } } @@ -1052,6 +1128,8 @@ impl Decodable for BeaconState { batched_block_roots, latest_eth1_data, eth1_data_votes, + cache_index_offset: 0, + caches: vec![EpochCache::empty(); CACHED_EPOCHS], }, i, )) @@ -1122,6 +1200,8 @@ impl TestRandom for BeaconState { batched_block_roots: <_>::random_for_test(rng), latest_eth1_data: <_>::random_for_test(rng), eth1_data_votes: <_>::random_for_test(rng), + cache_index_offset: 0, + caches: vec![EpochCache::empty(); CACHED_EPOCHS], } } } diff --git a/eth2/types/src/beacon_state/epoch_cache.rs b/eth2/types/src/beacon_state/epoch_cache.rs new file mode 100644 index 000000000..0653aaa3d --- /dev/null +++ b/eth2/types/src/beacon_state/epoch_cache.rs @@ -0,0 +1,77 @@ +use super::{AttestationDutyMap, BeaconState, CrosslinkCommittees, Error, ShardCommitteeIndexMap}; +use crate::{ChainSpec, Epoch}; +use log::trace; +use serde_derive::Serialize; +use std::collections::HashMap; + +#[derive(Debug, PartialEq, Clone, Serialize)] +pub struct EpochCache { + /// True if this cache has been initialized. + pub initialized: bool, + /// The crosslink committees for an epoch. + pub committees: Vec, + /// Maps validator index to a slot, shard and committee index for attestation. + pub attestation_duty_map: AttestationDutyMap, + /// Maps a shard to an index of `self.committees`. + pub shard_committee_index_map: ShardCommitteeIndexMap, +} + +impl EpochCache { + pub fn empty() -> EpochCache { + EpochCache { + initialized: false, + committees: vec![], + attestation_duty_map: AttestationDutyMap::new(), + shard_committee_index_map: ShardCommitteeIndexMap::new(), + } + } + + pub fn initialized( + state: &BeaconState, + epoch: Epoch, + spec: &ChainSpec, + ) -> Result { + let mut epoch_committees: Vec = + Vec::with_capacity(spec.epoch_length as usize); + let mut attestation_duty_map: AttestationDutyMap = HashMap::new(); + let mut shard_committee_index_map: ShardCommitteeIndexMap = HashMap::new(); + + for (epoch_committeess_index, slot) in epoch.slot_iter(spec.epoch_length).enumerate() { + let slot_committees = + state.calculate_crosslink_committees_at_slot(slot, false, spec)?; + + for (slot_committees_index, (committee, shard)) in slot_committees.iter().enumerate() { + // Empty committees are not permitted. + if committee.is_empty() { + return Err(Error::InsufficientValidators); + } + + trace!( + "shard: {}, epoch_i: {}, slot_i: {}", + shard, + epoch_committeess_index, + slot_committees_index + ); + + shard_committee_index_map + .insert(*shard, (epoch_committeess_index, slot_committees_index)); + + for (committee_index, validator_index) in committee.iter().enumerate() { + attestation_duty_map.insert( + *validator_index as u64, + (slot, *shard, committee_index as u64), + ); + } + } + + epoch_committees.push(slot_committees) + } + + Ok(EpochCache { + initialized: true, + committees: epoch_committees, + attestation_duty_map, + shard_committee_index_map, + }) + } +} diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs index 2b7c5b539..d503d8c6a 100644 --- a/eth2/types/src/beacon_state/tests.rs +++ b/eth2/types/src/beacon_state/tests.rs @@ -3,8 +3,8 @@ use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; use crate::{ - beacon_state::BeaconStateError, BeaconState, ChainSpec, Deposit, DepositData, DepositInput, - Eth1Data, Hash256, Keypair, + BeaconState, BeaconStateError, ChainSpec, Deposit, DepositData, DepositInput, Eth1Data, + Hash256, Keypair, }; use bls::create_proof_of_possession; use ssz::ssz_encode; @@ -73,6 +73,53 @@ pub fn can_produce_genesis_block() { builder.build().unwrap(); } +/// Tests that `get_attestation_participants` is consistent with the result of +/// get_crosslink_committees_at_slot` with a full bitfield. +#[test] +pub fn get_attestation_participants_consistency() { + let mut rng = XorShiftRng::from_seed([42; 16]); + + let mut builder = BeaconStateTestBuilder::with_random_validators(8); + builder.spec = ChainSpec::few_validators(); + + let mut state = builder.build().unwrap(); + let spec = builder.spec.clone(); + + state + .build_epoch_cache(RelativeEpoch::Previous, &spec) + .unwrap(); + state + .build_epoch_cache(RelativeEpoch::Current, &spec) + .unwrap(); + state.build_epoch_cache(RelativeEpoch::Next, &spec).unwrap(); + + for slot in state + .slot + .epoch(spec.epoch_length) + .slot_iter(spec.epoch_length) + { + let committees = state.get_crosslink_committees_at_slot(slot, &spec).unwrap(); + + for (committee, shard) in committees { + let mut attestation_data = AttestationData::random_for_test(&mut rng); + attestation_data.slot = slot; + attestation_data.shard = *shard; + + let mut bitfield = Bitfield::new(); + for (i, _) in committee.iter().enumerate() { + bitfield.set(i, true); + } + + assert_eq!( + state + .get_attestation_participants(&attestation_data, &bitfield, &spec) + .unwrap(), + *committee + ); + } + } +} + #[test] pub fn test_ssz_round_trip() { let mut rng = XorShiftRng::from_seed([42; 16]); diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index f2c128440..4f196b9e9 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -42,7 +42,9 @@ pub use crate::attestation_data_and_custody_bit::AttestationDataAndCustodyBit; pub use crate::attester_slashing::AttesterSlashing; pub use crate::beacon_block::BeaconBlock; pub use crate::beacon_block_body::BeaconBlockBody; -pub use crate::beacon_state::BeaconState; +pub use crate::beacon_state::{ + BeaconState, Error as BeaconStateError, InclusionError, RelativeEpoch, +}; pub use crate::casper_slashing::CasperSlashing; pub use crate::chain_spec::ChainSpec; pub use crate::crosslink::Crosslink; From 89ab0f683e08f54cea033c78bbb3b6ebe7a2424d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 22 Feb 2019 18:19:47 +1300 Subject: [PATCH 4/9] Change "few_validators" spec to be 8 shards. A bug arises when the number of shards is less than the slots in an epoch. --- eth2/types/src/chain_spec.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs index b5d5689e3..706ad417a 100644 --- a/eth2/types/src/chain_spec.rs +++ b/eth2/types/src/chain_spec.rs @@ -199,7 +199,7 @@ impl ChainSpec { let genesis_epoch = genesis_slot.epoch(epoch_length); Self { - shard_count: 1, + shard_count: 8, target_committee_size: 1, genesis_slot, genesis_epoch, From 7a28893bab2fbe13f6145497a9633406837f98bf Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 22 Feb 2019 18:48:35 +1300 Subject: [PATCH 5/9] Fix bug in Epoch.slot_iter() It wasn't running the whole range, plus it could get into a loop when used near the u64::max_value --- eth2/types/src/slot_epoch.rs | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/eth2/types/src/slot_epoch.rs b/eth2/types/src/slot_epoch.rs index eb5a8dced..ff4fd5b9b 100644 --- a/eth2/types/src/slot_epoch.rs +++ b/eth2/types/src/slot_epoch.rs @@ -72,7 +72,7 @@ impl Epoch { pub fn slot_iter(&self, epoch_length: u64) -> SlotIter { SlotIter { - current: self.start_slot(epoch_length), + current_iteration: 0, epoch: self, epoch_length, } @@ -80,7 +80,7 @@ impl Epoch { } pub struct SlotIter<'a> { - current: Slot, + current_iteration: u64, epoch: &'a Epoch, epoch_length: u64, } @@ -89,12 +89,13 @@ impl<'a> Iterator for SlotIter<'a> { type Item = Slot; fn next(&mut self) -> Option { - if self.current == self.epoch.end_slot(self.epoch_length) { + if self.current_iteration >= self.epoch_length { None } else { - let previous = self.current; - self.current += 1; - Some(previous) + let start_slot = self.epoch.start_slot(self.epoch_length); + let previous = self.current_iteration; + self.current_iteration += 1; + Some(start_slot + previous) } } } @@ -115,4 +116,22 @@ mod epoch_tests { use ssz::ssz_encode; all_tests!(Epoch); + + #[test] + fn slot_iter() { + let epoch_length = 8; + + let epoch = Epoch::new(0); + + let mut slots = vec![]; + for slot in epoch.slot_iter(epoch_length) { + slots.push(slot); + } + + assert_eq!(slots.len(), epoch_length as usize); + + for i in 0..epoch_length { + assert_eq!(Slot::from(i), slots[i as usize]) + } + } } From 779b6266a5da859cea327db2407d5880775a85e0 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 23 Feb 2019 18:45:32 +1300 Subject: [PATCH 6/9] Ensure shuffling is cached between slot calcs Previously it was being re-built for every slot, now it is being generated once-per-epoch. --- eth2/types/src/beacon_state.rs | 196 +++++++++++++-------- eth2/types/src/beacon_state/epoch_cache.rs | 11 +- 2 files changed, 134 insertions(+), 73 deletions(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 47805da80..03d2fd52f 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -2,12 +2,12 @@ use self::epoch_cache::EpochCache; use crate::test_utils::TestRandom; use crate::{ validator::StatusFlags, validator_registry::get_active_validator_indices, AttestationData, - Bitfield, ChainSpec, Crosslink, Deposit, Epoch, Eth1Data, Eth1DataVote, Fork, Hash256, - PendingAttestation, PublicKey, Signature, Slot, Validator, + Bitfield, ChainSpec, Crosslink, Deposit, DepositData, Epoch, Eth1Data, Eth1DataVote, Fork, + Hash256, PendingAttestation, PublicKey, Signature, Slot, Validator, }; use bls::verify_proof_of_possession; use honey_badger_split::SplitExt; -use log::trace; +use log::{debug, trace}; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; @@ -120,6 +120,7 @@ impl BeaconState { latest_eth1_data: Eth1Data, spec: &ChainSpec, ) -> Result { + debug!("Creating genesis state."); let initial_crosslink = Crosslink { epoch: spec.genesis_epoch, shard_block_root: spec.zero_hash, @@ -186,15 +187,14 @@ impl BeaconState { caches: vec![EpochCache::empty(); CACHED_EPOCHS], }; - for deposit in initial_validator_deposits { - let _index = genesis_state.process_deposit( - deposit.deposit_data.deposit_input.pubkey, - deposit.deposit_data.amount, - deposit.deposit_data.deposit_input.proof_of_possession, - deposit.deposit_data.deposit_input.withdrawal_credentials, - spec, - ); - } + let deposit_data = initial_validator_deposits + .iter() + .map(|deposit| &deposit.deposit_data) + .collect(); + + genesis_state.process_deposits_optimized(deposit_data, spec); + + trace!("Processed genesis deposits."); for validator_index in 0..genesis_state.validator_registry.len() { if genesis_state.get_effective_balance(validator_index, spec) >= spec.max_deposit_amount @@ -479,6 +479,77 @@ impl BeaconState { Ok(&cache.committees[slot_offset.as_usize()]) } + pub(crate) fn get_shuffling_for_slot( + &self, + slot: Slot, + registry_change: bool, + spec: &ChainSpec, + ) -> Result>, Error> { + let (_committees_per_epoch, seed, shuffling_epoch, _shuffling_start_shard) = + self.get_committee_params_at_slot(slot, registry_change, spec)?; + + self.get_shuffling(seed, shuffling_epoch, spec) + .ok_or_else(|| Error::UnableToShuffle) + } + + pub(crate) fn get_committee_params_at_slot( + &self, + slot: Slot, + registry_change: bool, + spec: &ChainSpec, + ) -> Result<(u64, Hash256, Epoch, u64), Error> { + let epoch = slot.epoch(spec.epoch_length); + let current_epoch = self.current_epoch(spec); + let previous_epoch = self.previous_epoch(spec); + let next_epoch = self.next_epoch(spec); + + if epoch == current_epoch { + trace!("get_crosslink_committees_at_slot: current_epoch"); + Ok(( + self.get_current_epoch_committee_count(spec), + self.current_epoch_seed, + self.current_calculation_epoch, + self.current_epoch_start_shard, + )) + } else if epoch == previous_epoch { + trace!("get_crosslink_committees_at_slot: previous_epoch"); + Ok(( + self.get_previous_epoch_committee_count(spec), + self.previous_epoch_seed, + self.previous_calculation_epoch, + self.previous_epoch_start_shard, + )) + } else if epoch == next_epoch { + trace!("get_crosslink_committees_at_slot: next_epoch"); + let current_committees_per_epoch = self.get_current_epoch_committee_count(spec); + let epochs_since_last_registry_update = + current_epoch - self.validator_registry_update_epoch; + let (seed, shuffling_start_shard) = if registry_change { + let next_seed = self.generate_seed(next_epoch, spec)?; + ( + next_seed, + (self.current_epoch_start_shard + current_committees_per_epoch) + % spec.shard_count, + ) + } else if (epochs_since_last_registry_update > 1) + & epochs_since_last_registry_update.is_power_of_two() + { + let next_seed = self.generate_seed(next_epoch, spec)?; + (next_seed, self.current_epoch_start_shard) + } else { + (self.current_epoch_seed, self.current_epoch_start_shard) + }; + Ok(( + self.get_next_epoch_committee_count(spec), + seed, + next_epoch, + shuffling_start_shard, + )) + } else { + Err(Error::EpochOutOfBounds) + } + } + /// Return the list of ``(committee, shard)`` tuples for the ``slot``. /// /// Note: There are two possible shufflings for crosslink committees for a @@ -491,63 +562,12 @@ impl BeaconState { &self, slot: Slot, registry_change: bool, + shuffling: Vec>, spec: &ChainSpec, ) -> Result, u64)>, Error> { - let epoch = slot.epoch(spec.epoch_length); - let current_epoch = self.current_epoch(spec); - let previous_epoch = self.previous_epoch(spec); - let next_epoch = self.next_epoch(spec); + let (committees_per_epoch, _seed, _shuffling_epoch, shuffling_start_shard) = + self.get_committee_params_at_slot(slot, registry_change, spec)?; - let (committees_per_epoch, seed, shuffling_epoch, shuffling_start_shard) = - if epoch == current_epoch { - trace!("get_crosslink_committees_at_slot: current_epoch"); - ( - self.get_current_epoch_committee_count(spec), - self.current_epoch_seed, - self.current_calculation_epoch, - self.current_epoch_start_shard, - ) - } else if epoch == previous_epoch { - trace!("get_crosslink_committees_at_slot: previous_epoch"); - ( - self.get_previous_epoch_committee_count(spec), - self.previous_epoch_seed, - self.previous_calculation_epoch, - self.previous_epoch_start_shard, - ) - } else if epoch == next_epoch { - trace!("get_crosslink_committees_at_slot: next_epoch"); - let current_committees_per_epoch = self.get_current_epoch_committee_count(spec); - let epochs_since_last_registry_update = - current_epoch - self.validator_registry_update_epoch; - let (seed, shuffling_start_shard) = if registry_change { - let next_seed = self.generate_seed(next_epoch, spec)?; - ( - next_seed, - (self.current_epoch_start_shard + current_committees_per_epoch) - % spec.shard_count, - ) - } else if (epochs_since_last_registry_update > 1) - & epochs_since_last_registry_update.is_power_of_two() - { - let next_seed = self.generate_seed(next_epoch, spec)?; - (next_seed, self.current_epoch_start_shard) - } else { - (self.current_epoch_seed, self.current_epoch_start_shard) - }; - ( - self.get_next_epoch_committee_count(spec), - seed, - next_epoch, - shuffling_start_shard, - ) - } else { - return Err(Error::EpochOutOfBounds); - }; - - let shuffling = self - .get_shuffling(seed, shuffling_epoch, spec) - .ok_or_else(|| Error::UnableToShuffle)?; let offset = slot.as_u64() % spec.epoch_length; let committees_per_slot = committees_per_epoch / spec.epoch_length; let slot_start_shard = @@ -724,6 +744,35 @@ impl BeaconState { self.validator_registry_update_epoch = current_epoch; } + + pub fn process_deposits_optimized( + &mut self, + deposits: Vec<&DepositData>, + spec: &ChainSpec, + ) -> Vec { + let mut added_indices = vec![]; + let mut pubkey_map: HashMap = HashMap::new(); + + for (i, validator) in self.validator_registry.iter().enumerate() { + pubkey_map.insert(validator.pubkey.clone(), i); + } + + for deposit_data in deposits { + let result = self.process_deposit( + deposit_data.deposit_input.pubkey.clone(), + deposit_data.amount, + deposit_data.deposit_input.proof_of_possession.clone(), + deposit_data.deposit_input.withdrawal_credentials, + Some(&pubkey_map), + spec, + ); + if let Ok(index) = result { + added_indices.push(index); + } + } + added_indices + } + /// Process a validator deposit, returning the validator index if the deposit is valid. /// /// Spec v0.2.0 @@ -733,6 +782,7 @@ impl BeaconState { amount: u64, proof_of_possession: Signature, withdrawal_credentials: Hash256, + pubkey_map: Option<&HashMap>, spec: &ChainSpec, ) -> Result { // TODO: ensure verify proof-of-possession represents the spec accurately. @@ -740,11 +790,15 @@ impl BeaconState { return Err(()); } - if let Some(index) = self - .validator_registry - .iter() - .position(|v| v.pubkey == pubkey) - { + let validator_index = if let Some(pubkey_map) = pubkey_map { + pubkey_map.get(&pubkey).and_then(|i| Some(*i)) + } else { + self.validator_registry + .iter() + .position(|v| v.pubkey == pubkey) + }; + + if let Some(index) = validator_index { if self.validator_registry[index].withdrawal_credentials == withdrawal_credentials { safe_add_assign!(self.validator_balances[index], amount); Ok(index) diff --git a/eth2/types/src/beacon_state/epoch_cache.rs b/eth2/types/src/beacon_state/epoch_cache.rs index 0653aaa3d..ee3a67813 100644 --- a/eth2/types/src/beacon_state/epoch_cache.rs +++ b/eth2/types/src/beacon_state/epoch_cache.rs @@ -36,9 +36,16 @@ impl EpochCache { let mut attestation_duty_map: AttestationDutyMap = HashMap::new(); let mut shard_committee_index_map: ShardCommitteeIndexMap = HashMap::new(); + let shuffling = + state.get_shuffling_for_slot(epoch.start_slot(spec.epoch_length), false, spec)?; + for (epoch_committeess_index, slot) in epoch.slot_iter(spec.epoch_length).enumerate() { - let slot_committees = - state.calculate_crosslink_committees_at_slot(slot, false, spec)?; + let slot_committees = state.calculate_crosslink_committees_at_slot( + slot, + false, + shuffling.clone(), + spec, + )?; for (slot_committees_index, (committee, shard)) in slot_committees.iter().enumerate() { // Empty committees are not permitted. From c49f425fe81f80ebecd0e061a97c33313c99f17d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 24 Feb 2019 18:25:17 +1300 Subject: [PATCH 7/9] Tidy, add comments to `BeaconState` --- eth2/types/src/beacon_state.rs | 73 +++++++++++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 5 deletions(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 03d2fd52f..88f5422f8 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -192,7 +192,7 @@ impl BeaconState { .map(|deposit| &deposit.deposit_data) .collect(); - genesis_state.process_deposits_optimized(deposit_data, spec); + genesis_state.process_deposits(deposit_data, spec); trace!("Processed genesis deposits."); @@ -243,6 +243,7 @@ impl BeaconState { Ok(()) } + /// Converts a `RelativeEpoch` into an `Epoch` with respect to the epoch of this state. fn absolute_epoch(&self, relative_epoch: RelativeEpoch, spec: &ChainSpec) -> Epoch { match relative_epoch { RelativeEpoch::Previous => self.previous_epoch(spec), @@ -251,6 +252,10 @@ impl BeaconState { } } + /// Converts an `Epoch` into a `RelativeEpoch` with respect to the epoch of this state. + /// + /// Returns an error if the given `epoch` not "previous", "current" or "next" compared to the + /// epoch of this tate. fn relative_epoch(&self, epoch: Epoch, spec: &ChainSpec) -> Result { match epoch { e if e == self.current_epoch(spec) => Ok(RelativeEpoch::Current), @@ -260,6 +265,16 @@ impl BeaconState { } } + /// Advances the cache for this state into the next epoch. + /// + /// This should be used if the `slot` of this state is advanced beyond an epoch boundary. + /// + /// The `Next` cache becomes the `Current` and the `Current` cache becomes the `Previous`. The + /// `Previous` cache is abandoned. + /// + /// Care should be taken to update the `Current` epoch in case a registry update is performed + /// -- `Next` epoch is always _without_ a registry change. If you perform a registry update, + /// you should rebuild the `Current` cache so it uses the new seed. pub fn advance_caches(&mut self) { let previous_cache_index = self.cache_index(RelativeEpoch::Previous); @@ -269,6 +284,7 @@ impl BeaconState { self.cache_index_offset %= CACHED_EPOCHS; } + /// Returns the index of `self.caches` for some `RelativeEpoch`. fn cache_index(&self, relative_epoch: RelativeEpoch) -> usize { let base_index = match relative_epoch { RelativeEpoch::Current => 1, @@ -279,6 +295,8 @@ impl BeaconState { (base_index + self.cache_index_offset) % CACHED_EPOCHS } + /// Returns the cache for some `RelativeEpoch`. Returns an error if the cache has not been + /// initialized. fn cache<'a>(&'a self, relative_epoch: RelativeEpoch) -> Result<&'a EpochCache, Error> { let cache = &self.caches[self.cache_index(relative_epoch)]; @@ -356,10 +374,11 @@ impl BeaconState { } /// Shuffle ``validators`` into crosslink committees seeded by ``seed`` and ``epoch``. + /// /// Return a list of ``committees_per_epoch`` committees where each /// committee is itself a list of validator indices. /// - /// Spec v0.1 + /// Spec v0.2.0 pub(crate) fn get_shuffling( &self, seed: Hash256, @@ -428,6 +447,9 @@ impl BeaconState { self.get_epoch_committee_count(current_active_validators.len(), spec) } + /// Return the index root at a recent `epoch`. + /// + /// Spec v0.2.0 pub fn get_active_index_root(&self, epoch: Epoch, spec: &ChainSpec) -> Option { let current_epoch = self.current_epoch(spec); @@ -443,7 +465,7 @@ impl BeaconState { } } - /// Generate a seed for the given ``epoch``. + /// Generate a seed for the given `epoch`. /// /// Spec v0.2.0 pub fn generate_seed(&self, epoch: Epoch, spec: &ChainSpec) -> Result { @@ -465,6 +487,11 @@ impl BeaconState { Ok(Hash256::from(&hash(&input[..])[..])) } + /// Returns the crosslink committees for some slot. + /// + /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. + /// + /// Spec v0.2.0 pub fn get_crosslink_committees_at_slot( &self, slot: Slot, @@ -479,6 +506,11 @@ impl BeaconState { Ok(&cache.committees[slot_offset.as_usize()]) } + /// Returns the crosslink committees for some slot. + /// + /// Utilizes the cache and will fail if the appropriate cache is not initialized. + /// + /// Spec v0.2.0 pub(crate) fn get_shuffling_for_slot( &self, slot: Slot, @@ -492,6 +524,18 @@ impl BeaconState { .ok_or_else(|| Error::UnableToShuffle) } + /// Returns the following params for the given slot: + /// + /// - epoch committee count + /// - epoch seed + /// - calculation epoch + /// - start shard + /// + /// In the spec, this functionality is included in the `get_crosslink_committees_at_slot(..)` + /// function. It is separated here to allow the division of shuffling and committee building, + /// as is required for efficient operations. + /// + /// Spec v0.2.0 pub(crate) fn get_committee_params_at_slot( &self, slot: Slot, @@ -555,7 +599,8 @@ impl BeaconState { /// Note: There are two possible shufflings for crosslink committees for a /// `slot` in the next epoch: with and without a `registry_change` /// - /// Note: this is equivalent to the `get_crosslink_committees_at_slot` function in the spec. + /// Note: does not utilize the cache, `get_crosslink_committees_at_slot` is an equivalent + /// function which uses the cache. /// /// Spec v0.2.0 pub(crate) fn calculate_crosslink_committees_at_slot( @@ -589,6 +634,8 @@ impl BeaconState { /// /// Only reads the current epoch. /// + /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. + /// /// Spec v0.2.0 pub fn attestation_slot_and_shard_for_validator( &self, @@ -745,7 +792,14 @@ impl BeaconState { self.validator_registry_update_epoch = current_epoch; } - pub fn process_deposits_optimized( + /// Process multiple deposits in sequence. + /// + /// Builds a hashmap of validator pubkeys to validator index and passes it to each successive + /// call to `process_deposit(..)`. This requires much less computation than successive calls to + /// `process_deposits(..)` without the hashmap. + /// + /// Spec v0.2.0 + pub fn process_deposits( &mut self, deposits: Vec<&DepositData>, spec: &ChainSpec, @@ -775,6 +829,10 @@ impl BeaconState { /// Process a validator deposit, returning the validator index if the deposit is valid. /// + /// Optionally accepts a hashmap of all validator pubkeys to their validator index. Without + /// this hashmap, each call to `process_deposits` requires an iteration though + /// `self.validator_registry`. This becomes highly inefficient at scale. + /// /// Spec v0.2.0 pub fn process_deposit( &mut self, @@ -1058,6 +1116,11 @@ impl BeaconState { Ok(all_participants) } + /// Returns the list of validator indices which participiated in the attestation. + /// + /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. + /// + /// Spec v0.2.0 pub fn get_attestation_participants( &self, attestation_data: &AttestationData, From ab10cbbdb5ae6d706a59e454658ded89693b2061 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 24 Feb 2019 18:52:12 +1300 Subject: [PATCH 8/9] Fix clippy lints, small typos --- beacon_node/beacon_chain/src/beacon_chain.rs | 7 +------ .../state_processing/src/epoch_processable.rs | 1 - eth2/types/src/beacon_state.rs | 20 +++++++++---------- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 9ee55e5a3..d0e7b7f8f 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -495,12 +495,7 @@ where // TODO: this is a first-in-best-dressed scenario that is not ideal; fork_choice should be // run instead. if self.head().beacon_block_root == parent_block_root { - self.update_canonical_head( - block.clone(), - block_root.clone(), - state.clone(), - state_root, - ); + self.update_canonical_head(block.clone(), block_root, state.clone(), state_root); // Update the local state variable. *self.state.write() = state.clone(); } diff --git a/eth2/state_processing/src/epoch_processable.rs b/eth2/state_processing/src/epoch_processable.rs index 409d40a2c..9b6c98c86 100644 --- a/eth2/state_processing/src/epoch_processable.rs +++ b/eth2/state_processing/src/epoch_processable.rs @@ -658,7 +658,6 @@ fn winning_root( continue; } - // TODO: `cargo fmt` makes this rather ugly; tidy up. let attesting_validator_indices = attestations .iter() .try_fold::<_, _, Result<_, BeaconStateError>>(vec![], |mut acc, a| { diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 88f5422f8..e6dc9f6a6 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -222,10 +222,10 @@ impl BeaconState { ) -> Result<(), Error> { let cache_index = self.cache_index(relative_epoch); - if self.caches[cache_index].initialized == false { - self.force_build_epoch_cache(relative_epoch, spec) - } else { + if self.caches[cache_index].initialized { Ok(()) + } else { + self.force_build_epoch_cache(relative_epoch, spec) } } @@ -297,13 +297,13 @@ impl BeaconState { /// Returns the cache for some `RelativeEpoch`. Returns an error if the cache has not been /// initialized. - fn cache<'a>(&'a self, relative_epoch: RelativeEpoch) -> Result<&'a EpochCache, Error> { + fn cache(&self, relative_epoch: RelativeEpoch) -> Result<&EpochCache, Error> { let cache = &self.caches[self.cache_index(relative_epoch)]; - if cache.initialized == false { - Err(Error::EpochCacheUninitialized(relative_epoch)) - } else { + if cache.initialized { Ok(cache) + } else { + Err(Error::EpochCacheUninitialized(relative_epoch)) } } @@ -548,7 +548,7 @@ impl BeaconState { let next_epoch = self.next_epoch(spec); if epoch == current_epoch { - trace!("get_crosslink_committees_at_slot: current_epoch"); + trace!("get_committee_params_at_slot: current_epoch"); Ok(( self.get_current_epoch_committee_count(spec), self.current_epoch_seed, @@ -556,7 +556,7 @@ impl BeaconState { self.current_epoch_start_shard, )) } else if epoch == previous_epoch { - trace!("get_crosslink_committees_at_slot: previous_epoch"); + trace!("get_committee_params_at_slot: previous_epoch"); Ok(( self.get_previous_epoch_committee_count(spec), self.previous_epoch_seed, @@ -564,7 +564,7 @@ impl BeaconState { self.previous_epoch_start_shard, )) } else if epoch == next_epoch { - trace!("get_crosslink_committees_at_slot: next_epoch"); + trace!("get_committee_params_at_slot: next_epoch"); let current_committees_per_epoch = self.get_current_epoch_committee_count(spec); let epochs_since_last_registry_update = current_epoch - self.validator_registry_update_epoch; From 27e7bbd72fe27d7d9d7440638a72509f77ae2583 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 25 Feb 2019 08:35:21 +1300 Subject: [PATCH 9/9] Fix typo in BeaconState Co-Authored-By: paulhauner --- eth2/types/src/beacon_state.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index e6dc9f6a6..0270bb87c 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -229,7 +229,7 @@ impl BeaconState { } } - /// Always builds an epoch cache, even if it is alread initialized. + /// Always builds an epoch cache, even if it is already initialized. pub fn force_build_epoch_cache( &mut self, relative_epoch: RelativeEpoch,