From 3a384d93f85001bd316819571a635a4cfab1c2a4 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 17 Mar 2019 17:47:12 +1100 Subject: [PATCH] Allow state processing to compile under v0.5.0 --- .../src/per_block_processing.rs | 4 +- .../validate_attestation.rs | 4 +- .../src/per_epoch_processing.rs | 172 ++++++++---------- .../get_attestation_participants.rs | 37 ++++ .../inclusion_distance.rs | 15 +- .../process_validator_registry.rs | 72 ++++++++ .../validator_statuses.rs | 60 +++--- .../src/per_epoch_processing/winning_root.rs | 100 +++++++--- eth2/types/src/beacon_state.rs | 161 ++++++++++------ eth2/types/src/beacon_state/epoch_cache.rs | 30 +-- eth2/types/src/crosslink_committee.rs | 1 + .../testing_beacon_block_builder.rs | 5 +- .../testing_beacon_state_builder.rs | 3 +- 13 files changed, 422 insertions(+), 242 deletions(-) create mode 100644 eth2/state_processing/src/per_epoch_processing/get_attestation_participants.rs create mode 100644 eth2/state_processing/src/per_epoch_processing/process_validator_registry.rs diff --git a/eth2/state_processing/src/per_block_processing.rs b/eth2/state_processing/src/per_block_processing.rs index 377f92e8b..c6b22fa75 100644 --- a/eth2/state_processing/src/per_block_processing.rs +++ b/eth2/state_processing/src/per_block_processing.rs @@ -72,8 +72,8 @@ fn per_block_processing_signature_optional( process_block_header(state, block, spec)?; // Ensure the current and previous epoch cache is built. - state.build_epoch_cache(RelativeEpoch::Current, spec)?; state.build_epoch_cache(RelativeEpoch::Previous, spec)?; + state.build_epoch_cache(RelativeEpoch::Current, spec)?; if should_verify_block_signature { verify_block_signature(&state, &block, &spec)?; @@ -94,7 +94,7 @@ fn per_block_processing_signature_optional( /// /// Spec v0.5.0 pub fn process_block_header( - state: &BeaconState, + state: &mut BeaconState, block: &BeaconBlock, spec: &ChainSpec, ) -> Result<(), Error> { diff --git a/eth2/state_processing/src/per_block_processing/validate_attestation.rs b/eth2/state_processing/src/per_block_processing/validate_attestation.rs index 68a51b2df..272eeb18b 100644 --- a/eth2/state_processing/src/per_block_processing/validate_attestation.rs +++ b/eth2/state_processing/src/per_block_processing/validate_attestation.rs @@ -142,10 +142,8 @@ fn validate_attestation_signature_optional( ); // Get the committee for the specific shard that this attestation is for. - let relative_epoch = RelativeEpoch::from_slot(state.slot, attestation.data.slot, spec) - .map_err(|_| BeaconStateError::EpochOutOfBounds)?; // Should not fail due to previous checks. let crosslink_committee = state - .get_crosslink_committees_at_slot(attestation.data.slot, relative_epoch, spec)? + .get_crosslink_committees_at_slot(attestation.data.slot, spec)? .iter() .find(|c| c.shard == attestation.data.shard) .ok_or_else(|| { diff --git a/eth2/state_processing/src/per_epoch_processing.rs b/eth2/state_processing/src/per_epoch_processing.rs index 8c4b8e88b..2f1cc3551 100644 --- a/eth2/state_processing/src/per_epoch_processing.rs +++ b/eth2/state_processing/src/per_epoch_processing.rs @@ -1,5 +1,6 @@ use errors::EpochProcessingError as Error; use integer_sqrt::IntegerSquareRoot; +use process_validator_registry::process_validator_registry; use rayon::prelude::*; use ssz::TreeHash; use std::collections::HashMap; @@ -8,7 +9,9 @@ use validator_statuses::{TotalBalances, ValidatorStatuses}; use winning_root::{winning_root, WinningRoot}; pub mod errors; +pub mod get_attestation_participants; pub mod inclusion_distance; +pub mod process_validator_registry; pub mod tests; pub mod validator_statuses; pub mod winning_root; @@ -25,10 +28,9 @@ pub type WinningRootHashSet = HashMap; /// /// Spec v0.4.0 pub fn per_epoch_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> { - // Ensure all of the caches are built. + // Ensure the previous and next epoch caches are built. state.build_epoch_cache(RelativeEpoch::Previous, spec)?; state.build_epoch_cache(RelativeEpoch::Current, spec)?; - state.build_epoch_cache(RelativeEpoch::Next, spec)?; let mut statuses = initialize_validator_statuses(&state, spec)?; @@ -61,7 +63,7 @@ pub fn per_epoch_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result /// Returns a list of active validator indices for the state's current epoch. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn calculate_active_validator_indices(state: &BeaconState, spec: &ChainSpec) -> Vec { get_active_validator_indices( &state.validator_registry, @@ -76,26 +78,28 @@ pub fn calculate_active_validator_indices(state: &BeaconState, spec: &ChainSpec) /// - previous epoch attesters /// - etc. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn initialize_validator_statuses( state: &BeaconState, spec: &ChainSpec, ) -> Result { - let mut statuses = ValidatorStatuses::new(state, spec); + let mut statuses = ValidatorStatuses::new(state, spec)?; - statuses.process_attestations(&state, &state.latest_attestations, spec)?; + statuses.process_attestations(&state, spec)?; Ok(statuses) } -/// Spec v0.4.0 +/// Maybe resets the eth1 period. +/// +/// Spec v0.5.0 pub fn process_eth1_data(state: &mut BeaconState, spec: &ChainSpec) { let next_epoch = state.next_epoch(spec); let voting_period = spec.epochs_per_eth1_voting_period; if next_epoch % voting_period == 0 { for eth1_data_vote in &state.eth1_data_votes { - if eth1_data_vote.vote_count * 2 > voting_period { + if eth1_data_vote.vote_count * 2 > voting_period * spec.slots_per_epoch { state.latest_eth1_data = eth1_data_vote.eth1_data.clone(); } } @@ -119,7 +123,7 @@ pub fn process_justification( let previous_epoch = state.previous_epoch(spec); let current_epoch = state.current_epoch(spec); - let mut new_justified_epoch = state.justified_epoch; + let mut new_justified_epoch = state.current_justified_epoch; state.justification_bitfield <<= 1; // If > 2/3 of the total balance attested to the previous epoch boundary @@ -168,8 +172,10 @@ pub fn process_justification( // - The presently justified epoch was two epochs ago. // // Then, set the finalized epoch to two epochs ago. - if (state.justification_bitfield % 8 == 0b111) & (state.justified_epoch == previous_epoch - 1) { - state.finalized_epoch = state.justified_epoch; + if (state.justification_bitfield % 8 == 0b111) + & (state.current_justified_epoch == previous_epoch - 1) + { + state.finalized_epoch = state.current_justified_epoch; } // If: // @@ -177,12 +183,14 @@ pub fn process_justification( // - Set the previous epoch to be justified. // // Then, set the finalized epoch to be the previous epoch. - if (state.justification_bitfield % 4 == 0b11) & (state.justified_epoch == previous_epoch) { - state.finalized_epoch = state.justified_epoch; + if (state.justification_bitfield % 4 == 0b11) + & (state.current_justified_epoch == previous_epoch) + { + state.finalized_epoch = state.current_justified_epoch; } - state.previous_justified_epoch = state.justified_epoch; - state.justified_epoch = new_justified_epoch; + state.previous_justified_epoch = state.current_justified_epoch; + state.current_justified_epoch = new_justified_epoch; } /// Updates the following fields on the `BeaconState`: @@ -191,23 +199,11 @@ pub fn process_justification( /// /// Also returns a `WinningRootHashSet` for later use during epoch processing. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn process_crosslinks( state: &mut BeaconState, spec: &ChainSpec, ) -> Result { - let current_epoch_attestations: Vec<&PendingAttestation> = state - .latest_attestations - .par_iter() - .filter(|a| a.data.slot.epoch(spec.slots_per_epoch) == state.current_epoch(spec)) - .collect(); - - let previous_epoch_attestations: Vec<&PendingAttestation> = state - .latest_attestations - .par_iter() - .filter(|a| a.data.slot.epoch(spec.slots_per_epoch) == state.previous_epoch(spec)) - .collect(); - let mut winning_root_for_shards: WinningRootHashSet = HashMap::new(); let previous_and_current_epoch_slots: Vec = state @@ -221,24 +217,18 @@ pub fn process_crosslinks( let crosslink_committees_at_slot = state.get_crosslink_committees_at_slot(slot, spec)?.clone(); - for (crosslink_committee, shard) in crosslink_committees_at_slot { - let shard = shard as u64; + for c in crosslink_committees_at_slot { + let shard = c.shard as u64; - let winning_root = winning_root( - state, - shard, - ¤t_epoch_attestations[..], - &previous_epoch_attestations[..], - spec, - )?; + let winning_root = winning_root(state, shard, spec)?; if let Some(winning_root) = winning_root { - let total_committee_balance = state.get_total_balance(&crosslink_committee, spec); + let total_committee_balance = state.get_total_balance(&c.committee, spec)?; // TODO: I think this has a bug. if (3 * winning_root.total_attesting_balance) >= (2 * total_committee_balance) { state.latest_crosslinks[shard as usize] = Crosslink { - epoch: state.current_epoch(spec), + epoch: slot.epoch(spec.slots_per_epoch), crosslink_data_root: winning_root.crosslink_data_root, } } @@ -294,7 +284,10 @@ pub fn process_rewards_and_penalities( .map(|(index, &balance)| { let mut balance = balance; let status = &statuses.statuses[index]; - let base_reward = state.base_reward(index, base_reward_quotient, spec); + let base_reward = get_base_reward(state, index, total_balances.previous_epoch, spec) + .expect( + "Cannot fail to access a validator balance when iterating validator balances.", + ); if epochs_since_finality <= 4 { // Expected FFG source @@ -330,11 +323,15 @@ pub fn process_rewards_and_penalities( safe_sub_assign!(balance, base_reward); }; } else { - let inactivity_penalty = state.inactivity_penalty( + let inactivity_penalty = get_inactivity_penalty( + state, index, - epochs_since_finality, - base_reward_quotient, + epochs_since_finality.as_u64(), + total_balances.previous_epoch, spec, + ) + .expect( + "Cannot fail to access a validator balance when iterating validator balances.", ); if status.is_active_in_previous_epoch { @@ -349,7 +346,10 @@ pub fn process_rewards_and_penalities( } if state.validator_registry[index].slashed { - let base_reward = state.base_reward(index, base_reward_quotient, spec); + let base_reward = + get_base_reward(state, index, total_balances.previous_epoch, spec).expect( + "Cannot fail to access a validator balance when iterating validator balances.", + ); safe_sub_assign!(balance, 2 * inactivity_penalty + base_reward); } } @@ -384,7 +384,10 @@ pub fn process_rewards_and_penalities( let proposer_index = status.inclusion_info.proposer_index; let inclusion_distance = status.inclusion_info.distance; - let base_reward = state.base_reward(proposer_index, base_reward_quotient, spec); + let base_reward = + get_base_reward(state, proposer_index, total_balances.previous_epoch, spec).expect( + "Cannot fail to access a validator balance when iterating validator balances.", + ); if inclusion_distance > 0 && inclusion_distance < Slot::max_value() { safe_add_assign!( @@ -399,53 +402,37 @@ pub fn process_rewards_and_penalities( Ok(()) } -/// Peforms a validator registry update, if required. +/// Returns the base reward for some validator. /// -/// Spec v0.4.0 -pub fn process_validator_registry(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> { - let current_epoch = state.current_epoch(spec); - let next_epoch = state.next_epoch(spec); - - state.previous_shuffling_epoch = state.current_shuffling_epoch; - state.previous_shuffling_start_shard = state.current_shuffling_start_shard; - - state.previous_shuffling_seed = state.current_shuffling_seed; - - let should_update_validator_registy = if state.finalized_epoch - > state.validator_registry_update_epoch - { - (0..state.get_current_epoch_committee_count(spec)).all(|i| { - let shard = (state.current_shuffling_start_shard + i as u64) % spec.shard_count; - state.latest_crosslinks[shard as usize].epoch > state.validator_registry_update_epoch - }) +/// Spec v0.5.0 +pub fn get_base_reward( + state: &BeaconState, + index: usize, + previous_total_balance: u64, + spec: &ChainSpec, +) -> Result { + if previous_total_balance == 0 { + Ok(0) } else { - false - }; - - if should_update_validator_registy { - state.update_validator_registry(spec); - - state.current_shuffling_epoch = next_epoch; - state.current_shuffling_start_shard = (state.current_shuffling_start_shard - + state.get_current_epoch_committee_count(spec) as u64) - % spec.shard_count; - state.current_shuffling_seed = state.generate_seed(state.current_shuffling_epoch, spec)? - } else { - let epochs_since_last_registry_update = - current_epoch - state.validator_registry_update_epoch; - if (epochs_since_last_registry_update > 1) - & epochs_since_last_registry_update.is_power_of_two() - { - state.current_shuffling_epoch = next_epoch; - state.current_shuffling_seed = - state.generate_seed(state.current_shuffling_epoch, spec)? - } + let adjusted_quotient = previous_total_balance.integer_sqrt() / spec.base_reward_quotient; + Ok(state.get_effective_balance(index, spec)? / adjusted_quotient / 5) } +} - state.process_slashings(spec); - state.process_exit_queue(spec); - - Ok(()) +/// Returns the inactivity penalty for some validator. +/// +/// Spec v0.5.0 +pub fn get_inactivity_penalty( + state: &BeaconState, + index: usize, + epochs_since_finality: u64, + previous_total_balance: u64, + spec: &ChainSpec, +) -> Result { + Ok(get_base_reward(state, index, previous_total_balance, spec)? + + state.get_effective_balance(index, spec)? * epochs_since_finality + / spec.inactivity_penalty_quotient + / 2) } /// Updates the state's `latest_active_index_roots` field with a tree hash the active validator @@ -486,12 +473,5 @@ pub fn update_latest_slashed_balances(state: &mut BeaconState, spec: &ChainSpec) /// /// Spec v0.4.0 pub fn clean_attestations(state: &mut BeaconState, spec: &ChainSpec) { - let current_epoch = state.current_epoch(spec); - - state.latest_attestations = state - .latest_attestations - .iter() - .filter(|a| a.data.slot.epoch(spec.slots_per_epoch) >= current_epoch) - .cloned() - .collect(); + state.previous_epoch_attestations = vec![]; } diff --git a/eth2/state_processing/src/per_epoch_processing/get_attestation_participants.rs b/eth2/state_processing/src/per_epoch_processing/get_attestation_participants.rs new file mode 100644 index 000000000..d822e434d --- /dev/null +++ b/eth2/state_processing/src/per_epoch_processing/get_attestation_participants.rs @@ -0,0 +1,37 @@ +use types::{beacon_state::helpers::verify_bitfield_length, *}; + +/// Returns validator indices which participated in the attestation. +/// +/// Spec v0.5.0 +pub fn get_attestation_participants( + state: &BeaconState, + attestation_data: &AttestationData, + bitfield: &Bitfield, + spec: &ChainSpec, +) -> Result, BeaconStateError> { + let epoch = attestation_data.slot.epoch(spec.slots_per_epoch); + + let crosslink_committee = + state.get_crosslink_committee_for_shard(epoch, attestation_data.shard, spec)?; + + if crosslink_committee.slot != attestation_data.slot { + return Err(BeaconStateError::NoCommitteeForShard); + } + + let committee = &crosslink_committee.committee; + + if !verify_bitfield_length(&bitfield, committee.len()) { + return Err(BeaconStateError::InvalidBitfield); + } + + let mut participants = Vec::with_capacity(committee.len()); + for (i, validator_index) in committee.iter().enumerate() { + match bitfield.get(i) { + Ok(bit) if bit == true => participants.push(*validator_index), + _ => {} + } + } + participants.shrink_to_fit(); + + Ok(participants) +} diff --git a/eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs b/eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs index 243dc67f0..b52485947 100644 --- a/eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs +++ b/eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs @@ -1,12 +1,11 @@ use super::errors::InclusionError; +use super::get_attestation_participants::get_attestation_participants; use types::*; /// Returns the distance between the first included attestation for some validator and this /// slot. /// -/// Note: In the spec this is defined "inline", not as a helper function. -/// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn inclusion_distance( state: &BeaconState, attestations: &[&PendingAttestation], @@ -19,9 +18,7 @@ pub fn inclusion_distance( /// Returns the slot of the earliest included attestation for some validator. /// -/// Note: In the spec this is defined "inline", not as a helper function. -/// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn inclusion_slot( state: &BeaconState, attestations: &[&PendingAttestation], @@ -34,9 +31,7 @@ pub fn inclusion_slot( /// Finds the earliest included attestation for some validator. /// -/// Note: In the spec this is defined "inline", not as a helper function. -/// -/// Spec v0.4.0 +/// Spec v0.5.0 fn earliest_included_attestation( state: &BeaconState, attestations: &[&PendingAttestation], @@ -47,7 +42,7 @@ fn earliest_included_attestation( for (i, a) in attestations.iter().enumerate() { let participants = - state.get_attestation_participants(&a.data, &a.aggregation_bitfield, spec)?; + get_attestation_participants(state, &a.data, &a.aggregation_bitfield, spec)?; if participants.iter().any(|i| *i == validator_index) { included_attestations.push(i); } diff --git a/eth2/state_processing/src/per_epoch_processing/process_validator_registry.rs b/eth2/state_processing/src/per_epoch_processing/process_validator_registry.rs new file mode 100644 index 000000000..c830bfc24 --- /dev/null +++ b/eth2/state_processing/src/per_epoch_processing/process_validator_registry.rs @@ -0,0 +1,72 @@ +use super::Error; +use types::*; + +/// Peforms a validator registry update, if required. +/// +/// Spec v0.4.0 +pub fn process_validator_registry(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> { + let current_epoch = state.current_epoch(spec); + let next_epoch = state.next_epoch(spec); + + state.previous_shuffling_epoch = state.current_shuffling_epoch; + state.previous_shuffling_start_shard = state.current_shuffling_start_shard; + + state.previous_shuffling_seed = state.current_shuffling_seed; + + if should_update_validator_registry(state, spec)? { + state.update_validator_registry(spec); + + state.current_shuffling_epoch = next_epoch; + state.current_shuffling_start_shard = (state.current_shuffling_start_shard + + spec.get_epoch_committee_count( + state + .get_active_validator_indices(current_epoch, spec)? + .len(), + ) as u64) + % spec.shard_count; + state.current_shuffling_seed = state.generate_seed(state.current_shuffling_epoch, spec)? + } else { + let epochs_since_last_registry_update = + current_epoch - state.validator_registry_update_epoch; + if (epochs_since_last_registry_update > 1) + & epochs_since_last_registry_update.is_power_of_two() + { + state.current_shuffling_epoch = next_epoch; + state.current_shuffling_seed = + state.generate_seed(state.current_shuffling_epoch, spec)? + } + } + + state.process_slashings(spec); + state.process_exit_queue(spec); + + Ok(()) +} + +/// Returns `true` if the validator registry should be updated during an epoch processing. +/// +/// Spec v0.5.0 +pub fn should_update_validator_registry( + state: &BeaconState, + spec: &ChainSpec, +) -> Result { + if state.finalized_epoch <= state.validator_registry_update_epoch { + return Ok(false); + } + + let num_active_validators = state + .get_active_validator_indices(state.current_epoch(spec), spec)? + .len(); + let current_epoch_committee_count = spec.get_epoch_committee_count(num_active_validators); + + for shard in (0..current_epoch_committee_count) + .into_iter() + .map(|i| (state.current_shuffling_start_shard + i as u64) % spec.shard_count) + { + if state.latest_crosslinks[shard as usize].epoch <= state.validator_registry_update_epoch { + return Ok(false); + } + } + + Ok(true) +} diff --git a/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs b/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs index f76900f3b..bcbca8244 100644 --- a/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs +++ b/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs @@ -1,3 +1,4 @@ +use super::get_attestation_participants::get_attestation_participants; use super::WinningRootHashSet; use types::*; @@ -147,8 +148,8 @@ impl ValidatorStatuses { /// - Active validators /// - Total balances for the current and previous epochs. /// - /// Spec v0.4.0 - pub fn new(state: &BeaconState, spec: &ChainSpec) -> Self { + /// Spec v0.5.0 + pub fn new(state: &BeaconState, spec: &ChainSpec) -> Result { let mut statuses = Vec::with_capacity(state.validator_registry.len()); let mut total_balances = TotalBalances::default(); @@ -157,37 +158,40 @@ impl ValidatorStatuses { if validator.is_active_at(state.current_epoch(spec)) { status.is_active_in_current_epoch = true; - total_balances.current_epoch += state.get_effective_balance(i, spec); + total_balances.current_epoch += state.get_effective_balance(i, spec)?; } if validator.is_active_at(state.previous_epoch(spec)) { status.is_active_in_previous_epoch = true; - total_balances.previous_epoch += state.get_effective_balance(i, spec); + total_balances.previous_epoch += state.get_effective_balance(i, spec)?; } statuses.push(status); } - Self { + Ok(Self { statuses, total_balances, - } + }) } /// Process some attestations from the given `state` updating the `statuses` and /// `total_balances` fields. /// - /// Spec v0.4.0 + /// Spec v0.5.0 pub fn process_attestations( &mut self, state: &BeaconState, - attestations: &[PendingAttestation], spec: &ChainSpec, ) -> Result<(), BeaconStateError> { - for a in attestations { + for a in state + .previous_epoch_attestations + .iter() + .chain(state.current_epoch_attestations.iter()) + { let attesting_indices = - state.get_attestation_participants(&a.data, &a.aggregation_bitfield, spec)?; - let attesting_balance = state.get_total_balance(&attesting_indices, spec); + get_attestation_participants(state, &a.data, &a.aggregation_bitfield, spec)?; + let attesting_balance = state.get_total_balance(&attesting_indices, spec)?; let mut status = AttesterStatus::default(); @@ -206,10 +210,15 @@ impl ValidatorStatuses { status.is_previous_epoch_attester = true; // The inclusion slot and distance are only required for previous epoch attesters. + let relative_epoch = RelativeEpoch::from_slot(state.slot, a.data.slot, spec)?; status.inclusion_info = InclusionInfo { slot: a.inclusion_slot, distance: inclusion_distance(a), - proposer_index: state.get_beacon_proposer_index(a.inclusion_slot, spec)?, + proposer_index: state.get_beacon_proposer_index( + a.inclusion_slot, + relative_epoch, + spec, + )?, }; if has_common_epoch_boundary_root(a, state, state.previous_epoch(spec), spec)? { @@ -235,7 +244,7 @@ impl ValidatorStatuses { /// Update the `statuses` for each validator based upon whether or not they attested to the /// "winning" shard block root for the previous epoch. /// - /// Spec v0.4.0 + /// Spec v0.5.0 pub fn process_winning_roots( &mut self, state: &BeaconState, @@ -248,11 +257,10 @@ impl ValidatorStatuses { state.get_crosslink_committees_at_slot(slot, spec)?; // Loop through each committee in the slot. - for (crosslink_committee, shard) in crosslink_committees_at_slot { + for c in crosslink_committees_at_slot { // If there was some winning crosslink root for the committee's shard. - if let Some(winning_root) = winning_roots.get(&shard) { - let total_committee_balance = - state.get_total_balance(&crosslink_committee, spec); + if let Some(winning_root) = winning_roots.get(&c.shard) { + let total_committee_balance = state.get_total_balance(&c.committee, spec)?; for &validator_index in &winning_root.attesting_validator_indices { // Take note of the balance information for the winning root, it will be // used later to calculate rewards for that validator. @@ -272,14 +280,14 @@ impl ValidatorStatuses { /// Returns the distance between when the attestation was created and when it was included in a /// block. /// -/// Spec v0.4.0 +/// Spec v0.5.0 fn inclusion_distance(a: &PendingAttestation) -> Slot { a.inclusion_slot - a.data.slot } /// Returns `true` if some `PendingAttestation` is from the supplied `epoch`. /// -/// Spec v0.4.0 +/// Spec v0.5.0 fn is_from_epoch(a: &PendingAttestation, epoch: Epoch, spec: &ChainSpec) -> bool { a.data.slot.epoch(spec.slots_per_epoch) == epoch } @@ -287,7 +295,7 @@ fn is_from_epoch(a: &PendingAttestation, epoch: Epoch, spec: &ChainSpec) -> bool /// Returns `true` if a `PendingAttestation` and `BeaconState` share the same beacon block hash for /// the first slot of the given epoch. /// -/// Spec v0.4.0 +/// Spec v0.5.0 fn has_common_epoch_boundary_root( a: &PendingAttestation, state: &BeaconState, @@ -295,25 +303,21 @@ fn has_common_epoch_boundary_root( spec: &ChainSpec, ) -> Result { let slot = epoch.start_slot(spec.slots_per_epoch); - let state_boundary_root = *state - .get_block_root(slot, spec) - .ok_or_else(|| BeaconStateError::InsufficientBlockRoots)?; + let state_boundary_root = *state.get_block_root(slot, spec)?; - Ok(a.data.epoch_boundary_root == state_boundary_root) + Ok(a.data.target_root == state_boundary_root) } /// Returns `true` if a `PendingAttestation` and `BeaconState` share the same beacon block hash for /// the current slot of the `PendingAttestation`. /// -/// Spec v0.4.0 +/// Spec v0.5.0 fn has_common_beacon_block_root( a: &PendingAttestation, state: &BeaconState, spec: &ChainSpec, ) -> Result { - let state_block_root = *state - .get_block_root(a.data.slot, spec) - .ok_or_else(|| BeaconStateError::InsufficientBlockRoots)?; + let state_block_root = *state.get_block_root(a.data.slot, spec)?; Ok(a.data.beacon_block_root == state_block_root) } diff --git a/eth2/state_processing/src/per_epoch_processing/winning_root.rs b/eth2/state_processing/src/per_epoch_processing/winning_root.rs index 07678f93b..97cff3e13 100644 --- a/eth2/state_processing/src/per_epoch_processing/winning_root.rs +++ b/eth2/state_processing/src/per_epoch_processing/winning_root.rs @@ -1,3 +1,4 @@ +use super::get_attestation_participants::get_attestation_participants; use std::collections::HashSet; use std::iter::FromIterator; use types::*; @@ -13,14 +14,14 @@ impl WinningRoot { /// Returns `true` if `self` is a "better" candidate than `other`. /// /// A winning root is "better" than another if it has a higher `total_attesting_balance`. Ties - /// are broken by favouring the lower `crosslink_data_root` value. + /// are broken by favouring the higher `crosslink_data_root` value. /// - /// Spec v0.4.0 + /// Spec v0.5.0 pub fn is_better_than(&self, other: &Self) -> bool { if self.total_attesting_balance > other.total_attesting_balance { true } else if self.total_attesting_balance == other.total_attesting_balance { - self.crosslink_data_root < other.crosslink_data_root + self.crosslink_data_root > other.crosslink_data_root } else { false } @@ -33,22 +34,21 @@ impl WinningRoot { /// The `WinningRoot` object also contains additional fields that are useful in later stages of /// per-epoch processing. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn winning_root( state: &BeaconState, shard: u64, - current_epoch_attestations: &[&PendingAttestation], - previous_epoch_attestations: &[&PendingAttestation], spec: &ChainSpec, ) -> Result, BeaconStateError> { let mut winning_root: Option = None; let crosslink_data_roots: HashSet = HashSet::from_iter( - previous_epoch_attestations + state + .previous_epoch_attestations .iter() - .chain(current_epoch_attestations.iter()) + .chain(state.current_epoch_attestations.iter()) .filter_map(|a| { - if a.data.shard == shard { + if is_eligible_for_winning_root(state, a, shard) { Some(a.data.crosslink_data_root) } else { None @@ -57,18 +57,17 @@ pub fn winning_root( ); for crosslink_data_root in crosslink_data_roots { - let attesting_validator_indices = get_attesting_validator_indices( - state, - shard, - current_epoch_attestations, - previous_epoch_attestations, - &crosslink_data_root, - spec, - )?; + let attesting_validator_indices = + get_attesting_validator_indices(state, shard, &crosslink_data_root, spec)?; - let total_attesting_balance: u64 = attesting_validator_indices - .iter() - .fold(0, |acc, i| acc + state.get_effective_balance(*i, spec)); + let total_attesting_balance: u64 = + attesting_validator_indices + .iter() + .try_fold(0_u64, |acc, i| { + state + .get_effective_balance(*i, spec) + .and_then(|bal| Ok(acc + bal)) + })?; let candidate = WinningRoot { crosslink_data_root, @@ -88,25 +87,36 @@ pub fn winning_root( Ok(winning_root) } -/// Returns all indices which voted for a given crosslink. May contain duplicates. +/// Returns `true` if pending attestation `a` is eligible to become a winning root. /// -/// Spec v0.4.0 +/// Spec v0.5.0 +fn is_eligible_for_winning_root(state: &BeaconState, a: &PendingAttestation, shard: Shard) -> bool { + if shard >= state.latest_crosslinks.len() as u64 { + return false; + } + + a.data.previous_crosslink == state.latest_crosslinks[shard as usize] +} + +/// Returns all indices which voted for a given crosslink. Does not contain duplicates. +/// +/// Spec v0.5.0 fn get_attesting_validator_indices( state: &BeaconState, shard: u64, - current_epoch_attestations: &[&PendingAttestation], - previous_epoch_attestations: &[&PendingAttestation], crosslink_data_root: &Hash256, spec: &ChainSpec, ) -> Result, BeaconStateError> { let mut indices = vec![]; - for a in current_epoch_attestations + for a in state + .current_epoch_attestations .iter() - .chain(previous_epoch_attestations.iter()) + .chain(state.previous_epoch_attestations.iter()) { if (a.data.shard == shard) && (a.data.crosslink_data_root == *crosslink_data_root) { - indices.append(&mut state.get_attestation_participants( + indices.append(&mut get_attestation_participants( + state, &a.data, &a.aggregation_bitfield, spec, @@ -114,5 +124,41 @@ fn get_attesting_validator_indices( } } + // Sort the list (required for dedup). "Unstable" means the sort may re-order equal elements, + // this causes no issue here. + // + // These sort + dedup ops are potentially good CPU time optimisation targets. + indices.sort_unstable(); + // Remove all duplicate indices (requires a sorted list). + indices.dedup(); + Ok(indices) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn is_better_than() { + let worse = WinningRoot { + crosslink_data_root: Hash256::from_slice(&[1; 32]), + attesting_validator_indices: vec![], + total_attesting_balance: 42, + }; + + let better = WinningRoot { + crosslink_data_root: Hash256::from_slice(&[2; 32]), + ..worse.clone() + }; + + assert!(better.is_better_than(&worse)); + + let better = WinningRoot { + total_attesting_balance: worse.total_attesting_balance + 1, + ..worse.clone() + }; + + assert!(better.is_better_than(&worse)); + } +} diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 7c77a5a3e..a90f09759 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -1,8 +1,7 @@ use self::epoch_cache::EpochCache; use crate::test_utils::TestRandom; -use crate::{validator_registry::get_active_validator_indices, *}; +use crate::*; use int_to_bytes::int_to_bytes32; -use log::trace; use pubkey_cache::PubkeyCache; use rand::RngCore; use serde_derive::{Deserialize, Serialize}; @@ -39,6 +38,7 @@ pub enum Error { InsufficientAttestations, InsufficientCommittees, InsufficientSlashedBalances, + NoCommitteeForShard, EpochCacheUninitialized(RelativeEpoch), PubkeyCacheInconsistent, PubkeyCacheIncomplete { @@ -349,17 +349,49 @@ impl BeaconState { self.current_epoch(spec) + 1 } + /// Returns the active validator indices for the given epoch, assuming there is no validator + /// registry update in the next epoch. + /// + /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. + /// + /// Spec v0.5.0 + pub fn get_active_validator_indices( + &self, + epoch: Epoch, + spec: &ChainSpec, + ) -> Result<&[usize], Error> { + // If the slot is in the next epoch, assume there was no validator registry update. + let relative_epoch = + match RelativeEpoch::from_epoch(self.slot.epoch(spec.slots_per_epoch), epoch) { + Err(RelativeEpochError::AmbiguiousNextEpoch) => { + Ok(RelativeEpoch::NextWithoutRegistryChange) + } + e => e, + }?; + + let cache = self.cache(relative_epoch, spec)?; + + Ok(&cache.active_validator_indices) + } + /// Returns the crosslink committees for some slot. /// /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. /// - /// Spec v0.4.0 + /// Spec v0.5.0 pub fn get_crosslink_committees_at_slot( &self, slot: Slot, - relative_epoch: RelativeEpoch, spec: &ChainSpec, ) -> Result<&Vec, Error> { + // If the slot is in the next epoch, assume there was no validator registry update. + let relative_epoch = match RelativeEpoch::from_slot(self.slot, slot, spec) { + Err(RelativeEpochError::AmbiguiousNextEpoch) => { + Ok(RelativeEpoch::NextWithoutRegistryChange) + } + e => e, + }?; + let cache = self.cache(relative_epoch, spec)?; Ok(cache @@ -367,15 +399,46 @@ impl BeaconState { .ok_or_else(|| Error::SlotOutOfBounds)?) } + /// Returns the crosslink committees for some shard in an epoch. + /// + /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. + /// + /// Spec v0.4.0 + pub fn get_crosslink_committee_for_shard( + &self, + epoch: Epoch, + shard: Shard, + spec: &ChainSpec, + ) -> Result<&CrosslinkCommittee, Error> { + // If the slot is in the next epoch, assume there was no validator registry update. + let relative_epoch = match RelativeEpoch::from_epoch(self.current_epoch(spec), epoch) { + Err(RelativeEpochError::AmbiguiousNextEpoch) => { + Ok(RelativeEpoch::NextWithoutRegistryChange) + } + e => e, + }?; + + let cache = self.cache(relative_epoch, spec)?; + + Ok(cache + .get_crosslink_committee_for_shard(shard, spec) + .ok_or_else(|| Error::NoCommitteeForShard)?) + } + /// Return the block root at a recent `slot`. /// /// Spec v0.5.0 - pub fn get_block_root(&self, slot: Slot, spec: &ChainSpec) -> Option<&Hash256> { + pub fn get_block_root( + &self, + slot: Slot, + spec: &ChainSpec, + ) -> Result<&Hash256, BeaconStateError> { if (self.slot <= slot + spec.slots_per_historical_root as u64) && (slot < self.slot) { self.latest_block_roots .get(slot.as_usize() % spec.slots_per_historical_root) + .ok_or_else(|| Error::InsufficientBlockRoots) } else { - None + Err(Error::EpochOutOfBounds) } } @@ -476,12 +539,12 @@ impl BeaconState { relative_epoch: RelativeEpoch, spec: &ChainSpec, ) -> Result { - let committees = self.get_crosslink_committees_at_slot(slot, relative_epoch, spec)?; - trace!( - "get_beacon_proposer_index: slot: {}, committees_count: {}", - slot, - committees.len() - ); + let cache = self.cache(relative_epoch, spec)?; + + let committees = cache + .get_crosslink_committees_at_slot(slot, spec) + .ok_or_else(|| Error::SlotOutOfBounds)?; + committees .first() .ok_or(Error::InsufficientValidators) @@ -751,13 +814,14 @@ impl BeaconState { .ok_or_else(|| Error::UnknownValidator)?) } - /// Process the slashings. + /// Process slashings. + /// + /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. /// /// Spec v0.4.0 pub fn process_slashings(&mut self, spec: &ChainSpec) -> Result<(), Error> { let current_epoch = self.current_epoch(spec); - let active_validator_indices = - get_active_validator_indices(&self.validator_registry, current_epoch); + let active_validator_indices = self.get_active_validator_indices(current_epoch, spec)?; let total_balance = self.get_total_balance(&active_validator_indices[..], spec)?; for (index, validator) in self.validator_registry.iter().enumerate() { @@ -818,11 +882,12 @@ impl BeaconState { /// Update validator registry, activating/exiting validators if possible. /// + /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. + /// /// Spec v0.4.0 pub fn update_validator_registry(&mut self, spec: &ChainSpec) -> Result<(), Error> { let current_epoch = self.current_epoch(spec); - let active_validator_indices = - get_active_validator_indices(&self.validator_registry, current_epoch); + let active_validator_indices = self.get_active_validator_indices(current_epoch, spec)?; let total_balance = self.get_total_balance(&active_validator_indices[..], spec)?; let max_balance_churn = std::cmp::max( @@ -867,54 +932,32 @@ impl BeaconState { /// Iterate through the validator registry and eject active validators with balance below /// ``EJECTION_BALANCE``. /// - /// Spec v0.4.0 - pub fn process_ejections(&mut self, spec: &ChainSpec) { - for validator_index in - get_active_validator_indices(&self.validator_registry, self.current_epoch(spec)) - { - if self.validator_balances[validator_index] < spec.ejection_balance { - self.exit_validator(validator_index, spec) - } + /// Spec v0.5.0 + pub fn process_ejections(&mut self, spec: &ChainSpec) -> Result<(), Error> { + // There is an awkward double (triple?) loop here because we can't loop across the borrowed + // active validator indices and mutate state in the one loop. + let exitable: Vec = self + .get_active_validator_indices(self.current_epoch(spec), spec)? + .iter() + .filter_map(|&i| { + if self.validator_balances[i as usize] < spec.ejection_balance { + Some(i) + } else { + None + } + }) + .collect(); + + for validator_index in exitable { + self.exit_validator(validator_index, spec) } - } - /// Returns the penality that should be applied to some validator for inactivity. - /// - /// Note: this is defined "inline" in the spec, not as a helper function. - /// - /// Spec v0.4.0 - pub fn inactivity_penalty( - &self, - validator_index: usize, - epochs_since_finality: Epoch, - base_reward_quotient: u64, - spec: &ChainSpec, - ) -> Result { - let effective_balance = self.get_effective_balance(validator_index, spec)?; - let base_reward = self.base_reward(validator_index, base_reward_quotient, spec)?; - Ok(base_reward - + effective_balance * epochs_since_finality.as_u64() - / spec.inactivity_penalty_quotient - / 2) - } - - /// Returns the base reward for some validator. - /// - /// Note: In the spec this is defined "inline", not as a helper function. - /// - /// Spec v0.4.0 - pub fn base_reward( - &self, - validator_index: usize, - base_reward_quotient: u64, - spec: &ChainSpec, - ) -> Result { - Ok(self.get_effective_balance(validator_index, spec)? / base_reward_quotient / 5) + Ok(()) } /// Return the combined effective balance of an array of validators. /// - /// Spec v0.4.0 + /// Spec v0.5.0 pub fn get_total_balance( &self, validator_indices: &[usize], diff --git a/eth2/types/src/beacon_state/epoch_cache.rs b/eth2/types/src/beacon_state/epoch_cache.rs index 6312ea5a5..0759a7617 100644 --- a/eth2/types/src/beacon_state/epoch_cache.rs +++ b/eth2/types/src/beacon_state/epoch_cache.rs @@ -13,7 +13,9 @@ pub struct EpochCache { /// Maps validator index to a slot, shard and committee index for attestation. pub attestation_duties: Vec>, /// Maps a shard to an index of `self.committees`. - pub shard_committee_indices: Vec<(Slot, usize)>, + pub shard_committee_indices: Vec>, + /// Indices of all active validators in the epoch + pub active_validator_indices: Vec, } impl EpochCache { @@ -31,18 +33,18 @@ impl EpochCache { let builder = match relative_epoch { RelativeEpoch::Previous => EpochCrosslinkCommitteesBuilder::for_previous_epoch( state, - active_validator_indices, + active_validator_indices.clone(), spec, ), RelativeEpoch::Current => EpochCrosslinkCommitteesBuilder::for_current_epoch( state, - active_validator_indices, + active_validator_indices.clone(), spec, ), RelativeEpoch::NextWithRegistryChange => { EpochCrosslinkCommitteesBuilder::for_next_epoch( state, - active_validator_indices, + active_validator_indices.clone(), true, spec, )? @@ -50,7 +52,7 @@ impl EpochCache { RelativeEpoch::NextWithoutRegistryChange => { EpochCrosslinkCommitteesBuilder::for_next_epoch( state, - active_validator_indices, + active_validator_indices.clone(), false, spec, )? @@ -64,7 +66,7 @@ impl EpochCache { // 2. `shard_committee_indices`: maps `Shard` into a `CrosslinkCommittee` in // `EpochCrosslinkCommittees`. let mut attestation_duties = vec![None; state.validator_registry.len()]; - let mut shard_committee_indices = vec![(Slot::default(), 0); spec.shard_count as usize]; + let mut shard_committee_indices = vec![None; spec.shard_count as usize]; for (i, slot_committees) in epoch_crosslink_committees .crosslink_committees .iter() @@ -75,7 +77,7 @@ impl EpochCache { for (j, crosslink_committee) in slot_committees.iter().enumerate() { let shard = crosslink_committee.shard; - shard_committee_indices[shard as usize] = (slot, j); + shard_committee_indices[shard as usize] = Some((slot, j)); for (k, validator_index) in crosslink_committee.committee.iter().enumerate() { let attestation_duty = AttestationDuty { @@ -93,6 +95,7 @@ impl EpochCache { epoch_crosslink_committees, attestation_duties, shard_committee_indices, + active_validator_indices, }) } @@ -110,9 +113,13 @@ impl EpochCache { shard: Shard, spec: &ChainSpec, ) -> Option<&CrosslinkCommittee> { - let (slot, committee) = self.shard_committee_indices.get(shard as usize)?; - let slot_committees = self.get_crosslink_committees_at_slot(*slot, spec)?; - slot_committees.get(*committee) + if shard > self.shard_committee_indices.len() as u64 { + None + } else { + let (slot, committee) = self.shard_committee_indices[shard as usize]?; + let slot_committees = self.get_crosslink_committees_at_slot(slot, spec)?; + slot_committees.get(committee) + } } } @@ -261,13 +268,14 @@ impl EpochCrosslinkCommitteesBuilder { let committees_per_slot = (self.committees_per_epoch / spec.slots_per_epoch) as usize; - for i in 0..spec.slots_per_epoch as usize { + for (i, slot) in self.epoch.slot_iter(spec.slots_per_epoch).enumerate() { for j in (0..committees.len()) .into_iter() .skip(i * committees_per_slot) .take(committees_per_slot) { let crosslink_committee = CrosslinkCommittee { + slot, shard, committee: committees.remove(j), }; diff --git a/eth2/types/src/crosslink_committee.rs b/eth2/types/src/crosslink_committee.rs index 06a6562fc..af1778a1b 100644 --- a/eth2/types/src/crosslink_committee.rs +++ b/eth2/types/src/crosslink_committee.rs @@ -4,6 +4,7 @@ use ssz_derive::{Decode, Encode, TreeHash}; #[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize, Decode, Encode, TreeHash)] pub struct CrosslinkCommittee { + pub slot: Slot, pub shard: Shard, pub committee: Vec, } diff --git a/eth2/types/src/test_utils/testing_beacon_block_builder.rs b/eth2/types/src/test_utils/testing_beacon_block_builder.rs index 402bd79d6..6e48c8c17 100644 --- a/eth2/types/src/test_utils/testing_beacon_block_builder.rs +++ b/eth2/types/src/test_utils/testing_beacon_block_builder.rs @@ -109,10 +109,7 @@ impl TestingBeaconBlockBuilder { break; } - let relative_epoch = RelativeEpoch::from_slot(state.slot, slot, spec).unwrap(); - for crosslink_committee in - state.get_crosslink_committees_at_slot(slot, relative_epoch, spec)? - { + for crosslink_committee in state.get_crosslink_committees_at_slot(slot, spec)? { if attestations_added >= num_attestations { break; } diff --git a/eth2/types/src/test_utils/testing_beacon_state_builder.rs b/eth2/types/src/test_utils/testing_beacon_state_builder.rs index 9e613f0e9..54e2fbe96 100644 --- a/eth2/types/src/test_utils/testing_beacon_state_builder.rs +++ b/eth2/types/src/test_utils/testing_beacon_state_builder.rs @@ -223,9 +223,8 @@ impl TestingBeaconStateBuilder { for slot in first_slot..last_slot + 1 { let slot = Slot::from(slot); - let relative_epoch = RelativeEpoch::from_slot(state.slot, slot, spec).unwrap(); let committees = state - .get_crosslink_committees_at_slot(slot, relative_epoch, spec) + .get_crosslink_committees_at_slot(slot, spec) .unwrap() .clone();