Push more epoch processing fns to 0.5.0

eth2/state_processing/src/common/verify_bitfield.rs

@@ -4,7 +4,7 @@ use types::*;
 ///
 /// Is title `verify_bitfield` in spec.
 ///
-/// Spec v0.4.0
+/// Spec v0.5.0
 pub fn verify_bitfield_length(bitfield: &Bitfield, committee_size: usize) -> bool {
     if bitfield.num_bytes() != ((committee_size + 7) / 8) {
         return false;

eth2/state_processing/src/per_epoch_processing.rs

@@ -3,10 +3,10 @@ use errors::EpochProcessingError as Error;
 use process_ejections::process_ejections;
 use process_exit_queue::process_exit_queue;
 use process_slashings::process_slashings;
-use process_validator_registry::process_validator_registry;
 use ssz::TreeHash;
 use std::collections::HashMap;
 use types::*;
+use update_registry_and_shuffling_data::update_registry_and_shuffling_data;
 use validator_statuses::{TotalBalances, ValidatorStatuses};
 use winning_root::{winning_root, WinningRoot};
 
@@ -17,9 +17,8 @@ pub mod inclusion_distance;
 pub mod process_ejections;
 pub mod process_exit_queue;
 pub mod process_slashings;
-pub mod process_validator_registry;
 pub mod tests;
-pub mod update_validator_registry;
+pub mod update_registry_and_shuffling_data;
 pub mod validator_statuses;
 pub mod winning_root;
 
@@ -39,30 +38,34 @@ pub fn per_epoch_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result
     state.build_epoch_cache(RelativeEpoch::Previous, spec)?;
     state.build_epoch_cache(RelativeEpoch::Current, spec)?;
 
-    let mut statuses = initialize_validator_statuses(&state, spec)?;
+    // Load the struct we use to assign validators into sets based on their participation.
+    //
+    // E.g., attestation in the previous epoch, attested to the head, etc.
+    let mut statuses = ValidatorStatuses::new(state, spec)?;
+    statuses.process_attestations(&state, spec)?;
 
     process_eth1_data(state, spec);
 
     update_justification_and_finalization(state, &statuses.total_balances, spec)?;
 
-    // Crosslinks
+    // Crosslinks.
     let winning_root_for_shards = process_crosslinks(state, spec)?;
 
-    // Rewards and Penalities
+    // Rewards and Penalities.
     apply_rewards(state, &mut statuses, &winning_root_for_shards, spec)?;
 
-    // Ejections
+    // Ejections.
     process_ejections(state, spec)?;
 
-    // Validator Registry
+    // Validator Registry.
-    process_validator_registry(state, spec)?;
+    update_registry_and_shuffling_data(state, statuses.total_balances.current_epoch, spec)?;
+
+    // Slashings and exit queue.
     process_slashings(state, spec)?;
     process_exit_queue(state, spec);
 
-    // Final updates
+    // Final updates.
-    update_active_tree_index_roots(state, spec)?;
+    finish_epoch_update(state, spec)?;
-    update_latest_slashed_balances(state, spec)?;
-    state.previous_epoch_attestations = vec![];
 
     // Rotate the epoch caches to suit the epoch transition.
     state.advance_caches();
@@ -70,25 +73,6 @@ pub fn per_epoch_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result
     Ok(())
 }
 
-/// Calculates various sets of attesters, including:
-///
-/// - current epoch attesters
-/// - current epoch boundary attesters
-/// - previous epoch attesters
-/// - etc.
-///
-/// Spec v0.5.0
-pub fn initialize_validator_statuses(
-    state: &BeaconState,
-    spec: &ChainSpec,
-) -> Result<ValidatorStatuses, BeaconStateError> {
-    let mut statuses = ValidatorStatuses::new(state, spec)?;
-
-    statuses.process_attestations(&state, spec)?;
-
-    Ok(statuses)
-}
-
 /// Maybe resets the eth1 period.
 ///
 /// Spec v0.5.0
@@ -224,41 +208,53 @@ pub fn process_crosslinks(
     Ok(winning_root_for_shards)
 }
 
-/// Updates the state's `latest_active_index_roots` field with a tree hash the active validator
+/// Finish up an epoch update.
-/// indices for the next epoch.
 ///
-/// Spec v0.4.0
+/// Spec v0.5.0
-pub fn update_active_tree_index_roots(
+pub fn finish_epoch_update(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> {
-    state: &mut BeaconState,
-    spec: &ChainSpec,
-) -> Result<(), Error> {
-    let next_epoch = state.next_epoch(spec);
-
-    let active_tree_root = state
-        .get_active_validator_indices(next_epoch + Epoch::from(spec.activation_exit_delay))
-        .to_vec()
-        .hash_tree_root();
-
-    state.set_active_index_root(next_epoch, Hash256::from_slice(&active_tree_root[..]), spec)?;
-
-    Ok(())
-}
-
-/// Advances the state's `latest_slashed_balances` field.
-///
-/// Spec v0.4.0
-pub fn update_latest_slashed_balances(
-    state: &mut BeaconState,
-    spec: &ChainSpec,
-) -> Result<(), Error> {
     let current_epoch = state.current_epoch(spec);
     let next_epoch = state.next_epoch(spec);
 
+    // This is a hack to allow us to update index roots and slashed balances for the next epoch.
+    //
+    // The indentation here is to make it obvious where the weird stuff happens.
+    {
+        state.slot += 1;
+
+        // Set active index root
+        let active_index_root = Hash256::from_slice(
+            &state
+                .get_active_validator_indices(next_epoch + spec.activation_exit_delay)
+                .hash_tree_root()[..],
+        );
+        state.set_active_index_root(next_epoch, active_index_root, spec)?;
+
+        // Set total slashed balances
         state.set_slashed_balance(
             next_epoch,
             state.get_slashed_balance(current_epoch, spec)?,
             spec,
         )?;
 
+        // Set randao mix
+        state.set_randao_mix(
+            next_epoch,
+            *state.get_randao_mix(current_epoch, spec)?,
+            spec,
+        )?;
+
+        state.slot -= 1;
+    }
+
+    if next_epoch.as_u64() % (spec.slots_per_historical_root as u64 / spec.slots_per_epoch) == 0 {
+        let historical_batch: HistoricalBatch = state.historical_batch();
+        state
+            .historical_roots
+            .push(Hash256::from_slice(&historical_batch.hash_tree_root()[..]));
+    }
+
+    state.previous_epoch_attestations = state.current_epoch_attestations.clone();
+    state.current_epoch_attestations = vec![];
+
     Ok(())
 }

eth2/state_processing/src/per_epoch_processing/process_validator_registry.rs

@@ -1,70 +0,0 @@
-use super::update_validator_registry::update_validator_registry;
-use super::Error;
-use types::*;
-
-/// Peforms a validator registry update, if required.
-///
-/// Spec v0.4.0
-pub fn process_validator_registry(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> {
-    let current_epoch = state.current_epoch(spec);
-    let next_epoch = state.next_epoch(spec);
-
-    state.previous_shuffling_epoch = state.current_shuffling_epoch;
-    state.previous_shuffling_start_shard = state.current_shuffling_start_shard;
-
-    state.previous_shuffling_seed = state.current_shuffling_seed;
-
-    if should_update_validator_registry(state, spec)? {
-        update_validator_registry(state, spec)?;
-
-        state.current_shuffling_epoch = next_epoch;
-        state.current_shuffling_start_shard = (state.current_shuffling_start_shard
-            + spec.get_epoch_committee_count(
-                state
-                    .get_cached_active_validator_indices(RelativeEpoch::Current, spec)?
-                    .len(),
-            ) as u64)
-            % spec.shard_count;
-        state.current_shuffling_seed = state.generate_seed(state.current_shuffling_epoch, spec)?
-    } else {
-        let epochs_since_last_registry_update =
-            current_epoch - state.validator_registry_update_epoch;
-        if (epochs_since_last_registry_update > 1)
-            & epochs_since_last_registry_update.is_power_of_two()
-        {
-            state.current_shuffling_epoch = next_epoch;
-            state.current_shuffling_seed =
-                state.generate_seed(state.current_shuffling_epoch, spec)?
-        }
-    }
-
-    Ok(())
-}
-
-/// Returns `true` if the validator registry should be updated during an epoch processing.
-///
-/// Spec v0.5.0
-pub fn should_update_validator_registry(
-    state: &BeaconState,
-    spec: &ChainSpec,
-) -> Result<bool, BeaconStateError> {
-    if state.finalized_epoch <= state.validator_registry_update_epoch {
-        return Ok(false);
-    }
-
-    let num_active_validators = state
-        .get_cached_active_validator_indices(RelativeEpoch::Current, spec)?
-        .len();
-    let current_epoch_committee_count = spec.get_epoch_committee_count(num_active_validators);
-
-    for shard in (0..current_epoch_committee_count)
-        .into_iter()
-        .map(|i| (state.current_shuffling_start_shard + i as u64) % spec.shard_count)
-    {
-        if state.latest_crosslinks[shard as usize].epoch <= state.validator_registry_update_epoch {
-            return Ok(false);
-        }
-    }
-
-    Ok(true)
-}

eth2/state_processing/src/per_epoch_processing/update_registry_and_shuffling_data.rs

@@ -0,0 +1,151 @@
+use super::super::common::exit_validator;
+use super::Error;
+use types::*;
+
+/// Peforms a validator registry update, if required.
+///
+/// Spec v0.5.0
+pub fn update_registry_and_shuffling_data(
+    state: &mut BeaconState,
+    current_total_balance: u64,
+    spec: &ChainSpec,
+) -> Result<(), Error> {
+    // First set previous shuffling data to current shuffling data.
+    state.previous_shuffling_epoch = state.current_shuffling_epoch;
+    state.previous_shuffling_start_shard = state.previous_shuffling_start_shard;
+    state.previous_shuffling_seed = state.previous_shuffling_seed;
+
+    let current_epoch = state.current_epoch(spec);
+    let next_epoch = current_epoch + 1;
+
+    // Check we should update, and if so, update.
+    if should_update_validator_registry(state, spec)? {
+        update_validator_registry(state, current_total_balance, spec)?;
+
+        // If we update the registry, update the shuffling data and shards as well.
+        state.current_shuffling_epoch = next_epoch;
+        state.current_shuffling_start_shard = {
+            let active_validators =
+                state.get_cached_active_validator_indices(RelativeEpoch::Current, spec)?;
+            let epoch_committee_count = spec.get_epoch_committee_count(active_validators.len());
+
+            (state.current_shuffling_start_shard + epoch_committee_count) % spec.shard_count
+        };
+        state.current_shuffling_seed = state.generate_seed(state.current_shuffling_epoch, spec)?;
+    } else {
+        // If processing at least on crosslink keeps failing, the reshuffle every power of two, but
+        // don't update the current_shuffling_start_shard.
+        let epochs_since_last_update = current_epoch - state.validator_registry_update_epoch;
+
+        if epochs_since_last_update > 1 && epochs_since_last_update.is_power_of_two() {
+            state.current_shuffling_epoch = next_epoch;
+            state.current_shuffling_seed =
+                state.generate_seed(state.current_shuffling_epoch, spec)?;
+        }
+    }
+
+    Ok(())
+}
+
+/// Returns `true` if the validator registry should be updated during an epoch processing.
+///
+/// Spec v0.5.0
+pub fn should_update_validator_registry(
+    state: &BeaconState,
+    spec: &ChainSpec,
+) -> Result<bool, BeaconStateError> {
+    if state.finalized_epoch <= state.validator_registry_update_epoch {
+        return Ok(false);
+    }
+
+    let num_active_validators = state
+        .get_cached_active_validator_indices(RelativeEpoch::Current, spec)?
+        .len();
+    let current_epoch_committee_count = spec.get_epoch_committee_count(num_active_validators);
+
+    for shard in (0..current_epoch_committee_count)
+        .into_iter()
+        .map(|i| (state.current_shuffling_start_shard + i as u64) % spec.shard_count)
+    {
+        if state.latest_crosslinks[shard as usize].epoch <= state.validator_registry_update_epoch {
+            return Ok(false);
+        }
+    }
+
+    Ok(true)
+}
+
+/// Update validator registry, activating/exiting validators if possible.
+///
+/// Note: Utilizes the cache and will fail if the appropriate cache is not initialized.
+///
+/// Spec v0.5.0
+pub fn update_validator_registry(
+    state: &mut BeaconState,
+    current_total_balance: u64,
+    spec: &ChainSpec,
+) -> Result<(), Error> {
+    let current_epoch = state.current_epoch(spec);
+
+    let max_balance_churn = std::cmp::max(
+        spec.max_deposit_amount,
+        current_total_balance / (2 * spec.max_balance_churn_quotient),
+    );
+
+    // Activate validators within the allowable balance churn.
+    let mut balance_churn = 0;
+    for index in 0..state.validator_registry.len() {
+        let not_activated =
+            state.validator_registry[index].activation_epoch == spec.far_future_epoch;
+        let has_enough_balance = state.validator_balances[index] >= spec.max_deposit_amount;
+
+        if not_activated && has_enough_balance {
+            // Check the balance churn would be within the allowance.
+            balance_churn += state.get_effective_balance(index, spec)?;
+            if balance_churn > max_balance_churn {
+                break;
+            }
+
+            activate_validator(state, index, false, spec);
+        }
+    }
+
+    // Exit validators within the allowable balance churn.
+    let mut balance_churn = 0;
+    for index in 0..state.validator_registry.len() {
+        let not_exited = state.validator_registry[index].exit_epoch == spec.far_future_epoch;
+        let has_initiated_exit = state.validator_registry[index].initiated_exit;
+
+        if not_exited && has_initiated_exit {
+            // Check the balance churn would be within the allowance.
+            balance_churn += state.get_effective_balance(index, spec)?;
+            if balance_churn > max_balance_churn {
+                break;
+            }
+
+            exit_validator(state, index, spec)?;
+        }
+    }
+
+    state.validator_registry_update_epoch = current_epoch;
+
+    Ok(())
+}
+
+/// Activate the validator of the given ``index``.
+///
+/// Spec v0.5.0
+pub fn activate_validator(
+    state: &mut BeaconState,
+    validator_index: usize,
+    is_genesis: bool,
+    spec: &ChainSpec,
+) {
+    let current_epoch = state.current_epoch(spec);
+
+    state.validator_registry[validator_index].activation_epoch = if is_genesis {
+        spec.genesis_epoch
+    } else {
+        state.get_delayed_activation_exit_epoch(current_epoch, spec)
+    }
+}

eth2/state_processing/src/per_epoch_processing/update_validator_registry.rs

@@ -1,52 +0,0 @@
-use crate::common::exit_validator;
-use types::{BeaconStateError as Error, *};
-
-/// Update validator registry, activating/exiting validators if possible.
-///
-/// Note: Utilizes the cache and will fail if the appropriate cache is not initialized.
-///
-/// Spec v0.4.0
-pub fn update_validator_registry(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> {
-    let current_epoch = state.current_epoch(spec);
-    let active_validator_indices =
-        state.get_cached_active_validator_indices(RelativeEpoch::Current, spec)?;
-    let total_balance = state.get_total_balance(&active_validator_indices[..], spec)?;
-
-    let max_balance_churn = std::cmp::max(
-        spec.max_deposit_amount,
-        total_balance / (2 * spec.max_balance_churn_quotient),
-    );
-
-    let mut balance_churn = 0;
-    for index in 0..state.validator_registry.len() {
-        let validator = &state.validator_registry[index];
-
-        if (validator.activation_epoch == spec.far_future_epoch)
-            & (state.validator_balances[index] == spec.max_deposit_amount)
-        {
-            balance_churn += state.get_effective_balance(index, spec)?;
-            if balance_churn > max_balance_churn {
-                break;
-            }
-            state.activate_validator(index, false, spec);
-        }
-    }
-
-    let mut balance_churn = 0;
-    for index in 0..state.validator_registry.len() {
-        let validator = &state.validator_registry[index];
-
-        if (validator.exit_epoch == spec.far_future_epoch) & (validator.initiated_exit) {
-            balance_churn += state.get_effective_balance(index, spec)?;
-            if balance_churn > max_balance_churn {
-                break;
-            }
-
-            exit_validator(state, index, spec)?;
-        }
-    }
-
-    state.validator_registry_update_epoch = current_epoch;
-
-    Ok(())
-}

eth2/types/src/beacon_state.rs

@@ -193,6 +193,13 @@ impl BeaconState {
         Hash256::from_slice(&self.hash_tree_root()[..])
     }
 
+    pub fn historical_batch(&self) -> HistoricalBatch {
+        HistoricalBatch {
+            block_roots: self.latest_block_roots.clone(),
+            state_roots: self.latest_state_roots.clone(),
+        }
+    }
+
     /// If a validator pubkey exists in the validator registry, returns `Some(i)`, otherwise
     /// returns `None`.
     ///
@@ -382,6 +389,26 @@ impl BeaconState {
         Ok(self.latest_block_roots[i] = block_root)
     }
 
+    /// Safely obtains the index for `latest_randao_mixes`
+    ///
+    /// Spec v0.5.0
+    fn get_randao_mix_index(&self, epoch: Epoch, spec: &ChainSpec) -> Result<usize, Error> {
+        let current_epoch = self.current_epoch(spec);
+
+        if (current_epoch - (spec.latest_randao_mixes_length as u64) < epoch)
+            & (epoch <= current_epoch)
+        {
+            let i = epoch.as_usize() % spec.latest_randao_mixes_length;
+            if i < self.latest_randao_mixes.len() {
+                Ok(i)
+            } else {
+                Err(Error::InsufficientRandaoMixes)
+            }
+        } else {
+            Err(Error::EpochOutOfBounds)
+        }
+    }
+
     /// XOR-assigns the existing `epoch` randao mix with the hash of the `signature`.
     ///
     /// # Errors:
@@ -406,24 +433,23 @@ impl BeaconState {
 
     /// Return the randao mix at a recent ``epoch``.
     ///
-    /// # Errors:
-    /// - `InsufficientRandaoMixes` if `self.latest_randao_mixes` is shorter than
-    /// `spec.latest_randao_mixes_length`.
-    /// - `EpochOutOfBounds` if the state no longer stores randao mixes for the given `epoch`.
-    ///
     /// Spec v0.5.0
     pub fn get_randao_mix(&self, epoch: Epoch, spec: &ChainSpec) -> Result<&Hash256, Error> {
-        let current_epoch = self.current_epoch(spec);
+        let i = self.get_randao_mix_index(epoch, spec)?;
-
+        Ok(&self.latest_randao_mixes[i])
-        if (current_epoch - (spec.latest_randao_mixes_length as u64) < epoch)
-            & (epoch <= current_epoch)
-        {
-            self.latest_randao_mixes
-                .get(epoch.as_usize() % spec.latest_randao_mixes_length)
-                .ok_or_else(|| Error::InsufficientRandaoMixes)
-        } else {
-            Err(Error::EpochOutOfBounds)
     }
+
+    /// Set the randao mix at a recent ``epoch``.
+    ///
+    /// Spec v0.5.0
+    pub fn set_randao_mix(
+        &mut self,
+        epoch: Epoch,
+        mix: Hash256,
+        spec: &ChainSpec,
+    ) -> Result<(), Error> {
+        let i = self.get_randao_mix_index(epoch, spec)?;
+        Ok(self.latest_randao_mixes[i] = mix)
     }
 
     /// Safely obtains the index for `latest_active_index_roots`, given some `epoch`.
@@ -588,24 +614,6 @@ impl BeaconState {
         epoch + 1 + spec.activation_exit_delay
     }
 
-    /// Activate the validator of the given ``index``.
-    ///
-    /// Spec v0.5.0
-    pub fn activate_validator(
-        &mut self,
-        validator_index: usize,
-        is_genesis: bool,
-        spec: &ChainSpec,
-    ) {
-        let current_epoch = self.current_epoch(spec);
-
-        self.validator_registry[validator_index].activation_epoch = if is_genesis {
-            spec.genesis_epoch
-        } else {
-            self.get_delayed_activation_exit_epoch(current_epoch, spec)
-        }
-    }
-
     /// Initiate an exit for the validator of the given `index`.
     ///
     /// Spec v0.5.0