Make EpochCache store shuffling and less maps

eth2/types/src/beacon_state.rs

@@ -34,6 +34,7 @@ pub enum Error {
     UnableToDetermineProducer,
     InvalidBitfield,
     ValidatorIsWithdrawable,
+    InsufficientValidators,
     InsufficientRandaoMixes,
     InsufficientBlockRoots,
     InsufficientIndexRoots,
@@ -277,67 +278,30 @@ impl<T: EthSpec> BeaconState<T> {
         self.current_epoch() + 1
     }
 
-    /// Return the number of committees at ``epoch``.
-    ///
-    /// Spec v0.6.1
-    pub fn get_epoch_committee_count(&self, epoch: Epoch, spec: &ChainSpec) -> u64 {
-        let active_validator_indices = self.get_active_validator_indices(epoch);
-        spec.get_epoch_committee_count(active_validator_indices.len())
-    }
-
-    /// Return the number of shards to increment `state.latest_start_shard` during `epoch`.
-    ///
-    /// Spec v0.6.1
-    pub fn get_shard_delta(&self, epoch: Epoch, spec: &ChainSpec) -> u64 {
-        std::cmp::min(
-            self.get_epoch_committee_count(epoch, spec),
-            T::ShardCount::to_u64() - T::ShardCount::to_u64() / spec.slots_per_epoch,
-        )
-    }
-
-    /// Return the start shard for an epoch less than or equal to the next epoch.
-    ///
-    /// Spec v0.6.1
-    pub fn get_epoch_start_shard(&self, epoch: Epoch, spec: &ChainSpec) -> Result<u64, Error> {
-        if epoch > self.current_epoch() + 1 {
-            return Err(Error::EpochOutOfBounds);
-        }
-        let shard_count = T::ShardCount::to_u64();
-        let mut check_epoch = self.current_epoch() + 1;
-        let mut shard = (self.latest_start_shard
-            + self.get_shard_delta(self.current_epoch(), spec))
-            % shard_count;
-        while check_epoch > epoch {
-            check_epoch -= 1;
-            shard = (shard + shard_count - self.get_shard_delta(check_epoch, spec)) % shard_count;
-        }
-        Ok(shard)
-    }
-
     /// Get the slot of an attestation.
     ///
+    /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized.
+    ///
     /// Spec v0.6.1
-    pub fn get_attestation_slot(
+    pub fn get_attestation_slot(&self, attestation_data: &AttestationData) -> Result<Slot, Error> {
-        &self,
+        let cc = self
-        attestation_data: &AttestationData,
+            .get_crosslink_committee_for_shard(
-        spec: &ChainSpec,
+                attestation_data.shard,
-    ) -> Result<Slot, Error> {
+                attestation_data.target_epoch,
-        let epoch = attestation_data.target_epoch;
+            )?
-        let committee_count = self.get_epoch_committee_count(epoch, spec);
+            .ok_or_else(|| Error::NoCommitteeForShard)?;
-        let offset = (attestation_data.shard + spec.shard_count
+        Ok(cc.slot)
-            - self.get_epoch_start_shard(epoch, spec)?)
-            % spec.shard_count;
-        Ok(epoch.start_slot(spec.slots_per_epoch)
-            + offset / (committee_count / spec.slots_per_epoch))
     }
 
     /// Return the cached active validator indices at some epoch.
     ///
+    /// Note: the indices are shuffled (i.e., not in ascending order).
+    ///
     /// Returns an error if that epoch is not cached, or the cache is not initialized.
     pub fn get_cached_active_validator_indices(&self, epoch: Epoch) -> Result<&[usize], Error> {
         let cache = self.cache(epoch)?;
 
-        Ok(&cache.active_validator_indices)
+        Ok(&cache.active_validator_indices())
     }
 
     /// Returns the active validator indices for the given epoch.
@@ -371,7 +335,7 @@ impl<T: EthSpec> BeaconState<T> {
         &self,
         shard: u64,
         epoch: Epoch,
-    ) -> Result<Option<&CrosslinkCommittee>, Error> {
+    ) -> Result<Option<CrosslinkCommittee>, Error> {
         let cache = self.cache(epoch)?;
 
         Ok(cache.get_crosslink_committee_for_shard(shard))
@@ -698,9 +662,7 @@ impl<T: EthSpec> BeaconState<T> {
     pub fn get_churn_limit(&self, spec: &ChainSpec) -> Result<u64, Error> {
         Ok(std::cmp::max(
             spec.min_per_epoch_churn_limit,
-            self.cache(self.current_epoch())?
+            self.cache(self.current_epoch())?.active_validator_count() as u64
-                .active_validator_indices
-                .len() as u64
                 / spec.churn_limit_quotient,
         ))
     }

eth2/types/src/beacon_state/epoch_cache.rs

@@ -16,17 +16,14 @@ mod tests;
 #[derive(Debug, Default, PartialEq, Clone, Serialize, Deserialize)]
 pub struct EpochCache {
     /// `Some(epoch)` if the cache is initialized, where `epoch` is the cache it holds.
-    pub initialized_epoch: Option<Epoch>,
+    initialized_epoch: Option<Epoch>,
-    /// All crosslink committees.
+    shuffling_start_shard: u64,
-    pub crosslink_committees: Vec<CrosslinkCommittee>,
+    shuffling: Vec<usize>,
-    /// Maps a shard to `self.epoch_crosslink_committees`.
+    shard_count: u64,
-    pub shard_crosslink_committees: Vec<Option<usize>>,
+    committee_count: usize,
-    /// Maps a slot to `self.epoch_crosslink_committees`.
+    slots_per_epoch: u64,
-    pub slot_crosslink_committees: Vec<Option<usize>>,
     /// Maps validator index to a slot, shard and committee index for attestation.
     pub attestation_duties: Vec<Option<AttestationDuty>>,
-    /// Indices of all active validators in the epoch
-    pub active_validator_indices: Vec<usize>,
 }
 
 impl EpochCache {
@@ -36,74 +33,135 @@ impl EpochCache {
         epoch: Epoch,
         spec: &ChainSpec,
     ) -> Result<EpochCache, BeaconStateError> {
-        if (epoch < state.previous_epoch()) || (epoch > state.next_epoch()) {
+        let relative_epoch = RelativeEpoch::from_epoch(state.current_epoch(), epoch)
-            return Err(BeaconStateError::EpochOutOfBounds);
+            .map_err(|_| BeaconStateError::EpochOutOfBounds)?;
-        }
 
         let active_validator_indices =
             get_active_validator_indices(&state.validator_registry, epoch);
 
-        let epoch_committee_count = state.get_epoch_committee_count(epoch, spec);
+        if active_validator_indices.is_empty() {
-
+            return Err(BeaconStateError::InsufficientValidators);
-        let crosslink_committees = compute_epoch_committees(
-            epoch,
-            state,
-            active_validator_indices.clone(),
-            epoch_committee_count,
-            spec,
-        )?;
-
-        let mut shard_crosslink_committees = vec![None; T::shard_count()];
-        let mut slot_crosslink_committees = vec![None; spec.slots_per_epoch as usize];
-        let mut attestation_duties = vec![None; state.validator_registry.len()];
-
-        for (i, crosslink_committee) in crosslink_committees.iter().enumerate() {
-            shard_crosslink_committees[crosslink_committee.shard as usize] = Some(i);
-
-            let slot_index = epoch
-                .position(crosslink_committee.slot, spec.slots_per_epoch)
-                .ok_or_else(|| BeaconStateError::SlotOutOfBounds)?;
-            slot_crosslink_committees[slot_index] = Some(i);
-
-            // Loop through each validator in the committee and store its attestation duties.
-            for (committee_index, validator_index) in
-                crosslink_committee.committee.iter().enumerate()
-            {
-                let attestation_duty = AttestationDuty {
-                    slot: crosslink_committee.slot,
-                    shard: crosslink_committee.shard,
-                    committee_index,
-                    committee_len: crosslink_committee.committee.len(),
-                };
-                attestation_duties[*validator_index] = Some(attestation_duty);
-            }
         }
 
-        dbg!(&shard_crosslink_committees);
+        let committee_count =
+            spec.get_epoch_committee_count(active_validator_indices.len()) as usize;
 
-        Ok(EpochCache {
+        let shuffling_start_shard = match relative_epoch {
-            initialized_epoch: Some(epoch),
+            RelativeEpoch::Current => state.latest_start_shard,
-            crosslink_committees,
+            RelativeEpoch::Previous => {
-            attestation_duties,
+                let committees_in_previous_epoch =
-            shard_crosslink_committees,
+                    spec.get_epoch_committee_count(active_validator_indices.len());
-            slot_crosslink_committees,
+
+                (state.latest_start_shard + T::shard_count() as u64 - committees_in_previous_epoch)
+                    % T::shard_count() as u64
+            }
+            RelativeEpoch::Next => {
+                let current_active_validators =
+                    get_active_validator_count(&state.validator_registry, state.current_epoch());
+                let committees_in_current_epoch =
+                    spec.get_epoch_committee_count(current_active_validators);
+
+                (state.latest_start_shard + committees_in_current_epoch) % T::shard_count as u64
+            }
+        };
+
+        let seed = state.generate_seed(epoch, spec)?;
+
+        let shuffling = shuffle_list(
             active_validator_indices,
-        })
+            spec.shuffle_round_count,
+            &seed[..],
+            false,
+        )
+        .ok_or_else(|| Error::UnableToShuffle)?;
+
+        let mut cache = EpochCache {
+            initialized_epoch: Some(epoch),
+            shuffling_start_shard,
+            shuffling,
+            shard_count: T::shard_count() as u64,
+            committee_count,
+            slots_per_epoch: T::slots_per_epoch(),
+            attestation_duties: vec![None; state.validator_registry.len()],
+        };
+
+        cache.build_attestation_duties();
+
+        Ok(cache)
+    }
+
+    fn build_attestation_duties(&mut self) {
+        for (i, committee) in self
+            .shuffling
+            .honey_badger_split(self.committee_count)
+            .enumerate()
+        {
+            let shard = (self.shuffling_start_shard + i as u64) % self.shard_count;
+
+            let slot = self.crosslink_slot_for_shard(shard).unwrap();
+
+            for (committee_index, validator_index) in committee.iter().enumerate() {
+                self.attestation_duties[*validator_index] = Some(AttestationDuty {
+                    slot,
+                    shard,
+                    committee_index,
+                    committee_len: committee.len(),
+                });
+            }
+        }
     }
 
     pub fn is_initialized_at(&self, epoch: Epoch) -> bool {
         Some(epoch) == self.initialized_epoch
     }
 
+    pub fn active_validator_indices(&self) -> &[usize] {
+        &self.shuffling
+    }
+
     /// Return `Some(CrosslinkCommittee)` if the given shard has a committee during the given
     /// `epoch`.
-    pub fn get_crosslink_committee_for_shard(&self, shard: Shard) -> Option<&CrosslinkCommittee> {
+    pub fn get_crosslink_committee_for_shard(&self, shard: Shard) -> Option<CrosslinkCommittee> {
-        if shard > self.shard_crosslink_committees.len() as u64 {
+        if shard >= self.shard_count || self.initialized_epoch.is_none() {
-            None
+            return None;
-        } else {
-            let i = self.shard_crosslink_committees[shard as usize]?;
-            Some(&self.crosslink_committees[i])
         }
+
+        let committee_index =
+            (shard + self.shard_count - self.shuffling_start_shard) % self.shard_count;
+        let committee = self.compute_committee(committee_index as usize, self.committee_count)?;
+        let slot = self.crosslink_slot_for_shard(shard)?;
+
+        Some(CrosslinkCommittee {
+            shard,
+            committee,
+            slot,
+        })
+    }
+
+    pub fn active_validator_count(&self) -> usize {
+        self.shuffling.len()
+    }
+
+    fn compute_committee(&self, index: usize, count: usize) -> Option<&[usize]> {
+        if self.initialized_epoch.is_none() {
+            return None;
+        }
+
+        let num_validators = self.shuffling.len();
+
+        // Note: `count != 0` is enforced in the constructor.
+        let start = (num_validators * index) / count;
+        let end = (num_validators * (index + 1)) / count;
+
+        Some(&self.shuffling[start..end])
+    }
+
+    fn crosslink_slot_for_shard(&self, shard: u64) -> Option<Slot> {
+        let offset = (shard + self.shard_count - self.shuffling_start_shard) % self.shard_count;
+        Some(
+            self.initialized_epoch?.start_slot(self.slots_per_epoch)
+                + offset / (self.committee_count as u64 / self.slots_per_epoch),
+        )
     }
 }
 
@@ -125,64 +183,6 @@ pub fn get_active_validator_indices(validators: &[Validator], epoch: Epoch) -> V
     active
 }
 
-pub fn compute_epoch_committees<T: EthSpec>(
+pub fn get_active_validator_count(validators: &[Validator], epoch: Epoch) -> usize {
-    epoch: Epoch,
+    validators.iter().filter(|v| v.is_active_at(epoch)).count()
-    state: &BeaconState<T>,
-    active_validator_indices: Vec<usize>,
-    epoch_committee_count: u64,
-    spec: &ChainSpec,
-) -> Result<Vec<CrosslinkCommittee>, BeaconStateError> {
-    let seed = state.generate_seed(epoch, spec)?;
-
-    // The shuffler fails on a empty list, so if there are no active validator indices, simply
-    // return an empty list.
-    let shuffled_active_validator_indices = if active_validator_indices.is_empty() {
-        vec![]
-    } else {
-        shuffle_list(
-            active_validator_indices,
-            spec.shuffle_round_count,
-            &seed[..],
-            false,
-        )
-        .ok_or_else(|| Error::UnableToShuffle)?
-    };
-
-    let epoch_start_shard = state.get_epoch_start_shard(epoch, spec)?;
-
-    Ok(shuffled_active_validator_indices
-        .honey_badger_split(epoch_committee_count as usize)
-        .enumerate()
-        .map(|(index, committee)| {
-            let shard = (epoch_start_shard + index as u64) % spec.shard_count;
-
-            dbg!(index);
-            dbg!(shard);
-
-            let slot = crosslink_committee_slot(
-                shard,
-                epoch,
-                epoch_start_shard,
-                epoch_committee_count,
-                spec,
-            );
-            CrosslinkCommittee {
-                slot,
-                shard,
-                committee: committee.to_vec(),
-            }
-        })
-        .collect())
-}
-
-fn crosslink_committee_slot(
-    shard: u64,
-    epoch: Epoch,
-    epoch_start_shard: u64,
-    epoch_committee_count: u64,
-    spec: &ChainSpec,
-) -> Slot {
-    // Excerpt from `get_attestation_slot` in the spec.
-    let offset = (shard + spec.shard_count - epoch_start_shard) % spec.shard_count;
-    epoch.start_slot(spec.slots_per_epoch) + offset / (epoch_committee_count / spec.slots_per_epoch)
 }

eth2/types/src/beacon_state/epoch_cache/tests.rs

@@ -13,11 +13,15 @@ fn execute_sane_cache_test<T: EthSpec>(
 ) {
     let active_indices: Vec<usize> = (0..validator_count).collect();
     let seed = state.generate_seed(epoch, spec).unwrap();
-    let start_shard = state.get_epoch_start_shard(epoch, spec).unwrap();
+    let start_shard = 0;
 
+    let mut ordered_indices = state
+        .get_cached_active_validator_indices(epoch)
+        .unwrap()
+        .to_vec();
+    ordered_indices.sort_unstable();
     assert_eq!(
-        &active_indices[..],
+        active_indices, ordered_indices,
-        state.get_cached_active_validator_indices(epoch).unwrap(),
         "Validator indices mismatch"
     );
 
@@ -29,15 +33,12 @@ fn execute_sane_cache_test<T: EthSpec>(
     for i in 0..T::shard_count() {
         let shard = (i + start_shard as usize) % T::shard_count();
 
-        dbg!(shard);
-        dbg!(start_shard);
-
         let c = state
             .get_crosslink_committee_for_shard(shard as u64, epoch)
             .unwrap()
             .unwrap();
 
-        for &i in &c.committee {
+        for &i in c.committee {
             assert_eq!(
                 i,
                 *expected_indices_iter.next().unwrap(),

eth2/types/src/crosslink_committee.rs

@@ -1,21 +1,25 @@
 use crate::*;
-use serde_derive::{Deserialize, Serialize};
-use ssz_derive::{Decode, Encode};
 use tree_hash_derive::{CachedTreeHash, TreeHash};
 
-#[derive(
+#[derive(Default, Clone, Debug, PartialEq, TreeHash, CachedTreeHash)]
-    Default,
+pub struct CrosslinkCommittee<'a> {
-    Clone,
+    pub slot: Slot,
-    Debug,
+    pub shard: Shard,
-    PartialEq,
+    pub committee: &'a [usize],
-    Serialize,
+}
-    Deserialize,
+
-    Decode,
+impl<'a> CrosslinkCommittee<'a> {
-    Encode,
+    pub fn into_owned(self) -> OwnedCrosslinkCommittee {
-    TreeHash,
+        OwnedCrosslinkCommittee {
-    CachedTreeHash,
+            slot: self.slot,
-)]
+            shard: self.shard,
-pub struct CrosslinkCommittee {
+            committee: self.committee.to_vec(),
+        }
+    }
+}
+
+#[derive(Default, Clone, Debug, PartialEq, TreeHash, CachedTreeHash)]
+pub struct OwnedCrosslinkCommittee {
     pub slot: Slot,
     pub shard: Shard,
     pub committee: Vec<usize>,

eth2/types/src/lib.rs

@@ -47,7 +47,7 @@ pub use crate::beacon_block_header::BeaconBlockHeader;
 pub use crate::beacon_state::{Error as BeaconStateError, *};
 pub use crate::chain_spec::{ChainSpec, Domain};
 pub use crate::crosslink::Crosslink;
-pub use crate::crosslink_committee::CrosslinkCommittee;
+pub use crate::crosslink_committee::{CrosslinkCommittee, OwnedCrosslinkCommittee};
 pub use crate::deposit::Deposit;
 pub use crate::deposit_data::DepositData;
 pub use crate::eth1_data::Eth1Data;

eth2/types/src/test_utils/builders/testing_beacon_block_builder.rs

@@ -116,8 +116,8 @@ impl TestingBeaconBlockBuilder {
 
                 committees.push((
                     slot,
-                    crosslink_committee.committee.clone(),
+                    crosslink_committee.committee.to_vec(),
-                    crosslink_committee.committee.clone(),
+                    crosslink_committee.committee.to_vec(),
                     crosslink_committee.shard,
                 ));
 

eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs

@@ -166,7 +166,7 @@ impl<T: EthSpec> TestingBeaconStateBuilder<T> {
     /// Note: this performs the build when called. Ensure that no changes are made that would
     /// invalidate this cache.
     pub fn build_caches(&mut self, spec: &ChainSpec) -> Result<(), BeaconStateError> {
-        self.state.build_all_caches(spec);
+        self.state.build_all_caches(spec).unwrap();
 
         Ok(())
     }
@@ -222,10 +222,12 @@ impl<T: EthSpec> TestingBeaconStateBuilder<T> {
         for slot in first_slot..=last_slot {
             let slot = Slot::from(slot);
 
-            let committees = state
+            let committees: Vec<OwnedCrosslinkCommittee> = state
                 .get_crosslink_committees_at_slot(slot, spec)
                 .unwrap()
-                .clone();
+                .into_iter()
+                .map(|c| c.clone().into_owned())
+                .collect();
 
             for crosslink_committee in committees {
                 let mut builder = TestingPendingAttestationBuilder::new(