From 9eb8c7411f807f4e90c62e25f314accbd4a7cdd4 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 19 May 2019 15:41:22 +1000 Subject: [PATCH] Make `EpochCache` store `shuffling` and less maps --- eth2/types/src/beacon_state.rs | 70 ++--- eth2/types/src/beacon_state/epoch_cache.rs | 240 +++++++++--------- .../src/beacon_state/epoch_cache/tests.rs | 15 +- eth2/types/src/crosslink_committee.rs | 34 +-- eth2/types/src/lib.rs | 2 +- .../builders/testing_beacon_block_builder.rs | 4 +- .../builders/testing_beacon_state_builder.rs | 8 +- 7 files changed, 171 insertions(+), 202 deletions(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index a21efa67f..756e02d72 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -34,6 +34,7 @@ pub enum Error { UnableToDetermineProducer, InvalidBitfield, ValidatorIsWithdrawable, + InsufficientValidators, InsufficientRandaoMixes, InsufficientBlockRoots, InsufficientIndexRoots, @@ -277,67 +278,30 @@ impl BeaconState { self.current_epoch() + 1 } - /// Return the number of committees at ``epoch``. - /// - /// Spec v0.6.1 - pub fn get_epoch_committee_count(&self, epoch: Epoch, spec: &ChainSpec) -> u64 { - let active_validator_indices = self.get_active_validator_indices(epoch); - spec.get_epoch_committee_count(active_validator_indices.len()) - } - - /// Return the number of shards to increment `state.latest_start_shard` during `epoch`. - /// - /// Spec v0.6.1 - pub fn get_shard_delta(&self, epoch: Epoch, spec: &ChainSpec) -> u64 { - std::cmp::min( - self.get_epoch_committee_count(epoch, spec), - T::ShardCount::to_u64() - T::ShardCount::to_u64() / spec.slots_per_epoch, - ) - } - - /// Return the start shard for an epoch less than or equal to the next epoch. - /// - /// Spec v0.6.1 - pub fn get_epoch_start_shard(&self, epoch: Epoch, spec: &ChainSpec) -> Result { - if epoch > self.current_epoch() + 1 { - return Err(Error::EpochOutOfBounds); - } - let shard_count = T::ShardCount::to_u64(); - let mut check_epoch = self.current_epoch() + 1; - let mut shard = (self.latest_start_shard - + self.get_shard_delta(self.current_epoch(), spec)) - % shard_count; - while check_epoch > epoch { - check_epoch -= 1; - shard = (shard + shard_count - self.get_shard_delta(check_epoch, spec)) % shard_count; - } - Ok(shard) - } - /// Get the slot of an attestation. /// + /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. + /// /// Spec v0.6.1 - pub fn get_attestation_slot( - &self, - attestation_data: &AttestationData, - spec: &ChainSpec, - ) -> Result { - let epoch = attestation_data.target_epoch; - let committee_count = self.get_epoch_committee_count(epoch, spec); - let offset = (attestation_data.shard + spec.shard_count - - self.get_epoch_start_shard(epoch, spec)?) - % spec.shard_count; - Ok(epoch.start_slot(spec.slots_per_epoch) - + offset / (committee_count / spec.slots_per_epoch)) + pub fn get_attestation_slot(&self, attestation_data: &AttestationData) -> Result { + let cc = self + .get_crosslink_committee_for_shard( + attestation_data.shard, + attestation_data.target_epoch, + )? + .ok_or_else(|| Error::NoCommitteeForShard)?; + Ok(cc.slot) } /// Return the cached active validator indices at some epoch. /// + /// Note: the indices are shuffled (i.e., not in ascending order). + /// /// Returns an error if that epoch is not cached, or the cache is not initialized. pub fn get_cached_active_validator_indices(&self, epoch: Epoch) -> Result<&[usize], Error> { let cache = self.cache(epoch)?; - Ok(&cache.active_validator_indices) + Ok(&cache.active_validator_indices()) } /// Returns the active validator indices for the given epoch. @@ -371,7 +335,7 @@ impl BeaconState { &self, shard: u64, epoch: Epoch, - ) -> Result, Error> { + ) -> Result, Error> { let cache = self.cache(epoch)?; Ok(cache.get_crosslink_committee_for_shard(shard)) @@ -698,9 +662,7 @@ impl BeaconState { pub fn get_churn_limit(&self, spec: &ChainSpec) -> Result { Ok(std::cmp::max( spec.min_per_epoch_churn_limit, - self.cache(self.current_epoch())? - .active_validator_indices - .len() as u64 + self.cache(self.current_epoch())?.active_validator_count() as u64 / spec.churn_limit_quotient, )) } diff --git a/eth2/types/src/beacon_state/epoch_cache.rs b/eth2/types/src/beacon_state/epoch_cache.rs index 386b6c920..33ba56071 100644 --- a/eth2/types/src/beacon_state/epoch_cache.rs +++ b/eth2/types/src/beacon_state/epoch_cache.rs @@ -16,17 +16,14 @@ mod tests; #[derive(Debug, Default, PartialEq, Clone, Serialize, Deserialize)] pub struct EpochCache { /// `Some(epoch)` if the cache is initialized, where `epoch` is the cache it holds. - pub initialized_epoch: Option, - /// All crosslink committees. - pub crosslink_committees: Vec, - /// Maps a shard to `self.epoch_crosslink_committees`. - pub shard_crosslink_committees: Vec>, - /// Maps a slot to `self.epoch_crosslink_committees`. - pub slot_crosslink_committees: Vec>, + initialized_epoch: Option, + shuffling_start_shard: u64, + shuffling: Vec, + shard_count: u64, + committee_count: usize, + slots_per_epoch: u64, /// Maps validator index to a slot, shard and committee index for attestation. pub attestation_duties: Vec>, - /// Indices of all active validators in the epoch - pub active_validator_indices: Vec, } impl EpochCache { @@ -36,74 +33,135 @@ impl EpochCache { epoch: Epoch, spec: &ChainSpec, ) -> Result { - if (epoch < state.previous_epoch()) || (epoch > state.next_epoch()) { - return Err(BeaconStateError::EpochOutOfBounds); - } + let relative_epoch = RelativeEpoch::from_epoch(state.current_epoch(), epoch) + .map_err(|_| BeaconStateError::EpochOutOfBounds)?; let active_validator_indices = get_active_validator_indices(&state.validator_registry, epoch); - let epoch_committee_count = state.get_epoch_committee_count(epoch, spec); - - let crosslink_committees = compute_epoch_committees( - epoch, - state, - active_validator_indices.clone(), - epoch_committee_count, - spec, - )?; - - let mut shard_crosslink_committees = vec![None; T::shard_count()]; - let mut slot_crosslink_committees = vec![None; spec.slots_per_epoch as usize]; - let mut attestation_duties = vec![None; state.validator_registry.len()]; - - for (i, crosslink_committee) in crosslink_committees.iter().enumerate() { - shard_crosslink_committees[crosslink_committee.shard as usize] = Some(i); - - let slot_index = epoch - .position(crosslink_committee.slot, spec.slots_per_epoch) - .ok_or_else(|| BeaconStateError::SlotOutOfBounds)?; - slot_crosslink_committees[slot_index] = Some(i); - - // Loop through each validator in the committee and store its attestation duties. - for (committee_index, validator_index) in - crosslink_committee.committee.iter().enumerate() - { - let attestation_duty = AttestationDuty { - slot: crosslink_committee.slot, - shard: crosslink_committee.shard, - committee_index, - committee_len: crosslink_committee.committee.len(), - }; - attestation_duties[*validator_index] = Some(attestation_duty); - } + if active_validator_indices.is_empty() { + return Err(BeaconStateError::InsufficientValidators); } - dbg!(&shard_crosslink_committees); + let committee_count = + spec.get_epoch_committee_count(active_validator_indices.len()) as usize; - Ok(EpochCache { - initialized_epoch: Some(epoch), - crosslink_committees, - attestation_duties, - shard_crosslink_committees, - slot_crosslink_committees, + let shuffling_start_shard = match relative_epoch { + RelativeEpoch::Current => state.latest_start_shard, + RelativeEpoch::Previous => { + let committees_in_previous_epoch = + spec.get_epoch_committee_count(active_validator_indices.len()); + + (state.latest_start_shard + T::shard_count() as u64 - committees_in_previous_epoch) + % T::shard_count() as u64 + } + RelativeEpoch::Next => { + let current_active_validators = + get_active_validator_count(&state.validator_registry, state.current_epoch()); + let committees_in_current_epoch = + spec.get_epoch_committee_count(current_active_validators); + + (state.latest_start_shard + committees_in_current_epoch) % T::shard_count as u64 + } + }; + + let seed = state.generate_seed(epoch, spec)?; + + let shuffling = shuffle_list( active_validator_indices, - }) + spec.shuffle_round_count, + &seed[..], + false, + ) + .ok_or_else(|| Error::UnableToShuffle)?; + + let mut cache = EpochCache { + initialized_epoch: Some(epoch), + shuffling_start_shard, + shuffling, + shard_count: T::shard_count() as u64, + committee_count, + slots_per_epoch: T::slots_per_epoch(), + attestation_duties: vec![None; state.validator_registry.len()], + }; + + cache.build_attestation_duties(); + + Ok(cache) + } + + fn build_attestation_duties(&mut self) { + for (i, committee) in self + .shuffling + .honey_badger_split(self.committee_count) + .enumerate() + { + let shard = (self.shuffling_start_shard + i as u64) % self.shard_count; + + let slot = self.crosslink_slot_for_shard(shard).unwrap(); + + for (committee_index, validator_index) in committee.iter().enumerate() { + self.attestation_duties[*validator_index] = Some(AttestationDuty { + slot, + shard, + committee_index, + committee_len: committee.len(), + }); + } + } } pub fn is_initialized_at(&self, epoch: Epoch) -> bool { Some(epoch) == self.initialized_epoch } + pub fn active_validator_indices(&self) -> &[usize] { + &self.shuffling + } + /// Return `Some(CrosslinkCommittee)` if the given shard has a committee during the given /// `epoch`. - pub fn get_crosslink_committee_for_shard(&self, shard: Shard) -> Option<&CrosslinkCommittee> { - if shard > self.shard_crosslink_committees.len() as u64 { - None - } else { - let i = self.shard_crosslink_committees[shard as usize]?; - Some(&self.crosslink_committees[i]) + pub fn get_crosslink_committee_for_shard(&self, shard: Shard) -> Option { + if shard >= self.shard_count || self.initialized_epoch.is_none() { + return None; } + + let committee_index = + (shard + self.shard_count - self.shuffling_start_shard) % self.shard_count; + let committee = self.compute_committee(committee_index as usize, self.committee_count)?; + let slot = self.crosslink_slot_for_shard(shard)?; + + Some(CrosslinkCommittee { + shard, + committee, + slot, + }) + } + + pub fn active_validator_count(&self) -> usize { + self.shuffling.len() + } + + fn compute_committee(&self, index: usize, count: usize) -> Option<&[usize]> { + if self.initialized_epoch.is_none() { + return None; + } + + let num_validators = self.shuffling.len(); + + // Note: `count != 0` is enforced in the constructor. + let start = (num_validators * index) / count; + let end = (num_validators * (index + 1)) / count; + + Some(&self.shuffling[start..end]) + } + + fn crosslink_slot_for_shard(&self, shard: u64) -> Option { + let offset = (shard + self.shard_count - self.shuffling_start_shard) % self.shard_count; + Some( + self.initialized_epoch?.start_slot(self.slots_per_epoch) + + offset / (self.committee_count as u64 / self.slots_per_epoch), + ) } } @@ -125,64 +183,6 @@ pub fn get_active_validator_indices(validators: &[Validator], epoch: Epoch) -> V active } -pub fn compute_epoch_committees( - epoch: Epoch, - state: &BeaconState, - active_validator_indices: Vec, - epoch_committee_count: u64, - spec: &ChainSpec, -) -> Result, BeaconStateError> { - let seed = state.generate_seed(epoch, spec)?; - - // The shuffler fails on a empty list, so if there are no active validator indices, simply - // return an empty list. - let shuffled_active_validator_indices = if active_validator_indices.is_empty() { - vec![] - } else { - shuffle_list( - active_validator_indices, - spec.shuffle_round_count, - &seed[..], - false, - ) - .ok_or_else(|| Error::UnableToShuffle)? - }; - - let epoch_start_shard = state.get_epoch_start_shard(epoch, spec)?; - - Ok(shuffled_active_validator_indices - .honey_badger_split(epoch_committee_count as usize) - .enumerate() - .map(|(index, committee)| { - let shard = (epoch_start_shard + index as u64) % spec.shard_count; - - dbg!(index); - dbg!(shard); - - let slot = crosslink_committee_slot( - shard, - epoch, - epoch_start_shard, - epoch_committee_count, - spec, - ); - CrosslinkCommittee { - slot, - shard, - committee: committee.to_vec(), - } - }) - .collect()) -} - -fn crosslink_committee_slot( - shard: u64, - epoch: Epoch, - epoch_start_shard: u64, - epoch_committee_count: u64, - spec: &ChainSpec, -) -> Slot { - // Excerpt from `get_attestation_slot` in the spec. - let offset = (shard + spec.shard_count - epoch_start_shard) % spec.shard_count; - epoch.start_slot(spec.slots_per_epoch) + offset / (epoch_committee_count / spec.slots_per_epoch) +pub fn get_active_validator_count(validators: &[Validator], epoch: Epoch) -> usize { + validators.iter().filter(|v| v.is_active_at(epoch)).count() } diff --git a/eth2/types/src/beacon_state/epoch_cache/tests.rs b/eth2/types/src/beacon_state/epoch_cache/tests.rs index 9b7a34f15..7b218dd86 100644 --- a/eth2/types/src/beacon_state/epoch_cache/tests.rs +++ b/eth2/types/src/beacon_state/epoch_cache/tests.rs @@ -13,11 +13,15 @@ fn execute_sane_cache_test( ) { let active_indices: Vec = (0..validator_count).collect(); let seed = state.generate_seed(epoch, spec).unwrap(); - let start_shard = state.get_epoch_start_shard(epoch, spec).unwrap(); + let start_shard = 0; + let mut ordered_indices = state + .get_cached_active_validator_indices(epoch) + .unwrap() + .to_vec(); + ordered_indices.sort_unstable(); assert_eq!( - &active_indices[..], - state.get_cached_active_validator_indices(epoch).unwrap(), + active_indices, ordered_indices, "Validator indices mismatch" ); @@ -29,15 +33,12 @@ fn execute_sane_cache_test( for i in 0..T::shard_count() { let shard = (i + start_shard as usize) % T::shard_count(); - dbg!(shard); - dbg!(start_shard); - let c = state .get_crosslink_committee_for_shard(shard as u64, epoch) .unwrap() .unwrap(); - for &i in &c.committee { + for &i in c.committee { assert_eq!( i, *expected_indices_iter.next().unwrap(), diff --git a/eth2/types/src/crosslink_committee.rs b/eth2/types/src/crosslink_committee.rs index 25c42c07b..188d56255 100644 --- a/eth2/types/src/crosslink_committee.rs +++ b/eth2/types/src/crosslink_committee.rs @@ -1,21 +1,25 @@ use crate::*; -use serde_derive::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode}; use tree_hash_derive::{CachedTreeHash, TreeHash}; -#[derive( - Default, - Clone, - Debug, - PartialEq, - Serialize, - Deserialize, - Decode, - Encode, - TreeHash, - CachedTreeHash, -)] -pub struct CrosslinkCommittee { +#[derive(Default, Clone, Debug, PartialEq, TreeHash, CachedTreeHash)] +pub struct CrosslinkCommittee<'a> { + pub slot: Slot, + pub shard: Shard, + pub committee: &'a [usize], +} + +impl<'a> CrosslinkCommittee<'a> { + pub fn into_owned(self) -> OwnedCrosslinkCommittee { + OwnedCrosslinkCommittee { + slot: self.slot, + shard: self.shard, + committee: self.committee.to_vec(), + } + } +} + +#[derive(Default, Clone, Debug, PartialEq, TreeHash, CachedTreeHash)] +pub struct OwnedCrosslinkCommittee { pub slot: Slot, pub shard: Shard, pub committee: Vec, diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index bb40106d1..e2225d638 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -47,7 +47,7 @@ pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_state::{Error as BeaconStateError, *}; pub use crate::chain_spec::{ChainSpec, Domain}; pub use crate::crosslink::Crosslink; -pub use crate::crosslink_committee::CrosslinkCommittee; +pub use crate::crosslink_committee::{CrosslinkCommittee, OwnedCrosslinkCommittee}; pub use crate::deposit::Deposit; pub use crate::deposit_data::DepositData; pub use crate::eth1_data::Eth1Data; diff --git a/eth2/types/src/test_utils/builders/testing_beacon_block_builder.rs b/eth2/types/src/test_utils/builders/testing_beacon_block_builder.rs index 4e2ab57b4..089b97540 100644 --- a/eth2/types/src/test_utils/builders/testing_beacon_block_builder.rs +++ b/eth2/types/src/test_utils/builders/testing_beacon_block_builder.rs @@ -116,8 +116,8 @@ impl TestingBeaconBlockBuilder { committees.push(( slot, - crosslink_committee.committee.clone(), - crosslink_committee.committee.clone(), + crosslink_committee.committee.to_vec(), + crosslink_committee.committee.to_vec(), crosslink_committee.shard, )); diff --git a/eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs b/eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs index 68da6d37f..25bdb6164 100644 --- a/eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs +++ b/eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs @@ -166,7 +166,7 @@ impl TestingBeaconStateBuilder { /// Note: this performs the build when called. Ensure that no changes are made that would /// invalidate this cache. pub fn build_caches(&mut self, spec: &ChainSpec) -> Result<(), BeaconStateError> { - self.state.build_all_caches(spec); + self.state.build_all_caches(spec).unwrap(); Ok(()) } @@ -222,10 +222,12 @@ impl TestingBeaconStateBuilder { for slot in first_slot..=last_slot { let slot = Slot::from(slot); - let committees = state + let committees: Vec = state .get_crosslink_committees_at_slot(slot, spec) .unwrap() - .clone(); + .into_iter() + .map(|c| c.clone().into_owned()) + .collect(); for crosslink_committee in committees { let mut builder = TestingPendingAttestationBuilder::new(