cerc-io/lighthouse
10
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Make EpochCache store shuffling and less maps

Browse Source
This commit is contained in:
Paul Hauner 2019-05-19 15:41:22 +10:00
parent 089febb944
commit 9eb8c7411f
No known key found for this signature in database
GPG Key ID: 5E2CFF9B75FA63DF
7 changed files with 171 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
eth2/types/src/beacon_state.rs
View File

@ -34,6 +34,7 @@ pub enum Error {
UnableToDetermineProducer,
InvalidBitfield,
ValidatorIsWithdrawable,
InsufficientValidators,
InsufficientRandaoMixes,
InsufficientBlockRoots,
InsufficientIndexRoots,
@ -277,67 +278,30 @@ impl<T: EthSpec> BeaconState<T> {
self.current_epoch() + 1
}
/// Return the number of committees at ``epoch``.
///
/// Spec v0.6.1
pub fn get_epoch_committee_count(&self, epoch: Epoch, spec: &ChainSpec) -> u64 {
let active_validator_indices = self.get_active_validator_indices(epoch);
spec.get_epoch_committee_count(active_validator_indices.len())
}
/// Return the number of shards to increment `state.latest_start_shard` during `epoch`.
///
/// Spec v0.6.1
pub fn get_shard_delta(&self, epoch: Epoch, spec: &ChainSpec) -> u64 {
std::cmp::min(
self.get_epoch_committee_count(epoch, spec),
T::ShardCount::to_u64() - T::ShardCount::to_u64() / spec.slots_per_epoch,
)
}
/// Return the start shard for an epoch less than or equal to the next epoch.
///
/// Spec v0.6.1
pub fn get_epoch_start_shard(&self, epoch: Epoch, spec: &ChainSpec) -> Result<u64, Error> {
if epoch > self.current_epoch() + 1 {
return Err(Error::EpochOutOfBounds);
}
let shard_count = T::ShardCount::to_u64();
let mut check_epoch = self.current_epoch() + 1;
let mut shard = (self.latest_start_shard
+ self.get_shard_delta(self.current_epoch(), spec))
% shard_count;
while check_epoch > epoch {
check_epoch -= 1;
shard = (shard + shard_count - self.get_shard_delta(check_epoch, spec)) % shard_count;
}
Ok(shard)
}
/// Get the slot of an attestation.
///
/// Note: Utilizes the cache and will fail if the appropriate cache is not initialized.
///
/// Spec v0.6.1
pub fn get_attestation_slot(
&self,
attestation_data: &AttestationData,
spec: &ChainSpec,
) -> Result<Slot, Error> {
let epoch = attestation_data.target_epoch;
let committee_count = self.get_epoch_committee_count(epoch, spec);
let offset = (attestation_data.shard + spec.shard_count
- self.get_epoch_start_shard(epoch, spec)?)
% spec.shard_count;
Ok(epoch.start_slot(spec.slots_per_epoch)
+ offset / (committee_count / spec.slots_per_epoch))
pub fn get_attestation_slot(&self, attestation_data: &AttestationData) -> Result<Slot, Error> {
let cc = self
.get_crosslink_committee_for_shard(
attestation_data.shard,
attestation_data.target_epoch,
)?
.ok_or_else(|| Error::NoCommitteeForShard)?;
Ok(cc.slot)
}
/// Return the cached active validator indices at some epoch.
///
/// Note: the indices are shuffled (i.e., not in ascending order).
///
/// Returns an error if that epoch is not cached, or the cache is not initialized.
pub fn get_cached_active_validator_indices(&self, epoch: Epoch) -> Result<&[usize], Error> {
let cache = self.cache(epoch)?;
Ok(&cache.active_validator_indices)
Ok(&cache.active_validator_indices())
}
/// Returns the active validator indices for the given epoch.
@ -371,7 +335,7 @@ impl<T: EthSpec> BeaconState<T> {
&self,
shard: u64,
epoch: Epoch,
) -> Result<Option<&CrosslinkCommittee>, Error> {
) -> Result<Option<CrosslinkCommittee>, Error> {
let cache = self.cache(epoch)?;
Ok(cache.get_crosslink_committee_for_shard(shard))
@ -698,9 +662,7 @@ impl<T: EthSpec> BeaconState<T> {
pub fn get_churn_limit(&self, spec: &ChainSpec) -> Result<u64, Error> {
Ok(std::cmp::max(
spec.min_per_epoch_churn_limit,
self.cache(self.current_epoch())?
.active_validator_indices
.len() as u64
self.cache(self.current_epoch())?.active_validator_count() as u64
/ spec.churn_limit_quotient,
))
}

232
eth2/types/src/beacon_state/epoch_cache.rs
View File

@ -16,17 +16,14 @@ mod tests;
#[derive(Debug, Default, PartialEq, Clone, Serialize, Deserialize)]
pub struct EpochCache {
/// `Some(epoch)` if the cache is initialized, where `epoch` is the cache it holds.
pub initialized_epoch: Option<Epoch>,
/// All crosslink committees.
pub crosslink_committees: Vec<CrosslinkCommittee>,
/// Maps a shard to `self.epoch_crosslink_committees`.
pub shard_crosslink_committees: Vec<Option<usize>>,
/// Maps a slot to `self.epoch_crosslink_committees`.
pub slot_crosslink_committees: Vec<Option<usize>>,
initialized_epoch: Option<Epoch>,
shuffling_start_shard: u64,
shuffling: Vec<usize>,
shard_count: u64,
committee_count: usize,
slots_per_epoch: u64,
/// Maps validator index to a slot, shard and committee index for attestation.
pub attestation_duties: Vec<Option<AttestationDuty>>,
/// Indices of all active validators in the epoch
pub active_validator_indices: Vec<usize>,
}
impl EpochCache {
@ -36,74 +33,135 @@ impl EpochCache {
epoch: Epoch,
spec: &ChainSpec,
) -> Result<EpochCache, BeaconStateError> {
if (epoch < state.previous_epoch()) || (epoch > state.next_epoch()) {
return Err(BeaconStateError::EpochOutOfBounds);
}
let relative_epoch = RelativeEpoch::from_epoch(state.current_epoch(), epoch)
.map_err(|_| BeaconStateError::EpochOutOfBounds)?;
let active_validator_indices =
get_active_validator_indices(&state.validator_registry, epoch);
let epoch_committee_count = state.get_epoch_committee_count(epoch, spec);
if active_validator_indices.is_empty() {
return Err(BeaconStateError::InsufficientValidators);
}
let crosslink_committees = compute_epoch_committees(
epoch,
state,
active_validator_indices.clone(),
epoch_committee_count,
spec,
)?;
let committee_count =
spec.get_epoch_committee_count(active_validator_indices.len()) as usize;
let mut shard_crosslink_committees = vec![None; T::shard_count()];
let mut slot_crosslink_committees = vec![None; spec.slots_per_epoch as usize];
let mut attestation_duties = vec![None; state.validator_registry.len()];
let shuffling_start_shard = match relative_epoch {
RelativeEpoch::Current => state.latest_start_shard,
RelativeEpoch::Previous => {
let committees_in_previous_epoch =
spec.get_epoch_committee_count(active_validator_indices.len());
for (i, crosslink_committee) in crosslink_committees.iter().enumerate() {
shard_crosslink_committees[crosslink_committee.shard as usize] = Some(i);
(state.latest_start_shard + T::shard_count() as u64 - committees_in_previous_epoch)
% T::shard_count() as u64
}
RelativeEpoch::Next => {
let current_active_validators =
get_active_validator_count(&state.validator_registry, state.current_epoch());
let committees_in_current_epoch =
spec.get_epoch_committee_count(current_active_validators);
let slot_index = epoch
.position(crosslink_committee.slot, spec.slots_per_epoch)
.ok_or_else(|| BeaconStateError::SlotOutOfBounds)?;
slot_crosslink_committees[slot_index] = Some(i);
// Loop through each validator in the committee and store its attestation duties.
for (committee_index, validator_index) in
crosslink_committee.committee.iter().enumerate()
{
let attestation_duty = AttestationDuty {
slot: crosslink_committee.slot,
shard: crosslink_committee.shard,
committee_index,
committee_len: crosslink_committee.committee.len(),
(state.latest_start_shard + committees_in_current_epoch) % T::shard_count as u64
}
};
attestation_duties[*validator_index] = Some(attestation_duty);
}
}
dbg!(&shard_crosslink_committees);
let seed = state.generate_seed(epoch, spec)?;
Ok(EpochCache {
initialized_epoch: Some(epoch),
crosslink_committees,
attestation_duties,
shard_crosslink_committees,
slot_crosslink_committees,
let shuffling = shuffle_list(
active_validator_indices,
})
spec.shuffle_round_count,
&seed[..],
false,
)
.ok_or_else(|| Error::UnableToShuffle)?;
let mut cache = EpochCache {
initialized_epoch: Some(epoch),
shuffling_start_shard,
shuffling,
shard_count: T::shard_count() as u64,
committee_count,
slots_per_epoch: T::slots_per_epoch(),
attestation_duties: vec![None; state.validator_registry.len()],
};
cache.build_attestation_duties();
Ok(cache)
}
fn build_attestation_duties(&mut self) {
for (i, committee) in self
.shuffling
.honey_badger_split(self.committee_count)
.enumerate()
{
let shard = (self.shuffling_start_shard + i as u64) % self.shard_count;
let slot = self.crosslink_slot_for_shard(shard).unwrap();
for (committee_index, validator_index) in committee.iter().enumerate() {
self.attestation_duties[*validator_index] = Some(AttestationDuty {
slot,
shard,
committee_index,
committee_len: committee.len(),
});
}
}
}
pub fn is_initialized_at(&self, epoch: Epoch) -> bool {
Some(epoch) == self.initialized_epoch
}
pub fn active_validator_indices(&self) -> &[usize] {
&self.shuffling
}
/// Return `Some(CrosslinkCommittee)` if the given shard has a committee during the given
/// `epoch`.
pub fn get_crosslink_committee_for_shard(&self, shard: Shard) -> Option<&CrosslinkCommittee> {
if shard > self.shard_crosslink_committees.len() as u64 {
None
} else {
let i = self.shard_crosslink_committees[shard as usize]?;
Some(&self.crosslink_committees[i])
pub fn get_crosslink_committee_for_shard(&self, shard: Shard) -> Option<CrosslinkCommittee> {
if shard >= self.shard_count || self.initialized_epoch.is_none() {
return None;
}
let committee_index =
(shard + self.shard_count - self.shuffling_start_shard) % self.shard_count;
let committee = self.compute_committee(committee_index as usize, self.committee_count)?;
let slot = self.crosslink_slot_for_shard(shard)?;
Some(CrosslinkCommittee {
shard,
committee,
slot,
})
}
pub fn active_validator_count(&self) -> usize {
self.shuffling.len()
}
fn compute_committee(&self, index: usize, count: usize) -> Option<&[usize]> {
if self.initialized_epoch.is_none() {
return None;
}
let num_validators = self.shuffling.len();
// Note: `count != 0` is enforced in the constructor.
let start = (num_validators * index) / count;
let end = (num_validators * (index + 1)) / count;
Some(&self.shuffling[start..end])
}
fn crosslink_slot_for_shard(&self, shard: u64) -> Option<Slot> {
let offset = (shard + self.shard_count - self.shuffling_start_shard) % self.shard_count;
Some(
self.initialized_epoch?.start_slot(self.slots_per_epoch)
+ offset / (self.committee_count as u64 / self.slots_per_epoch),
)
}
}
@ -125,64 +183,6 @@ pub fn get_active_validator_indices(validators: &[Validator], epoch: Epoch) -> V
active
}
pub fn compute_epoch_committees<T: EthSpec>(
epoch: Epoch,
state: &BeaconState<T>,
active_validator_indices: Vec<usize>,
epoch_committee_count: u64,
spec: &ChainSpec,
) -> Result<Vec<CrosslinkCommittee>, BeaconStateError> {
let seed = state.generate_seed(epoch, spec)?;
// The shuffler fails on a empty list, so if there are no active validator indices, simply
// return an empty list.
let shuffled_active_validator_indices = if active_validator_indices.is_empty() {
vec![]
} else {
shuffle_list(
active_validator_indices,
spec.shuffle_round_count,
&seed[..],
false,
)
.ok_or_else(|| Error::UnableToShuffle)?
};
let epoch_start_shard = state.get_epoch_start_shard(epoch, spec)?;
Ok(shuffled_active_validator_indices
.honey_badger_split(epoch_committee_count as usize)
.enumerate()
.map(|(index, committee)| {
let shard = (epoch_start_shard + index as u64) % spec.shard_count;
dbg!(index);
dbg!(shard);
let slot = crosslink_committee_slot(
shard,
epoch,
epoch_start_shard,
epoch_committee_count,
spec,
);
CrosslinkCommittee {
slot,
shard,
committee: committee.to_vec(),
}
})
.collect())
}
fn crosslink_committee_slot(
shard: u64,
epoch: Epoch,
epoch_start_shard: u64,
epoch_committee_count: u64,
spec: &ChainSpec,
) -> Slot {
// Excerpt from `get_attestation_slot` in the spec.
let offset = (shard + spec.shard_count - epoch_start_shard) % spec.shard_count;
epoch.start_slot(spec.slots_per_epoch) + offset / (epoch_committee_count / spec.slots_per_epoch)
pub fn get_active_validator_count(validators: &[Validator], epoch: Epoch) -> usize {
validators.iter().filter(|v| v.is_active_at(epoch)).count()
}

15
eth2/types/src/beacon_state/epoch_cache/tests.rs
View File

@ -13,11 +13,15 @@ fn execute_sane_cache_test<T: EthSpec>(
) {
let active_indices: Vec<usize> = (0..validator_count).collect();
let seed = state.generate_seed(epoch, spec).unwrap();
let start_shard = state.get_epoch_start_shard(epoch, spec).unwrap();
let start_shard = 0;
let mut ordered_indices = state
.get_cached_active_validator_indices(epoch)
.unwrap()
.to_vec();
ordered_indices.sort_unstable();
assert_eq!(
&active_indices[..],
state.get_cached_active_validator_indices(epoch).unwrap(),
active_indices, ordered_indices,
"Validator indices mismatch"
);
@ -29,15 +33,12 @@ fn execute_sane_cache_test<T: EthSpec>(
for i in 0..T::shard_count() {
let shard = (i + start_shard as usize) % T::shard_count();
dbg!(shard);
dbg!(start_shard);
let c = state
.get_crosslink_committee_for_shard(shard as u64, epoch)
.unwrap()
.unwrap();
for &i in &c.committee {
for &i in c.committee {
assert_eq!(
i,
*expected_indices_iter.next().unwrap(),

34
eth2/types/src/crosslink_committee.rs
View File

@ -1,21 +1,25 @@
use crate::*;
use serde_derive::{Deserialize, Serialize};
use ssz_derive::{Decode, Encode};
use tree_hash_derive::{CachedTreeHash, TreeHash};
#[derive(
Default,
Clone,
Debug,
PartialEq,
Serialize,
Deserialize,
Decode,
Encode,
TreeHash,
CachedTreeHash,
)]
pub struct CrosslinkCommittee {
#[derive(Default, Clone, Debug, PartialEq, TreeHash, CachedTreeHash)]
pub struct CrosslinkCommittee<'a> {
pub slot: Slot,
pub shard: Shard,
pub committee: &'a [usize],
}
impl<'a> CrosslinkCommittee<'a> {
pub fn into_owned(self) -> OwnedCrosslinkCommittee {
OwnedCrosslinkCommittee {
slot: self.slot,
shard: self.shard,
committee: self.committee.to_vec(),
}
}
}
#[derive(Default, Clone, Debug, PartialEq, TreeHash, CachedTreeHash)]
pub struct OwnedCrosslinkCommittee {
pub slot: Slot,
pub shard: Shard,
pub committee: Vec<usize>,

2
eth2/types/src/lib.rs
View File

@ -47,7 +47,7 @@ pub use crate::beacon_block_header::BeaconBlockHeader;
pub use crate::beacon_state::{Error as BeaconStateError, *};
pub use crate::chain_spec::{ChainSpec, Domain};
pub use crate::crosslink::Crosslink;
pub use crate::crosslink_committee::CrosslinkCommittee;
pub use crate::crosslink_committee::{CrosslinkCommittee, OwnedCrosslinkCommittee};
pub use crate::deposit::Deposit;
pub use crate::deposit_data::DepositData;
pub use crate::eth1_data::Eth1Data;

4
eth2/types/src/test_utils/builders/testing_beacon_block_builder.rs
View File

@ -116,8 +116,8 @@ impl TestingBeaconBlockBuilder {
committees.push((
slot,
crosslink_committee.committee.clone(),
crosslink_committee.committee.clone(),
crosslink_committee.committee.to_vec(),
crosslink_committee.committee.to_vec(),
crosslink_committee.shard,
));

8
eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs
View File

@ -166,7 +166,7 @@ impl<T: EthSpec> TestingBeaconStateBuilder<T> {
/// Note: this performs the build when called. Ensure that no changes are made that would
/// invalidate this cache.
pub fn build_caches(&mut self, spec: &ChainSpec) -> Result<(), BeaconStateError> {
self.state.build_all_caches(spec);
self.state.build_all_caches(spec).unwrap();
Ok(())
}
@ -222,10 +222,12 @@ impl<T: EthSpec> TestingBeaconStateBuilder<T> {
for slot in first_slot..=last_slot {
let slot = Slot::from(slot);
let committees = state
let committees: Vec<OwnedCrosslinkCommittee> = state
.get_crosslink_committees_at_slot(slot, spec)
.unwrap()
.clone();
.into_iter()
.map(|c| c.clone().into_owned())
.collect();
for crosslink_committee in committees {
let mut builder = TestingPendingAttestationBuilder::new(