cerc-io/lighthouse
16
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Optimize historic committee calculation for the HTTP API (#3272)

Browse Source 
## Issue Addressed

Closes https://github.com/sigp/lighthouse/issues/3270

## Proposed Changes

Optimize the calculation of historic beacon committees in the HTTP API.

This is achieved by allowing committee caches to be constructed for historic epochs, and constructing these committee caches on the fly in the API. This is much faster than reconstructing the state at the requested epoch, which usually takes upwards of 20s, and sometimes minutes with SPRP=8192. The depth of the `randao_mixes` array allows us to look back 64K epochs/0.8 years from a single state, which is pretty awesome!

We always use the `state_id` provided by the caller, but will return a nice 400 error if the epoch requested is out of range for the state requested, e.g.

```bash
# Prater
curl "http://localhost:5052/eth/v1/beacon/states/3170304/committees?epoch=33538"
```

```json
{"code":400,"message":"BAD_REQUEST: epoch out of bounds, try state at slot 1081344","stacktraces":[]}
```

Queries will be fastest when aligned to `slot % SPRP == 0`, so the hint suggests a slot that is 0 mod 8192.
This commit is contained in:
Michael Sproul 2022-07-04 02:56:11 +00:00
parent 66bb5c716c
commit 61ed5f0ec6
4 changed files with 89 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
beacon_node/http_api/src/lib.rs
View File

@ -657,26 +657,41 @@ pub fn serve<T: BeaconChainTypes>(
.and(warp::path::end())
.and_then(
|state_id: StateId, chain: Arc<BeaconChain<T>>, query: api_types::CommitteesQuery| {
// the api spec says if the epoch is not present then the epoch of the state should be used
let query_state_id = query.epoch.map_or(state_id, |epoch| {
StateId::slot(epoch.start_slot(T::EthSpec::slots_per_epoch()))
});
blocking_json_task(move || {
query_state_id.map_state(&chain, |state| {
let epoch = state.slot().epoch(T::EthSpec::slots_per_epoch());
state_id.map_state(&chain, |state| {
let current_epoch = state.current_epoch();
let epoch = query.epoch.unwrap_or(current_epoch);
let committee_cache = if state
.committee_cache_is_initialized(RelativeEpoch::Current)
let committee_cache = match RelativeEpoch::from_epoch(current_epoch, epoch)
{
state
.committee_cache(RelativeEpoch::Current)
.map(Cow::Borrowed)
} else {
CommitteeCache::initialized(state, epoch, &chain.spec).map(Cow::Owned)
Ok(relative_epoch)
if state.committee_cache_is_initialized(relative_epoch) =>
{
state.committee_cache(relative_epoch).map(Cow::Borrowed)
}
_ => CommitteeCache::initialized(state, epoch, &chain.spec)
.map(Cow::Owned),
}
.map_err(BeaconChainError::BeaconStateError)
.map_err(warp_utils::reject::beacon_chain_error)?;
.map_err(|e| match e {
BeaconStateError::EpochOutOfBounds => {
let max_sprp = T::EthSpec::slots_per_historical_root() as u64;
let first_subsequent_restore_point_slot =
((epoch.start_slot(T::EthSpec::slots_per_epoch()) / max_sprp)
+ 1)
* max_sprp;
if epoch < current_epoch {
warp_utils::reject::custom_bad_request(format!(
"epoch out of bounds, try state at slot {}",
first_subsequent_restore_point_slot,
))
} else {
warp_utils::reject::custom_bad_request(
"epoch out of bounds, too far in future".into(),
)
}
}
_ => warp_utils::reject::beacon_chain_error(e.into()),
})?;
// Use either the supplied slot or all slots in the epoch.
let slots = query.slot.map(|slot| vec![slot]).unwrap_or_else(|| {

7
consensus/types/src/beacon_state.rs
View File

@ -963,6 +963,13 @@ impl<T: EthSpec> BeaconState<T> {
}
}
/// Return the minimum epoch for which `get_randao_mix` will return a non-error value.
pub fn min_randao_epoch(&self) -> Epoch {
self.current_epoch()
.saturating_add(1u64)
.saturating_sub(T::EpochsPerHistoricalVector::to_u64())
}
/// XOR-assigns the existing `epoch` randao mix with the hash of the `signature`.
///
/// # Errors:

14
consensus/types/src/beacon_state/committee_cache.rs
View File

@ -38,8 +38,18 @@ impl CommitteeCache {
epoch: Epoch,
spec: &ChainSpec,
) -> Result<CommitteeCache, Error> {
RelativeEpoch::from_epoch(state.current_epoch(), epoch)
.map_err(|_| Error::EpochOutOfBounds)?;
// Check that the cache is being built for an in-range epoch.
//
// We allow caches to be constructed for historic epochs, per:
//
// https://github.com/sigp/lighthouse/issues/3270
let reqd_randao_epoch = epoch
.saturating_sub(spec.min_seed_lookahead)
.saturating_sub(1u64);
if reqd_randao_epoch < state.min_randao_epoch() || epoch > state.current_epoch() + 1 {
return Err(Error::EpochOutOfBounds);
}
// May cause divide-by-zero errors.
if T::slots_per_epoch() == 0 {

50
consensus/types/src/beacon_state/committee_cache/tests.rs
View File

@ -42,7 +42,7 @@ async fn new_state<T: EthSpec>(validator_count: usize, slot: Slot) -> BeaconStat
.add_attested_blocks_at_slots(
head_state,
Hash256::zero(),
(1..slot.as_u64())
(1..=slot.as_u64())
.map(Slot::new)
.collect::<Vec<_>>()
.as_slice(),
@ -86,6 +86,8 @@ async fn shuffles_for_the_right_epoch() {
let mut state = new_state::<MinimalEthSpec>(num_validators, slot).await;
let spec = &MinimalEthSpec::default_spec();
assert_eq!(state.current_epoch(), epoch);
let distinct_hashes: Vec<Hash256> = (0..MinimalEthSpec::epochs_per_historical_vector())
.map(|i| Hash256::from_low_u64_be(i as u64))
.collect();
@ -124,15 +126,41 @@ async fn shuffles_for_the_right_epoch() {
}
};
let cache = CommitteeCache::initialized(&state, state.current_epoch(), spec).unwrap();
assert_eq!(cache.shuffling(), shuffling_with_seed(current_seed));
assert_shuffling_positions_accurate(&cache);
// We can initialize the committee cache at recent epochs in the past, and one epoch into the
// future.
for e in (0..=epoch.as_u64() + 1).map(Epoch::new) {
let seed = state.get_seed(e, Domain::BeaconAttester, spec).unwrap();
let cache = CommitteeCache::initialized(&state, e, spec)
.unwrap_or_else(|_| panic!("failed at epoch {}", e));
assert_eq!(cache.shuffling(), shuffling_with_seed(seed));
assert_shuffling_positions_accurate(&cache);
}
let cache = CommitteeCache::initialized(&state, state.previous_epoch(), spec).unwrap();
assert_eq!(cache.shuffling(), shuffling_with_seed(previous_seed));
assert_shuffling_positions_accurate(&cache);
let cache = CommitteeCache::initialized(&state, state.next_epoch().unwrap(), spec).unwrap();
assert_eq!(cache.shuffling(), shuffling_with_seed(next_seed));
assert_shuffling_positions_accurate(&cache);
// We should *not* be able to build a committee cache for the epoch after the next epoch.
assert_eq!(
CommitteeCache::initialized(&state, epoch + 2, spec),
Err(BeaconStateError::EpochOutOfBounds)
);
}
#[tokio::test]
async fn min_randao_epoch_correct() {
let num_validators = MinimalEthSpec::minimum_validator_count() * 2;
let current_epoch = Epoch::new(MinimalEthSpec::epochs_per_historical_vector() as u64 * 2);
let mut state = new_state::<MinimalEthSpec>(
num_validators,
Epoch::new(1).start_slot(MinimalEthSpec::slots_per_epoch()),
)
.await;
// Override the epoch so that there's some room to move.
*state.slot_mut() = current_epoch.start_slot(MinimalEthSpec::slots_per_epoch());
assert_eq!(state.current_epoch(), current_epoch);
// The min_randao_epoch should be the minimum epoch such that `get_randao_mix` returns `Ok`.
let min_randao_epoch = state.min_randao_epoch();
state.get_randao_mix(min_randao_epoch).unwrap();
state.get_randao_mix(min_randao_epoch - 1).unwrap_err();
state.get_randao_mix(min_randao_epoch + 1).unwrap();
}