cerc-io/lighthouse
10
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Add configurable block replayer (#2863)

Browse Source 
## Issue Addressed

Successor to #2431

## Proposed Changes

* Add a `BlockReplayer` struct to abstract over the intricacies of calling `per_slot_processing` and `per_block_processing` while avoiding unnecessary tree hashing.
* Add a variant of the forwards state root iterator that does not require an `end_state`.
* Use the `BlockReplayer` when reconstructing states in the database. Use the efficient forwards iterator for frozen states.
* Refactor the iterators to remove `Arc<HotColdDB>` (this seems to be neater than making _everything_ an `Arc<HotColdDB>` as I did in #2431).

Supplying the state roots allow us to avoid building a tree hash cache at all when reconstructing historic states, which saves around 1 second flat (regardless of `slots-per-restore-point`). This is a small percentage of worst-case state load times with 200K validators and SPRP=2048 (~15s vs ~16s) but a significant speed-up for more frequent restore points: state loads with SPRP=32 should be now consistently <500ms instead of 1.5s (a ~3x speedup).

## Additional Info

Required by https://github.com/sigp/lighthouse/pull/2628
This commit is contained in:
Michael Sproul 2021-12-21 06:30:52 +00:00
parent 56d596ee42
commit a290a3c537
25 changed files with 956 additions and 444 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
beacon_node/beacon_chain/src/beacon_chain.rs
View File

@ -69,7 +69,7 @@ use state_processing::{
per_block_processing::{errors::AttestationValidationError, is_merge_transition_complete}, per_block_processing::{errors::AttestationValidationError, is_merge_transition_complete},
per_slot_processing, per_slot_processing,
state_advance::{complete_state_advance, partial_state_advance}, state_advance::{complete_state_advance, partial_state_advance},
BlockSignatureStrategy, SigVerifiedOp, BlockSignatureStrategy, SigVerifiedOp, VerifyBlockRoot,
}; };
use std::borrow::Cow; use std::borrow::Cow;
use std::cmp::Ordering; use std::cmp::Ordering;
@ -488,7 +488,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
pub fn forwards_iter_block_roots( pub fn forwards_iter_block_roots(
&self, &self,
start_slot: Slot, start_slot: Slot,
) -> Result<impl Iterator<Item = Result<(Hash256, Slot), Error>>, Error> { ) -> Result<impl Iterator<Item = Result<(Hash256, Slot), Error>> + '_, Error> {
let oldest_block_slot = self.store.get_oldest_block_slot(); let oldest_block_slot = self.store.get_oldest_block_slot();
if start_slot < oldest_block_slot { if start_slot < oldest_block_slot {
return Err(Error::HistoricalBlockError( return Err(Error::HistoricalBlockError(
@ -501,8 +501,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
let local_head = self.head()?; let local_head = self.head()?;
let iter = HotColdDB::forwards_block_roots_iterator( let iter = self.store.forwards_block_roots_iterator(
self.store.clone(),
start_slot, start_slot,
local_head.beacon_state, local_head.beacon_state,
local_head.beacon_block_root, local_head.beacon_block_root,
@ -512,6 +511,43 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
Ok(iter.map(|result| result.map_err(Into::into))) Ok(iter.map(|result| result.map_err(Into::into)))
} }
/// Even more efficient variant of `forwards_iter_block_roots` that will avoid cloning the head
/// state if it isn't required for the requested range of blocks.
pub fn forwards_iter_block_roots_until(
&self,
start_slot: Slot,
end_slot: Slot,
) -> Result<impl Iterator<Item = Result<(Hash256, Slot), Error>> + '_, Error> {
let oldest_block_slot = self.store.get_oldest_block_slot();
if start_slot < oldest_block_slot {
return Err(Error::HistoricalBlockError(
HistoricalBlockError::BlockOutOfRange {
slot: start_slot,
oldest_block_slot,
},
));
}
self.with_head(move |head| {
let iter = self.store.forwards_block_roots_iterator_until(
start_slot,
end_slot,
|| {
(
head.beacon_state.clone_with_only_committee_caches(),
head.beacon_block_root,
)
},
&self.spec,
)?;
Ok(iter
.map(|result| result.map_err(Into::into))
.take_while(move |result| {
result.as_ref().map_or(true, |(_, slot)| *slot <= end_slot)
}))
})
}
/// Traverse backwards from `block_root` to find the block roots of its ancestors. /// Traverse backwards from `block_root` to find the block roots of its ancestors.
/// ///
/// ## Notes /// ## Notes
@ -524,14 +560,14 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
pub fn rev_iter_block_roots_from( pub fn rev_iter_block_roots_from(
&self, &self,
block_root: Hash256, block_root: Hash256,
) -> Result<impl Iterator<Item = Result<(Hash256, Slot), Error>>, Error> { ) -> Result<impl Iterator<Item = Result<(Hash256, Slot), Error>> + '_, Error> {
let block = self let block = self
.get_block(&block_root)? .get_block(&block_root)?
.ok_or(Error::MissingBeaconBlock(block_root))?; .ok_or(Error::MissingBeaconBlock(block_root))?;
let state = self let state = self
.get_state(&block.state_root(), Some(block.slot()))? .get_state(&block.state_root(), Some(block.slot()))?
.ok_or_else(|| Error::MissingBeaconState(block.state_root()))?; .ok_or_else(|| Error::MissingBeaconState(block.state_root()))?;
let iter = BlockRootsIterator::owned(self.store.clone(), state); let iter = BlockRootsIterator::owned(&self.store, state);
Ok(std::iter::once(Ok((block_root, block.slot()))) Ok(std::iter::once(Ok((block_root, block.slot())))
.chain(iter) .chain(iter)
.map(|result| result.map_err(|e| e.into()))) .map(|result| result.map_err(|e| e.into())))
@ -618,12 +654,12 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
/// - As this iterator starts at the `head` of the chain (viz., the best block), the first slot /// - As this iterator starts at the `head` of the chain (viz., the best block), the first slot
/// returned may be earlier than the wall-clock slot. /// returned may be earlier than the wall-clock slot.
pub fn rev_iter_state_roots_from<'a>( pub fn rev_iter_state_roots_from<'a>(
&self, &'a self,
state_root: Hash256, state_root: Hash256,
state: &'a BeaconState<T::EthSpec>, state: &'a BeaconState<T::EthSpec>,
) -> impl Iterator<Item = Result<(Hash256, Slot), Error>> + 'a { ) -> impl Iterator<Item = Result<(Hash256, Slot), Error>> + 'a {
std::iter::once(Ok((state_root, state.slot()))) std::iter::once(Ok((state_root, state.slot())))
.chain(StateRootsIterator::new(self.store.clone(), state)) .chain(StateRootsIterator::new(&self.store, state))
.map(|result| result.map_err(Into::into)) .map(|result| result.map_err(Into::into))
} }
@ -637,11 +673,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
pub fn forwards_iter_state_roots( pub fn forwards_iter_state_roots(
&self, &self,
start_slot: Slot, start_slot: Slot,
) -> Result<impl Iterator<Item = Result<(Hash256, Slot), Error>>, Error> { ) -> Result<impl Iterator<Item = Result<(Hash256, Slot), Error>> + '_, Error> {
let local_head = self.head()?; let local_head = self.head()?;
let iter = HotColdDB::forwards_state_roots_iterator( let iter = self.store.forwards_state_roots_iterator(
self.store.clone(),
start_slot, start_slot,
local_head.beacon_state_root(), local_head.beacon_state_root(),
local_head.beacon_state, local_head.beacon_state,
@ -651,6 +686,36 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
Ok(iter.map(|result| result.map_err(Into::into))) Ok(iter.map(|result| result.map_err(Into::into)))
} }
/// Super-efficient forwards state roots iterator that avoids cloning the head if the state
/// roots lie entirely within the freezer database.
///
/// The iterator returned will include roots for `start_slot..=end_slot`, i.e. it
/// is endpoint inclusive.
pub fn forwards_iter_state_roots_until(
&self,
start_slot: Slot,
end_slot: Slot,
) -> Result<impl Iterator<Item = Result<(Hash256, Slot), Error>> + '_, Error> {
self.with_head(move |head| {
let iter = self.store.forwards_state_roots_iterator_until(
start_slot,
end_slot,
|| {
(
head.beacon_state.clone_with_only_committee_caches(),
head.beacon_state_root(),
)
},
&self.spec,
)?;
Ok(iter
.map(|result| result.map_err(Into::into))
.take_while(move |result| {
result.as_ref().map_or(true, |(_, slot)| *slot <= end_slot)
}))
})
}
/// Returns the block at the given slot, if any. Only returns blocks in the canonical chain. /// Returns the block at the given slot, if any. Only returns blocks in the canonical chain.
/// ///
/// Use the `skips` parameter to define the behaviour when `request_slot` is a skipped slot. /// Use the `skips` parameter to define the behaviour when `request_slot` is a skipped slot.
@ -708,7 +773,9 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
return Ok(Some(root)); return Ok(Some(root));
} }
process_results(self.forwards_iter_state_roots(request_slot)?, |mut iter| { process_results(
self.forwards_iter_state_roots_until(request_slot, request_slot)?,
|mut iter| {
if let Some((root, slot)) = iter.next() { if let Some((root, slot)) = iter.next() {
if slot == request_slot { if slot == request_slot {
Ok(Some(root)) Ok(Some(root))
@ -719,7 +786,8 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
} else { } else {
Ok(None) Ok(None)
} }
})? },
)?
} }
/// Returns the block root at the given slot, if any. Only returns roots in the canonical chain. /// Returns the block root at the given slot, if any. Only returns roots in the canonical chain.
@ -790,11 +858,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
return Ok(root_opt); return Ok(root_opt);
} }
if let Some(((prev_root, _), (curr_root, curr_slot))) = if let Some(((prev_root, _), (curr_root, curr_slot))) = process_results(
process_results(self.forwards_iter_block_roots(prev_slot)?, |iter| { self.forwards_iter_block_roots_until(prev_slot, request_slot)?,
iter.tuple_windows().next() |iter| iter.tuple_windows().next(),
})? )? {
{
// Sanity check. // Sanity check.
if curr_slot != request_slot { if curr_slot != request_slot {
return Err(Error::InconsistentForwardsIter { return Err(Error::InconsistentForwardsIter {
@ -842,7 +909,9 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
return Ok(Some(root)); return Ok(Some(root));
} }
process_results(self.forwards_iter_block_roots(request_slot)?, |mut iter| { process_results(
self.forwards_iter_block_roots_until(request_slot, request_slot)?,
|mut iter| {
if let Some((root, slot)) = iter.next() { if let Some((root, slot)) = iter.next() {
if slot == request_slot { if slot == request_slot {
Ok(Some(root)) Ok(Some(root))
@ -853,7 +922,8 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
} else { } else {
Ok(None) Ok(None)
} }
})? },
)?
} }
/// Returns the block at the given root, if any. /// Returns the block at the given root, if any.
@ -1112,7 +1182,8 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
Ok(state) Ok(state)
} }
Ordering::Less => { Ordering::Less => {
let state_root = process_results(self.forwards_iter_state_roots(slot)?, |iter| { let state_root =
process_results(self.forwards_iter_state_roots_until(slot, slot)?, |iter| {
iter.take_while(|(_, current_slot)| *current_slot >= slot) iter.take_while(|(_, current_slot)| *current_slot >= slot)
.find(|(_, current_slot)| *current_slot == slot) .find(|(_, current_slot)| *current_slot == slot)
.map(|(root, _slot)| root) .map(|(root, _slot)| root)
@ -1256,7 +1327,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
beacon_block_root: Hash256, beacon_block_root: Hash256,
state: &BeaconState<T::EthSpec>, state: &BeaconState<T::EthSpec>,
) -> Result<Option<Hash256>, Error> { ) -> Result<Option<Hash256>, Error> {
let iter = BlockRootsIterator::new(self.store.clone(), state); let iter = BlockRootsIterator::new(&self.store, state);
let iter_with_head = std::iter::once(Ok((beacon_block_root, state.slot()))) let iter_with_head = std::iter::once(Ok((beacon_block_root, state.slot())))
.chain(iter) .chain(iter)
.map(|result| result.map_err(|e| e.into())); .map(|result| result.map_err(|e| e.into()));
@ -2983,6 +3054,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
&block, &block,
None, None,
BlockSignatureStrategy::VerifyRandao, BlockSignatureStrategy::VerifyRandao,
VerifyBlockRoot::True,
&self.spec, &self.spec,
)?; )?;
drop(process_timer); drop(process_timer);
@ -3324,7 +3396,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.epoch .epoch
.start_slot(T::EthSpec::slots_per_epoch()); .start_slot(T::EthSpec::slots_per_epoch());
let new_finalized_state_root = process_results( let new_finalized_state_root = process_results(
StateRootsIterator::new(self.store.clone(), &head.beacon_state), StateRootsIterator::new(&self.store, &head.beacon_state),
|mut iter| { |mut iter| {
iter.find_map(|(state_root, slot)| { iter.find_map(|(state_root, slot)| {
if slot == new_finalized_slot { if slot == new_finalized_slot {

3
beacon_node/beacon_chain/src/block_verification.rs
View File

@ -65,7 +65,7 @@ use state_processing::{
block_signature_verifier::{BlockSignatureVerifier, Error as BlockSignatureVerifierError}, block_signature_verifier::{BlockSignatureVerifier, Error as BlockSignatureVerifierError},
per_block_processing, per_slot_processing, per_block_processing, per_slot_processing,
state_advance::partial_state_advance, state_advance::partial_state_advance,
BlockProcessingError, BlockSignatureStrategy, SlotProcessingError, BlockProcessingError, BlockSignatureStrategy, SlotProcessingError, VerifyBlockRoot,
}; };
use std::borrow::Cow; use std::borrow::Cow;
use std::fs; use std::fs;
@ -1185,6 +1185,7 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
Some(block_root), Some(block_root),
// Signatures were verified earlier in this function. // Signatures were verified earlier in this function.
BlockSignatureStrategy::NoVerification, BlockSignatureStrategy::NoVerification,
VerifyBlockRoot::True,
&chain.spec, &chain.spec,
) { ) {
match err { match err {

4
beacon_node/beacon_chain/src/errors.rs
View File

@ -20,7 +20,7 @@ use state_processing::{
}, },
signature_sets::Error as SignatureSetError, signature_sets::Error as SignatureSetError,
state_advance::Error as StateAdvanceError, state_advance::Error as StateAdvanceError,
BlockProcessingError, SlotProcessingError, BlockProcessingError, BlockReplayError, SlotProcessingError,
}; };
use std::time::Duration; use std::time::Duration;
use task_executor::ShutdownReason; use task_executor::ShutdownReason;
@ -86,6 +86,7 @@ pub enum BeaconChainError {
ValidatorPubkeyCacheIncomplete(usize), ValidatorPubkeyCacheIncomplete(usize),
SignatureSetError(SignatureSetError), SignatureSetError(SignatureSetError),
BlockSignatureVerifierError(state_processing::block_signature_verifier::Error), BlockSignatureVerifierError(state_processing::block_signature_verifier::Error),
BlockReplayError(BlockReplayError),
DuplicateValidatorPublicKey, DuplicateValidatorPublicKey,
ValidatorPubkeyCacheFileError(String), ValidatorPubkeyCacheFileError(String),
ValidatorIndexUnknown(usize), ValidatorIndexUnknown(usize),
@ -160,6 +161,7 @@ easy_from_to!(ArithError, BeaconChainError);
easy_from_to!(ForkChoiceStoreError, BeaconChainError); easy_from_to!(ForkChoiceStoreError, BeaconChainError);
easy_from_to!(HistoricalBlockError, BeaconChainError); easy_from_to!(HistoricalBlockError, BeaconChainError);
easy_from_to!(StateAdvanceError, BeaconChainError); easy_from_to!(StateAdvanceError, BeaconChainError);
easy_from_to!(BlockReplayError, BeaconChainError);
#[derive(Debug)] #[derive(Debug)]
pub enum BlockProductionError { pub enum BlockProductionError {

5
beacon_node/beacon_chain/src/fork_revert.rs
View File

@ -3,7 +3,9 @@ use fork_choice::{ForkChoice, PayloadVerificationStatus};
use itertools::process_results; use itertools::process_results;
use slog::{info, warn, Logger}; use slog::{info, warn, Logger};
use state_processing::state_advance::complete_state_advance; use state_processing::state_advance::complete_state_advance;
use state_processing::{per_block_processing, per_block_processing::BlockSignatureStrategy}; use state_processing::{
per_block_processing, per_block_processing::BlockSignatureStrategy, VerifyBlockRoot,
};
use std::sync::Arc; use std::sync::Arc;
use std::time::Duration; use std::time::Duration;
use store::{iter::ParentRootBlockIterator, HotColdDB, ItemStore}; use store::{iter::ParentRootBlockIterator, HotColdDB, ItemStore};
@ -161,6 +163,7 @@ pub fn reset_fork_choice_to_finalization<E: EthSpec, Hot: ItemStore<E>, Cold: It
&block, &block,
None, None,
BlockSignatureStrategy::NoVerification, BlockSignatureStrategy::NoVerification,
VerifyBlockRoot::True,
spec, spec,
) )
.map_err(|e| format!("Error replaying block: {:?}", e))?; .map_err(|e| format!("Error replaying block: {:?}", e))?;

8
beacon_node/beacon_chain/src/migrate.rs
View File

@ -360,13 +360,11 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
new_finalized_slot, new_finalized_slot,
(new_finalized_block_hash, new_finalized_state_hash), (new_finalized_block_hash, new_finalized_state_hash),
))) )))
.chain( .chain(RootsIterator::new(&store, new_finalized_state).map(|res| {
RootsIterator::new(store.clone(), new_finalized_state).map(|res| {
res.map(|(block_root, state_root, slot)| { res.map(|(block_root, state_root, slot)| {
(slot, (block_root.into(), state_root.into())) (slot, (block_root.into(), state_root.into()))
}) })
}), }))
)
.take_while(|res| { .take_while(|res| {
res.as_ref() res.as_ref()
.map_or(true, |(slot, _)| *slot >= old_finalized_slot) .map_or(true, |(slot, _)| *slot >= old_finalized_slot)
@ -416,7 +414,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
// Iterate backwards from this head, staging blocks and states for deletion. // Iterate backwards from this head, staging blocks and states for deletion.
let iter = std::iter::once(Ok((head_hash, head_state_root, head_slot))) let iter = std::iter::once(Ok((head_hash, head_state_root, head_slot)))
.chain(RootsIterator::from_block(store.clone(), head_hash)?); .chain(RootsIterator::from_block(&store, head_hash)?);
for maybe_tuple in iter { for maybe_tuple in iter {
let (block_root, state_root, slot) = maybe_tuple?; let (block_root, state_root, slot) = maybe_tuple?;

2
beacon_node/beacon_chain/src/schema_change/migration_schema_v7.rs
View File

@ -189,7 +189,7 @@ fn map_relevant_epochs_to_roots<T: BeaconChainTypes>(
// Iterate backwards from the given `head_root` and `head_slot` and find the block root at each epoch. // Iterate backwards from the given `head_root` and `head_slot` and find the block root at each epoch.
let mut iter = std::iter::once(Ok((head_root, head_slot))) let mut iter = std::iter::once(Ok((head_root, head_slot)))
.chain(BlockRootsIterator::from_block(db, head_root).map_err(|e| format!("{:?}", e))?); .chain(BlockRootsIterator::from_block(&db, head_root).map_err(|e| format!("{:?}", e))?);
let mut roots_by_epoch = HashMap::new(); let mut roots_by_epoch = HashMap::new();
for epoch in relevant_epochs { for epoch in relevant_epochs {
let start_slot = epoch.start_slot(T::EthSpec::slots_per_epoch()); let start_slot = epoch.start_slot(T::EthSpec::slots_per_epoch());

6
beacon_node/beacon_chain/src/test_utils.rs
View File

@ -31,13 +31,13 @@ use rayon::prelude::*;
use sensitive_url::SensitiveUrl; use sensitive_url::SensitiveUrl;
use slog::Logger; use slog::Logger;
use slot_clock::TestingSlotClock; use slot_clock::TestingSlotClock;
use state_processing::state_advance::complete_state_advance; use state_processing::{state_advance::complete_state_advance, StateRootStrategy};
use std::borrow::Cow; use std::borrow::Cow;
use std::collections::{HashMap, HashSet}; use std::collections::{HashMap, HashSet};
use std::str::FromStr; use std::str::FromStr;
use std::sync::Arc; use std::sync::Arc;
use std::time::Duration; use std::time::Duration;
use store::{config::StoreConfig, BlockReplay, HotColdDB, ItemStore, LevelDB, MemoryStore}; use store::{config::StoreConfig, HotColdDB, ItemStore, LevelDB, MemoryStore};
use task_executor::ShutdownReason; use task_executor::ShutdownReason;
use tree_hash::TreeHash; use tree_hash::TreeHash;
use types::sync_selection_proof::SyncSelectionProof; use types::sync_selection_proof::SyncSelectionProof;
@ -527,7 +527,7 @@ where
pub fn get_hot_state(&self, state_hash: BeaconStateHash) -> Option<BeaconState<E>> { pub fn get_hot_state(&self, state_hash: BeaconStateHash) -> Option<BeaconState<E>> {
self.chain self.chain
.store .store
.load_hot_state(&state_hash.into(), BlockReplay::Accurate) .load_hot_state(&state_hash.into(), StateRootStrategy::Accurate)
.unwrap() .unwrap()
} }

4
beacon_node/beacon_chain/tests/block_verification.rs
View File

@ -10,7 +10,7 @@ use slasher::{Config as SlasherConfig, Slasher};
use state_processing::{ use state_processing::{
common::get_indexed_attestation, common::get_indexed_attestation,
per_block_processing::{per_block_processing, BlockSignatureStrategy}, per_block_processing::{per_block_processing, BlockSignatureStrategy},
per_slot_processing, BlockProcessingError, per_slot_processing, BlockProcessingError, VerifyBlockRoot,
}; };
use std::sync::Arc; use std::sync::Arc;
use tempfile::tempdir; use tempfile::tempdir;
@ -978,6 +978,7 @@ fn add_base_block_to_altair_chain() {
&base_block, &base_block,
None, None,
BlockSignatureStrategy::NoVerification, BlockSignatureStrategy::NoVerification,
VerifyBlockRoot::True,
&harness.chain.spec, &harness.chain.spec,
), ),
Err(BlockProcessingError::InconsistentBlockFork( Err(BlockProcessingError::InconsistentBlockFork(
@ -1096,6 +1097,7 @@ fn add_altair_block_to_base_chain() {
&altair_block, &altair_block,
None, None,
BlockSignatureStrategy::NoVerification, BlockSignatureStrategy::NoVerification,
VerifyBlockRoot::True,
&harness.chain.spec, &harness.chain.spec,
), ),
Err(BlockProcessingError::InconsistentBlockFork( Err(BlockProcessingError::InconsistentBlockFork(

194
beacon_node/beacon_chain/tests/store_tests.rs
View File

@ -14,6 +14,7 @@ use lazy_static::lazy_static;
use logging::test_logger; use logging::test_logger;
use maplit::hashset; use maplit::hashset;
use rand::Rng; use rand::Rng;
use state_processing::BlockReplayer;
use std::collections::HashMap; use std::collections::HashMap;
use std::collections::HashSet; use std::collections::HashSet;
use std::convert::TryInto; use std::convert::TryInto;
@ -126,7 +127,7 @@ fn randomised_skips() {
"head should be at the current slot" "head should be at the current slot"
); );
check_split_slot(&harness, store); check_split_slot(&harness, store.clone());
check_chain_dump(&harness, num_blocks_produced + 1); check_chain_dump(&harness, num_blocks_produced + 1);
check_iterators(&harness); check_iterators(&harness);
} }
@ -358,6 +359,191 @@ fn epoch_boundary_state_attestation_processing() {
assert!(checked_pre_fin); assert!(checked_pre_fin);
} }
// Test that the `end_slot` for forwards block and state root iterators works correctly.
#[test]
fn forwards_iter_block_and_state_roots_until() {
let num_blocks_produced = E::slots_per_epoch() * 17;
let db_path = tempdir().unwrap();
let store = get_store(&db_path);
let harness = get_harness(store.clone(), LOW_VALIDATOR_COUNT);
let all_validators = &harness.get_all_validators();
let (mut head_state, mut head_state_root) = harness.get_current_state_and_root();
let head_block_root = harness.chain.head_info().unwrap().block_root;
let mut block_roots = vec![head_block_root];
let mut state_roots = vec![head_state_root];
for slot in (1..=num_blocks_produced).map(Slot::from) {
let (block_root, mut state) = harness
.add_attested_block_at_slot(slot, head_state, head_state_root, all_validators)
.unwrap();
head_state_root = state.update_tree_hash_cache().unwrap();
head_state = state;
block_roots.push(block_root.into());
state_roots.push(head_state_root);
}
check_finalization(&harness, num_blocks_produced);
check_split_slot(&harness, store.clone());
// The last restore point slot is the point at which the hybrid forwards iterator behaviour
// changes.
let last_restore_point_slot = store.get_latest_restore_point_slot();
assert!(last_restore_point_slot > 0);
let chain = &harness.chain;
let head_state = harness.get_current_state();
let head_slot = head_state.slot();
assert_eq!(head_slot, num_blocks_produced);
let test_range = |start_slot: Slot, end_slot: Slot| {
let mut block_root_iter = chain
.forwards_iter_block_roots_until(start_slot, end_slot)
.unwrap();
let mut state_root_iter = chain
.forwards_iter_state_roots_until(start_slot, end_slot)
.unwrap();
for slot in (start_slot.as_u64()..=end_slot.as_u64()).map(Slot::new) {
let block_root = block_roots[slot.as_usize()];
assert_eq!(block_root_iter.next().unwrap().unwrap(), (block_root, slot));
let state_root = state_roots[slot.as_usize()];
assert_eq!(state_root_iter.next().unwrap().unwrap(), (state_root, slot));
}
};
let split_slot = store.get_split_slot();
assert!(split_slot > last_restore_point_slot);
test_range(Slot::new(0), last_restore_point_slot);
test_range(last_restore_point_slot, last_restore_point_slot);
test_range(last_restore_point_slot - 1, last_restore_point_slot);
test_range(Slot::new(0), last_restore_point_slot - 1);
test_range(Slot::new(0), split_slot);
test_range(last_restore_point_slot - 1, split_slot);
test_range(Slot::new(0), head_state.slot());
}
#[test]
fn block_replay_with_inaccurate_state_roots() {
let num_blocks_produced = E::slots_per_epoch() * 3 + 31;
let db_path = tempdir().unwrap();
let store = get_store(&db_path);
let harness = get_harness(store.clone(), LOW_VALIDATOR_COUNT);
let chain = &harness.chain;
harness.extend_chain(
num_blocks_produced as usize,
BlockStrategy::OnCanonicalHead,
AttestationStrategy::AllValidators,
);
// Slot must not be 0 mod 32 or else no blocks will be replayed.
let (mut head_state, head_root) = harness.get_current_state_and_root();
assert_ne!(head_state.slot() % 32, 0);
let mut fast_head_state = store
.get_inconsistent_state_for_attestation_verification_only(
&head_root,
Some(head_state.slot()),
)
.unwrap()
.unwrap();
assert_eq!(head_state.validators(), fast_head_state.validators());
head_state.build_all_committee_caches(&chain.spec).unwrap();
fast_head_state
.build_all_committee_caches(&chain.spec)
.unwrap();
assert_eq!(
head_state
.get_cached_active_validator_indices(RelativeEpoch::Current)
.unwrap(),
fast_head_state
.get_cached_active_validator_indices(RelativeEpoch::Current)
.unwrap()
);
}
#[test]
fn block_replayer_hooks() {
let db_path = tempdir().unwrap();
let store = get_store(&db_path);
let harness = get_harness(store.clone(), LOW_VALIDATOR_COUNT);
let chain = &harness.chain;
let block_slots = vec![1, 3, 5, 10, 11, 12, 13, 14, 31, 32, 33]
.into_iter()
.map(Slot::new)
.collect::<Vec<_>>();
let max_slot = *block_slots.last().unwrap();
let all_slots = (0..=max_slot.as_u64()).map(Slot::new).collect::<Vec<_>>();
let (state, state_root) = harness.get_current_state_and_root();
let all_validators = harness.get_all_validators();
let (_, _, end_block_root, mut end_state) = harness.add_attested_blocks_at_slots(
state.clone(),
state_root,
&block_slots,
&all_validators,
);
let blocks = store
.load_blocks_to_replay(Slot::new(0), max_slot, end_block_root.into())
.unwrap();
let mut pre_slots = vec![];
let mut post_slots = vec![];
let mut pre_block_slots = vec![];
let mut post_block_slots = vec![];
let mut replay_state = BlockReplayer::<MinimalEthSpec>::new(state, &chain.spec)
.pre_slot_hook(Box::new(|state| {
pre_slots.push(state.slot());
Ok(())
}))
.post_slot_hook(Box::new(|state, epoch_summary, is_skip_slot| {
if is_skip_slot {
assert!(!block_slots.contains(&state.slot()));
} else {
assert!(block_slots.contains(&state.slot()));
}
if state.slot() % E::slots_per_epoch() == 0 {
assert!(epoch_summary.is_some());
}
post_slots.push(state.slot());
Ok(())
}))
.pre_block_hook(Box::new(|state, block| {
assert_eq!(state.slot(), block.slot());
pre_block_slots.push(block.slot());
Ok(())
}))
.post_block_hook(Box::new(|state, block| {
assert_eq!(state.slot(), block.slot());
post_block_slots.push(block.slot());
Ok(())
}))
.apply_blocks(blocks, None)
.unwrap()
.into_state();
// All but last slot seen by pre-slot hook
assert_eq!(&pre_slots, all_slots.split_last().unwrap().1);
// All but 0th slot seen by post-slot hook
assert_eq!(&post_slots, all_slots.split_first().unwrap().1);
// All blocks seen by both hooks
assert_eq!(pre_block_slots, block_slots);
assert_eq!(post_block_slots, block_slots);
// States match.
end_state.drop_all_caches().unwrap();
replay_state.drop_all_caches().unwrap();
assert_eq!(end_state, replay_state);
}
#[test] #[test]
fn delete_blocks_and_states() { fn delete_blocks_and_states() {
let db_path = tempdir().unwrap(); let db_path = tempdir().unwrap();
@ -430,7 +616,7 @@ fn delete_blocks_and_states() {
// Delete faulty fork // Delete faulty fork
// Attempting to load those states should find them unavailable // Attempting to load those states should find them unavailable
for (state_root, slot) in for (state_root, slot) in
StateRootsIterator::new(store.clone(), &faulty_head_state).map(Result::unwrap) StateRootsIterator::new(&store, &faulty_head_state).map(Result::unwrap)
{ {
if slot <= unforked_blocks { if slot <= unforked_blocks {
break; break;
@ -441,7 +627,7 @@ fn delete_blocks_and_states() {
// Double-deleting should also be OK (deleting non-existent things is fine) // Double-deleting should also be OK (deleting non-existent things is fine)
for (state_root, slot) in for (state_root, slot) in
StateRootsIterator::new(store.clone(), &faulty_head_state).map(Result::unwrap) StateRootsIterator::new(&store, &faulty_head_state).map(Result::unwrap)
{ {
if slot <= unforked_blocks { if slot <= unforked_blocks {
break; break;
@ -451,7 +637,7 @@ fn delete_blocks_and_states() {
// Deleting the blocks from the fork should remove them completely // Deleting the blocks from the fork should remove them completely
for (block_root, slot) in for (block_root, slot) in
BlockRootsIterator::new(store.clone(), &faulty_head_state).map(Result::unwrap) BlockRootsIterator::new(&store, &faulty_head_state).map(Result::unwrap)
{ {
if slot <= unforked_blocks + 1 { if slot <= unforked_blocks + 1 {
break; break;

11
beacon_node/store/src/chunked_iter.rs
View File

@ -1,27 +1,26 @@
use crate::chunked_vector::{chunk_key, Chunk, Field}; use crate::chunked_vector::{chunk_key, Chunk, Field};
use crate::{HotColdDB, ItemStore}; use crate::{HotColdDB, ItemStore};
use slog::error; use slog::error;
use std::sync::Arc;
use types::{ChainSpec, EthSpec, Slot}; use types::{ChainSpec, EthSpec, Slot};
/// Iterator over the values of a `BeaconState` vector field (like `block_roots`). /// Iterator over the values of a `BeaconState` vector field (like `block_roots`).
/// ///
/// Uses the freezer DB's separate table to load the values. /// Uses the freezer DB's separate table to load the values.
pub struct ChunkedVectorIter<F, E, Hot, Cold> pub struct ChunkedVectorIter<'a, F, E, Hot, Cold>
where where
F: Field<E>, F: Field<E>,
E: EthSpec, E: EthSpec,
Hot: ItemStore<E>, Hot: ItemStore<E>,
Cold: ItemStore<E>, Cold: ItemStore<E>,
{ {
pub(crate) store: Arc<HotColdDB<E, Hot, Cold>>, pub(crate) store: &'a HotColdDB<E, Hot, Cold>,
current_vindex: usize, current_vindex: usize,
pub(crate) end_vindex: usize, pub(crate) end_vindex: usize,
next_cindex: usize, next_cindex: usize,
current_chunk: Chunk<F::Value>, current_chunk: Chunk<F::Value>,
} }
impl<F, E, Hot, Cold> ChunkedVectorIter<F, E, Hot, Cold> impl<'a, F, E, Hot, Cold> ChunkedVectorIter<'a, F, E, Hot, Cold>
where where
F: Field<E>, F: Field<E>,
E: EthSpec, E: EthSpec,
@ -35,7 +34,7 @@ where
/// `HotColdDB::get_latest_restore_point_slot`. We pass it as a parameter so that the caller can /// `HotColdDB::get_latest_restore_point_slot`. We pass it as a parameter so that the caller can
/// maintain a stable view of the database (see `HybridForwardsBlockRootsIterator`). /// maintain a stable view of the database (see `HybridForwardsBlockRootsIterator`).
pub fn new( pub fn new(
store: Arc<HotColdDB<E, Hot, Cold>>, store: &'a HotColdDB<E, Hot, Cold>,
start_vindex: usize, start_vindex: usize,
last_restore_point_slot: Slot, last_restore_point_slot: Slot,
spec: &ChainSpec, spec: &ChainSpec,
@ -57,7 +56,7 @@ where
} }
} }
impl<F, E, Hot, Cold> Iterator for ChunkedVectorIter<F, E, Hot, Cold> impl<'a, F, E, Hot, Cold> Iterator for ChunkedVectorIter<'a, F, E, Hot, Cold>
where where
F: Field<E>, F: Field<E>,
E: EthSpec, E: EthSpec,

8
beacon_node/store/src/errors.rs
View File

@ -2,6 +2,7 @@ use crate::chunked_vector::ChunkError;
use crate::config::StoreConfigError; use crate::config::StoreConfigError;
use crate::hot_cold_store::HotColdDBError; use crate::hot_cold_store::HotColdDBError;
use ssz::DecodeError; use ssz::DecodeError;
use state_processing::BlockReplayError;
use types::{BeaconStateError, Hash256, Slot}; use types::{BeaconStateError, Hash256, Slot};
pub type Result<T> = std::result::Result<T, Error>; pub type Result<T> = std::result::Result<T, Error>;
@ -39,6 +40,7 @@ pub enum Error {
expected: Hash256, expected: Hash256,
computed: Hash256, computed: Hash256,
}, },
BlockReplayError(BlockReplayError),
} }
pub trait HandleUnavailable<T> { pub trait HandleUnavailable<T> {
@ -91,6 +93,12 @@ impl From<StoreConfigError> for Error {
} }
} }
impl From<BlockReplayError> for Error {
fn from(e: BlockReplayError) -> Error {
Error::BlockReplayError(e)
}
}
#[derive(Debug)] #[derive(Debug)]
pub struct DBError { pub struct DBError {
pub message: String, pub message: String,

341
beacon_node/store/src/forwards_iter.rs
View File

@ -1,74 +1,33 @@
use crate::chunked_iter::ChunkedVectorIter; use crate::chunked_iter::ChunkedVectorIter;
use crate::chunked_vector::{BlockRoots, StateRoots}; use crate::chunked_vector::{BlockRoots, Field, StateRoots};
use crate::errors::{Error, Result}; use crate::errors::{Error, Result};
use crate::iter::{BlockRootsIterator, StateRootsIterator}; use crate::iter::{BlockRootsIterator, StateRootsIterator};
use crate::{HotColdDB, ItemStore}; use crate::{HotColdDB, ItemStore};
use itertools::process_results; use itertools::process_results;
use std::sync::Arc;
use types::{BeaconState, ChainSpec, EthSpec, Hash256, Slot}; use types::{BeaconState, ChainSpec, EthSpec, Hash256, Slot};
/// Forwards block roots iterator that makes use of the `block_roots` table in the freezer DB. pub type HybridForwardsBlockRootsIterator<'a, E, Hot, Cold> =
pub struct FrozenForwardsBlockRootsIterator<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> { HybridForwardsIterator<'a, E, BlockRoots, Hot, Cold>;
inner: ChunkedVectorIter<BlockRoots, E, Hot, Cold>, pub type HybridForwardsStateRootsIterator<'a, E, Hot, Cold> =
} HybridForwardsIterator<'a, E, StateRoots, Hot, Cold>;
/// Forwards block roots iterator that reverses a backwards iterator (only good for short ranges). /// Trait unifying `BlockRoots` and `StateRoots` for forward iteration.
pub struct SimpleForwardsBlockRootsIterator { pub trait Root<E: EthSpec>: Field<E, Value = Hash256> {
// Values from the backwards iterator (in slot descending order) fn simple_forwards_iterator<Hot: ItemStore<E>, Cold: ItemStore<E>>(
values: Vec<(Hash256, Slot)>, store: &HotColdDB<E, Hot, Cold>,
}
/// Fusion of the above two approaches to forwards iteration. Fast and efficient.
pub enum HybridForwardsBlockRootsIterator<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
PreFinalization {
iter: Box<FrozenForwardsBlockRootsIterator<E, Hot, Cold>>,
/// Data required by the `PostFinalization` iterator when we get to it.
continuation_data: Box<Option<(BeaconState<E>, Hash256)>>,
},
PostFinalization {
iter: SimpleForwardsBlockRootsIterator,
},
}
impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>
FrozenForwardsBlockRootsIterator<E, Hot, Cold>
{
pub fn new(
store: Arc<HotColdDB<E, Hot, Cold>>,
start_slot: Slot, start_slot: Slot,
last_restore_point_slot: Slot, end_state: BeaconState<E>,
spec: &ChainSpec, end_root: Hash256,
) -> Self { ) -> Result<SimpleForwardsIterator>;
Self {
inner: ChunkedVectorIter::new(
store,
start_slot.as_usize(),
last_restore_point_slot,
spec,
),
}
}
} }
impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Iterator impl<E: EthSpec> Root<E> for BlockRoots {
for FrozenForwardsBlockRootsIterator<E, Hot, Cold> fn simple_forwards_iterator<Hot: ItemStore<E>, Cold: ItemStore<E>>(
{ store: &HotColdDB<E, Hot, Cold>,
type Item = (Hash256, Slot);
fn next(&mut self) -> Option<Self::Item> {
self.inner
.next()
.map(|(slot, block_hash)| (block_hash, Slot::from(slot)))
}
}
impl SimpleForwardsBlockRootsIterator {
pub fn new<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>(
store: Arc<HotColdDB<E, Hot, Cold>>,
start_slot: Slot, start_slot: Slot,
end_state: BeaconState<E>, end_state: BeaconState<E>,
end_block_root: Hash256, end_block_root: Hash256,
) -> Result<Self> { ) -> Result<SimpleForwardsIterator> {
// Iterate backwards from the end state, stopping at the start slot. // Iterate backwards from the end state, stopping at the start slot.
let values = process_results( let values = process_results(
std::iter::once(Ok((end_block_root, end_state.slot()))) std::iter::once(Ok((end_block_root, end_state.slot())))
@ -78,129 +37,41 @@ impl SimpleForwardsBlockRootsIterator {
.collect::<Vec<_>>() .collect::<Vec<_>>()
}, },
)?; )?;
Ok(Self { values }) Ok(SimpleForwardsIterator { values })
} }
} }
impl Iterator for SimpleForwardsBlockRootsIterator { impl<E: EthSpec> Root<E> for StateRoots {
type Item = Result<(Hash256, Slot)>; fn simple_forwards_iterator<Hot: ItemStore<E>, Cold: ItemStore<E>>(
store: &HotColdDB<E, Hot, Cold>,
fn next(&mut self) -> Option<Self::Item> {
// Pop from the end of the vector to get the block roots in slot-ascending order.
Ok(self.values.pop()).transpose()
}
}
impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>
HybridForwardsBlockRootsIterator<E, Hot, Cold>
{
pub fn new(
store: Arc<HotColdDB<E, Hot, Cold>>,
start_slot: Slot, start_slot: Slot,
end_state: BeaconState<E>, end_state: BeaconState<E>,
end_block_root: Hash256, end_state_root: Hash256,
spec: &ChainSpec, ) -> Result<SimpleForwardsIterator> {
) -> Result<Self> { // Iterate backwards from the end state, stopping at the start slot.
use HybridForwardsBlockRootsIterator::*; let values = process_results(
std::iter::once(Ok((end_state_root, end_state.slot())))
let latest_restore_point_slot = store.get_latest_restore_point_slot(); .chain(StateRootsIterator::owned(store, end_state)),
|iter| {
let result = if start_slot < latest_restore_point_slot { iter.take_while(|(_, slot)| *slot >= start_slot)
PreFinalization { .collect::<Vec<_>>()
iter: Box::new(FrozenForwardsBlockRootsIterator::new( },
store, )?;
start_slot, Ok(SimpleForwardsIterator { values })
latest_restore_point_slot,
spec,
)),
continuation_data: Box::new(Some((end_state, end_block_root))),
}
} else {
PostFinalization {
iter: SimpleForwardsBlockRootsIterator::new(
store,
start_slot,
end_state,
end_block_root,
)?,
}
};
Ok(result)
}
fn do_next(&mut self) -> Result<Option<(Hash256, Slot)>> {
use HybridForwardsBlockRootsIterator::*;
match self {
PreFinalization {
iter,
continuation_data,
} => {
match iter.next() {
Some(x) => Ok(Some(x)),
// Once the pre-finalization iterator is consumed, transition
// to a post-finalization iterator beginning from the last slot
// of the pre iterator.
None => {
let (end_state, end_block_root) =
continuation_data.take().ok_or(Error::NoContinuationData)?;
*self = PostFinalization {
iter: SimpleForwardsBlockRootsIterator::new(
iter.inner.store.clone(),
Slot::from(iter.inner.end_vindex),
end_state,
end_block_root,
)?,
};
self.do_next()
}
}
}
PostFinalization { iter } => iter.next().transpose(),
}
} }
} }
impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Iterator /// Forwards root iterator that makes use of a flat field table in the freezer DB.
for HybridForwardsBlockRootsIterator<E, Hot, Cold> pub struct FrozenForwardsIterator<'a, E: EthSpec, F: Root<E>, Hot: ItemStore<E>, Cold: ItemStore<E>>
{ {
type Item = Result<(Hash256, Slot)>; inner: ChunkedVectorIter<'a, F, E, Hot, Cold>,
fn next(&mut self) -> Option<Self::Item> {
self.do_next().transpose()
}
} }
/// Forwards state roots iterator that makes use of the `state_roots` table in the freezer DB. impl<'a, E: EthSpec, F: Root<E>, Hot: ItemStore<E>, Cold: ItemStore<E>>
pub struct FrozenForwardsStateRootsIterator<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> { FrozenForwardsIterator<'a, E, F, Hot, Cold>
inner: ChunkedVectorIter<StateRoots, E, Hot, Cold>,
}
/// Forwards state roots iterator that reverses a backwards iterator (only good for short ranges).
pub struct SimpleForwardsStateRootsIterator {
// Values from the backwards iterator (in slot descending order)
values: Vec<(Hash256, Slot)>,
}
/// Fusion of the above two approaches to forwards iteration. Fast and efficient.
pub enum HybridForwardsStateRootsIterator<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
PreFinalization {
iter: Box<FrozenForwardsStateRootsIterator<E, Hot, Cold>>,
/// Data required by the `PostFinalization` iterator when we get to it.
continuation_data: Box<Option<(BeaconState<E>, Hash256)>>,
},
PostFinalization {
iter: SimpleForwardsStateRootsIterator,
},
}
impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>
FrozenForwardsStateRootsIterator<E, Hot, Cold>
{ {
pub fn new( pub fn new(
store: Arc<HotColdDB<E, Hot, Cold>>, store: &'a HotColdDB<E, Hot, Cold>,
start_slot: Slot, start_slot: Slot,
last_restore_point_slot: Slot, last_restore_point_slot: Slot,
spec: &ChainSpec, spec: &ChainSpec,
@ -216,39 +87,25 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>
} }
} }
impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Iterator impl<'a, E: EthSpec, F: Root<E>, Hot: ItemStore<E>, Cold: ItemStore<E>> Iterator
for FrozenForwardsStateRootsIterator<E, Hot, Cold> for FrozenForwardsIterator<'a, E, F, Hot, Cold>
{ {
type Item = (Hash256, Slot); type Item = (Hash256, Slot);
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
self.inner self.inner
.next() .next()
.map(|(slot, state_hash)| (state_hash, Slot::from(slot))) .map(|(slot, root)| (root, Slot::from(slot)))
} }
} }
impl SimpleForwardsStateRootsIterator { /// Forwards root iterator that reverses a backwards iterator (only good for short ranges).
pub fn new<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>( pub struct SimpleForwardsIterator {
store: Arc<HotColdDB<E, Hot, Cold>>, // Values from the backwards iterator (in slot descending order)
start_slot: Slot, values: Vec<(Hash256, Slot)>,
end_state: BeaconState<E>,
end_state_root: Hash256,
) -> Result<Self> {
// Iterate backwards from the end state, stopping at the start slot.
let values = process_results(
std::iter::once(Ok((end_state_root, end_state.slot())))
.chain(StateRootsIterator::owned(store, end_state)),
|iter| {
iter.take_while(|(_, slot)| *slot >= start_slot)
.collect::<Vec<_>>()
},
)?;
Ok(Self { values })
}
} }
impl Iterator for SimpleForwardsStateRootsIterator { impl Iterator for SimpleForwardsIterator {
type Item = Result<(Hash256, Slot)>; type Item = Result<(Hash256, Slot)>;
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
@ -257,38 +114,75 @@ impl Iterator for SimpleForwardsStateRootsIterator {
} }
} }
impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> /// Fusion of the above two approaches to forwards iteration. Fast and efficient.
HybridForwardsStateRootsIterator<E, Hot, Cold> pub enum HybridForwardsIterator<'a, E: EthSpec, F: Root<E>, Hot: ItemStore<E>, Cold: ItemStore<E>> {
{ PreFinalization {
pub fn new( iter: Box<FrozenForwardsIterator<'a, E, F, Hot, Cold>>,
store: Arc<HotColdDB<E, Hot, Cold>>, /// Data required by the `PostFinalization` iterator when we get to it.
continuation_data: Option<Box<(BeaconState<E>, Hash256)>>,
},
PostFinalizationLazy {
continuation_data: Option<Box<(BeaconState<E>, Hash256)>>,
store: &'a HotColdDB<E, Hot, Cold>,
start_slot: Slot, start_slot: Slot,
end_state: BeaconState<E>, },
end_state_root: Hash256, PostFinalization {
iter: SimpleForwardsIterator,
},
}
impl<'a, E: EthSpec, F: Root<E>, Hot: ItemStore<E>, Cold: ItemStore<E>>
HybridForwardsIterator<'a, E, F, Hot, Cold>
{
/// Construct a new hybrid iterator.
///
/// The `get_state` closure should return a beacon state and final block/state root to backtrack
/// from in the case where the iterated range does not lie entirely within the frozen portion of
/// the database. If an `end_slot` is provided and it is before the database's latest restore
/// point slot then the `get_state` closure will not be called at all.
///
/// It is OK for `get_state` to hold a lock while this function is evaluated, as the returned
/// iterator is as lazy as possible and won't do any work apart from calling `get_state`.
///
/// Conversely, if `get_state` does extensive work (e.g. loading data from disk) then this
/// function may block for some time while `get_state` runs.
pub fn new(
store: &'a HotColdDB<E, Hot, Cold>,
start_slot: Slot,
end_slot: Option<Slot>,
get_state: impl FnOnce() -> (BeaconState<E>, Hash256),
spec: &ChainSpec, spec: &ChainSpec,
) -> Result<Self> { ) -> Result<Self> {
use HybridForwardsStateRootsIterator::*; use HybridForwardsIterator::*;
let latest_restore_point_slot = store.get_latest_restore_point_slot(); let latest_restore_point_slot = store.get_latest_restore_point_slot();
let result = if start_slot < latest_restore_point_slot { let result = if start_slot < latest_restore_point_slot {
PreFinalization { let iter = Box::new(FrozenForwardsIterator::new(
iter: Box::new(FrozenForwardsStateRootsIterator::new(
store, store,
start_slot, start_slot,
latest_restore_point_slot, latest_restore_point_slot,
spec, spec,
)), ));
continuation_data: Box::new(Some((end_state, end_state_root))),
// No continuation data is needed if the forwards iterator plans to halt before
// `end_slot`. If it tries to continue further a `NoContinuationData` error will be
// returned.
let continuation_data =
if end_slot.map_or(false, |end_slot| end_slot < latest_restore_point_slot) {
None
} else {
Some(Box::new(get_state()))
};
PreFinalization {
iter,
continuation_data,
} }
} else { } else {
PostFinalization { PostFinalizationLazy {
iter: SimpleForwardsStateRootsIterator::new( continuation_data: Some(Box::new(get_state())),
store, store,
start_slot, start_slot,
end_state,
end_state_root,
)?,
} }
}; };
@ -296,7 +190,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>
} }
fn do_next(&mut self) -> Result<Option<(Hash256, Slot)>> { fn do_next(&mut self) -> Result<Option<(Hash256, Slot)>> {
use HybridForwardsStateRootsIterator::*; use HybridForwardsIterator::*;
match self { match self {
PreFinalization { PreFinalization {
@ -309,28 +203,39 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>
// to a post-finalization iterator beginning from the last slot // to a post-finalization iterator beginning from the last slot
// of the pre iterator. // of the pre iterator.
None => { None => {
let (end_state, end_state_root) = let continuation_data = continuation_data.take();
continuation_data.take().ok_or(Error::NoContinuationData)?; let store = iter.inner.store;
let start_slot = Slot::from(iter.inner.end_vindex);
*self = PostFinalization { *self = PostFinalizationLazy {
iter: SimpleForwardsStateRootsIterator::new( continuation_data,
iter.inner.store.clone(), store,
Slot::from(iter.inner.end_vindex), start_slot,
end_state,
end_state_root,
)?,
}; };
self.do_next() self.do_next()
} }
} }
} }
PostFinalizationLazy {
continuation_data,
store,
start_slot,
} => {
let (end_state, end_root) =
*continuation_data.take().ok_or(Error::NoContinuationData)?;
*self = PostFinalization {
iter: F::simple_forwards_iterator(store, *start_slot, end_state, end_root)?,
};
self.do_next()
}
PostFinalization { iter } => iter.next().transpose(), PostFinalization { iter } => iter.next().transpose(),
} }
} }
} }
impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Iterator impl<'a, E: EthSpec, F: Root<E>, Hot: ItemStore<E>, Cold: ItemStore<E>> Iterator
for HybridForwardsStateRootsIterator<E, Hot, Cold> for HybridForwardsIterator<'a, E, F, Hot, Cold>
{ {
type Item = Result<(Hash256, Slot)>; type Item = Result<(Hash256, Slot)>;

201
beacon_node/store/src/hot_cold_store.rs
View File

@ -22,12 +22,11 @@ use leveldb::iterator::LevelDBIterator;
use lru::LruCache; use lru::LruCache;
use parking_lot::{Mutex, RwLock}; use parking_lot::{Mutex, RwLock};
use serde_derive::{Deserialize, Serialize}; use serde_derive::{Deserialize, Serialize};
use slog::{debug, error, info, trace, Logger}; use slog::{debug, error, info, trace, warn, Logger};
use ssz::{Decode, Encode}; use ssz::{Decode, Encode};
use ssz_derive::{Decode, Encode}; use ssz_derive::{Decode, Encode};
use state_processing::{ use state_processing::{
per_block_processing, per_slot_processing, BlockProcessingError, BlockSignatureStrategy, BlockProcessingError, BlockReplayer, SlotProcessingError, StateRootStrategy,
SlotProcessingError,
}; };
use std::cmp::min; use std::cmp::min;
use std::convert::TryInto; use std::convert::TryInto;
@ -37,16 +36,6 @@ use std::sync::Arc;
use std::time::Duration; use std::time::Duration;
use types::*; use types::*;
/// Defines how blocks should be replayed on states.
#[derive(PartialEq)]
pub enum BlockReplay {
/// Perform all transitions faithfully to the specification.
Accurate,
/// Don't compute state roots, eventually computing an invalid beacon state that can only be
/// used for obtaining shuffling.
InconsistentStateRoots,
}
/// On-disk database that stores finalized states efficiently. /// On-disk database that stores finalized states efficiently.
/// ///
/// Stores vector fields like the `block_roots` and `state_roots` separately, and only stores /// Stores vector fields like the `block_roots` and `state_roots` separately, and only stores
@ -373,10 +362,10 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
// chain. This way we avoid returning a state that doesn't match `state_root`. // chain. This way we avoid returning a state that doesn't match `state_root`.
self.load_cold_state(state_root) self.load_cold_state(state_root)
} else { } else {
self.load_hot_state(state_root, BlockReplay::Accurate) self.load_hot_state(state_root, StateRootStrategy::Accurate)
} }
} else { } else {
match self.load_hot_state(state_root, BlockReplay::Accurate)? { match self.load_hot_state(state_root, StateRootStrategy::Accurate)? {
Some(state) => Ok(Some(state)), Some(state) => Ok(Some(state)),
None => self.load_cold_state(state_root), None => self.load_cold_state(state_root),
} }
@ -414,7 +403,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
} }
.into()) .into())
} else { } else {
self.load_hot_state(state_root, BlockReplay::InconsistentStateRoots) self.load_hot_state(state_root, StateRootStrategy::Inconsistent)
} }
} }
@ -439,23 +428,55 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
} }
pub fn forwards_block_roots_iterator( pub fn forwards_block_roots_iterator(
store: Arc<Self>, &self,
start_slot: Slot, start_slot: Slot,
end_state: BeaconState<E>, end_state: BeaconState<E>,
end_block_root: Hash256, end_block_root: Hash256,
spec: &ChainSpec, spec: &ChainSpec,
) -> Result<impl Iterator<Item = Result<(Hash256, Slot), Error>>, Error> { ) -> Result<impl Iterator<Item = Result<(Hash256, Slot), Error>> + '_, Error> {
HybridForwardsBlockRootsIterator::new(store, start_slot, end_state, end_block_root, spec) HybridForwardsBlockRootsIterator::new(
self,
start_slot,
None,
|| (end_state, end_block_root),
spec,
)
}
pub fn forwards_block_roots_iterator_until(
&self,
start_slot: Slot,
end_slot: Slot,
get_state: impl FnOnce() -> (BeaconState<E>, Hash256),
spec: &ChainSpec,
) -> Result<HybridForwardsBlockRootsIterator<E, Hot, Cold>, Error> {
HybridForwardsBlockRootsIterator::new(self, start_slot, Some(end_slot), get_state, spec)
} }
pub fn forwards_state_roots_iterator( pub fn forwards_state_roots_iterator(
store: Arc<Self>, &self,
start_slot: Slot, start_slot: Slot,
end_state_root: Hash256, end_state_root: Hash256,
end_state: BeaconState<E>, end_state: BeaconState<E>,
spec: &ChainSpec, spec: &ChainSpec,
) -> Result<impl Iterator<Item = Result<(Hash256, Slot), Error>>, Error> { ) -> Result<impl Iterator<Item = Result<(Hash256, Slot), Error>> + '_, Error> {
HybridForwardsStateRootsIterator::new(store, start_slot, end_state, end_state_root, spec) HybridForwardsStateRootsIterator::new(
self,
start_slot,
None,
|| (end_state, end_state_root),
spec,
)
}
pub fn forwards_state_roots_iterator_until(
&self,
start_slot: Slot,
end_slot: Slot,
get_state: impl FnOnce() -> (BeaconState<E>, Hash256),
spec: &ChainSpec,
) -> Result<HybridForwardsStateRootsIterator<E, Hot, Cold>, Error> {
HybridForwardsStateRootsIterator::new(self, start_slot, Some(end_slot), get_state, spec)
} }
/// Load an epoch boundary state by using the hot state summary look-up. /// Load an epoch boundary state by using the hot state summary look-up.
@ -472,10 +493,10 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
{ {
// NOTE: minor inefficiency here because we load an unnecessary hot state summary // NOTE: minor inefficiency here because we load an unnecessary hot state summary
// //
// `BlockReplay` should be irrelevant here since we never replay blocks for an epoch // `StateRootStrategy` should be irrelevant here since we never replay blocks for an epoch
// boundary state in the hot DB. // boundary state in the hot DB.
let state = self let state = self
.load_hot_state(&epoch_boundary_state_root, BlockReplay::Accurate)? .load_hot_state(&epoch_boundary_state_root, StateRootStrategy::Accurate)?
.ok_or(HotColdDBError::MissingEpochBoundaryState( .ok_or(HotColdDBError::MissingEpochBoundaryState(
epoch_boundary_state_root, epoch_boundary_state_root,
))?; ))?;
@ -620,7 +641,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
pub fn load_hot_state( pub fn load_hot_state(
&self, &self,
state_root: &Hash256, state_root: &Hash256,
block_replay: BlockReplay, state_root_strategy: StateRootStrategy,
) -> Result<Option<BeaconState<E>>, Error> { ) -> Result<Option<BeaconState<E>>, Error> {
metrics::inc_counter(&metrics::BEACON_STATE_HOT_GET_COUNT); metrics::inc_counter(&metrics::BEACON_STATE_HOT_GET_COUNT);
@ -648,7 +669,13 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
} else { } else {
let blocks = let blocks =
self.load_blocks_to_replay(boundary_state.slot(), slot, latest_block_root)?; self.load_blocks_to_replay(boundary_state.slot(), slot, latest_block_root)?;
self.replay_blocks(boundary_state, blocks, slot, block_replay)? self.replay_blocks(
boundary_state,
blocks,
slot,
no_state_root_iter(),
state_root_strategy,
)?
}; };
Ok(Some(state)) Ok(Some(state))
@ -777,7 +804,22 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
)?; )?;
// 3. Replay the blocks on top of the low restore point. // 3. Replay the blocks on top of the low restore point.
self.replay_blocks(low_restore_point, blocks, slot, BlockReplay::Accurate) // Use a forwards state root iterator to avoid doing any tree hashing.
// The state root of the high restore point should never be used, so is safely set to 0.
let state_root_iter = self.forwards_state_roots_iterator_until(
low_restore_point.slot(),
slot,
|| (high_restore_point, Hash256::zero()),
&self.spec,
)?;
self.replay_blocks(
low_restore_point,
blocks,
slot,
Some(state_root_iter),
StateRootStrategy::Accurate,
)
} }
/// Get the restore point with the given index, or if it is out of bounds, the split state. /// Get the restore point with the given index, or if it is out of bounds, the split state.
@ -860,89 +902,35 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
/// to have any caches built, beyond those immediately required by block processing. /// to have any caches built, beyond those immediately required by block processing.
fn replay_blocks( fn replay_blocks(
&self, &self,
mut state: BeaconState<E>, state: BeaconState<E>,
mut blocks: Vec<SignedBeaconBlock<E>>, blocks: Vec<SignedBeaconBlock<E>>,
target_slot: Slot, target_slot: Slot,
block_replay: BlockReplay, state_root_iter: Option<impl Iterator<Item = Result<(Hash256, Slot), Error>>>,
state_root_strategy: StateRootStrategy,
) -> Result<BeaconState<E>, Error> { ) -> Result<BeaconState<E>, Error> {
if block_replay == BlockReplay::InconsistentStateRoots { let mut block_replayer = BlockReplayer::new(state, &self.spec)
for i in 0..blocks.len() { .state_root_strategy(state_root_strategy)
let prev_block_root = if i > 0 { .no_signature_verification()
blocks[i - 1].canonical_root() .minimal_block_root_verification();
} else {
// Not read.
Hash256::zero()
};
let (state_root, parent_root) = match &mut blocks[i] { let have_state_root_iterator = state_root_iter.is_some();
SignedBeaconBlock::Base(block) => ( if let Some(state_root_iter) = state_root_iter {
&mut block.message.state_root, block_replayer = block_replayer.state_root_iter(state_root_iter);
&mut block.message.parent_root,
),
SignedBeaconBlock::Altair(block) => (
&mut block.message.state_root,
&mut block.message.parent_root,
),
SignedBeaconBlock::Merge(block) => (
&mut block.message.state_root,
&mut block.message.parent_root,
),
};
*state_root = Hash256::zero();
if i > 0 {
*parent_root = prev_block_root;
}
}
} }
let state_root_from_prev_block = |i: usize, state: &BeaconState<E>| { block_replayer
if i > 0 { .apply_blocks(blocks, Some(target_slot))
let prev_block = blocks[i - 1].message(); .map(|block_replayer| {
if prev_block.slot() == state.slot() { if have_state_root_iterator && block_replayer.state_root_miss() {
Some(prev_block.state_root()) warn!(
} else { self.log,
None "State root iterator miss";
} "slot" => target_slot,
} else { );
None
}
};
for (i, block) in blocks.iter().enumerate() {
if block.slot() <= state.slot() {
continue;
} }
while state.slot() < block.slot() { block_replayer.into_state()
let state_root = match block_replay { })
BlockReplay::Accurate => state_root_from_prev_block(i, &state),
BlockReplay::InconsistentStateRoots => Some(Hash256::zero()),
};
per_slot_processing(&mut state, state_root, &self.spec)
.map_err(HotColdDBError::BlockReplaySlotError)?;
}
per_block_processing(
&mut state,
block,
None,
BlockSignatureStrategy::NoVerification,
&self.spec,
)
.map_err(HotColdDBError::BlockReplayBlockError)?;
}
while state.slot() < target_slot {
let state_root = match block_replay {
BlockReplay::Accurate => state_root_from_prev_block(blocks.len(), &state),
BlockReplay::InconsistentStateRoots => Some(Hash256::zero()),
};
per_slot_processing(&mut state, state_root, &self.spec)
.map_err(HotColdDBError::BlockReplaySlotError)?;
}
Ok(state)
} }
/// Fetch a copy of the current split slot from memory. /// Fetch a copy of the current split slot from memory.
@ -1309,7 +1297,7 @@ pub fn migrate_database<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>(
// 1. Copy all of the states between the head and the split slot, from the hot DB // 1. Copy all of the states between the head and the split slot, from the hot DB
// to the cold DB. // to the cold DB.
let state_root_iter = StateRootsIterator::new(store.clone(), frozen_head); let state_root_iter = StateRootsIterator::new(&store, frozen_head);
for maybe_pair in state_root_iter.take_while(|result| match result { for maybe_pair in state_root_iter.take_while(|result| match result {
Ok((_, slot)) => { Ok((_, slot)) => {
slot >= &current_split_slot slot >= &current_split_slot
@ -1423,6 +1411,11 @@ impl StoreItem for Split {
} }
} }
/// Type hint.
fn no_state_root_iter() -> Option<std::iter::Empty<Result<(Hash256, Slot), Error>>> {
None
}
/// Struct for summarising a state in the hot database. /// Struct for summarising a state in the hot database.
/// ///
/// Allows full reconstruction by replaying blocks. /// Allows full reconstruction by replaying blocks.

51
beacon_node/store/src/iter.rs
View File

@ -2,7 +2,6 @@ use crate::errors::HandleUnavailable;
use crate::{Error, HotColdDB, ItemStore}; use crate::{Error, HotColdDB, ItemStore};
use std::borrow::Cow; use std::borrow::Cow;
use std::marker::PhantomData; use std::marker::PhantomData;
use std::sync::Arc;
use types::{ use types::{
typenum::Unsigned, BeaconState, BeaconStateError, EthSpec, Hash256, SignedBeaconBlock, Slot, typenum::Unsigned, BeaconState, BeaconStateError, EthSpec, Hash256, SignedBeaconBlock, Slot,
}; };
@ -13,19 +12,19 @@ use types::{
/// ///
/// It is assumed that all ancestors for this object are stored in the database. If this is not the /// It is assumed that all ancestors for this object are stored in the database. If this is not the
/// case, the iterator will start returning `None` prior to genesis. /// case, the iterator will start returning `None` prior to genesis.
pub trait AncestorIter<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>, I: Iterator> { pub trait AncestorIter<'a, E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>, I: Iterator> {
/// Returns an iterator over the roots of the ancestors of `self`. /// Returns an iterator over the roots of the ancestors of `self`.
fn try_iter_ancestor_roots(&self, store: Arc<HotColdDB<E, Hot, Cold>>) -> Option<I>; fn try_iter_ancestor_roots(&self, store: &'a HotColdDB<E, Hot, Cold>) -> Option<I>;
} }
impl<'a, E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> impl<'a, E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>
AncestorIter<E, Hot, Cold, BlockRootsIterator<'a, E, Hot, Cold>> for SignedBeaconBlock<E> AncestorIter<'a, E, Hot, Cold, BlockRootsIterator<'a, E, Hot, Cold>> for SignedBeaconBlock<E>
{ {
/// Iterates across all available prior block roots of `self`, starting at the most recent and ending /// Iterates across all available prior block roots of `self`, starting at the most recent and ending
/// at genesis. /// at genesis.
fn try_iter_ancestor_roots( fn try_iter_ancestor_roots(
&self, &self,
store: Arc<HotColdDB<E, Hot, Cold>>, store: &'a HotColdDB<E, Hot, Cold>,
) -> Option<BlockRootsIterator<'a, E, Hot, Cold>> { ) -> Option<BlockRootsIterator<'a, E, Hot, Cold>> {
let state = store let state = store
.get_state(&self.message().state_root(), Some(self.slot())) .get_state(&self.message().state_root(), Some(self.slot()))
@ -36,13 +35,13 @@ impl<'a, E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>
} }
impl<'a, E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> impl<'a, E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>
AncestorIter<E, Hot, Cold, StateRootsIterator<'a, E, Hot, Cold>> for BeaconState<E> AncestorIter<'a, E, Hot, Cold, StateRootsIterator<'a, E, Hot, Cold>> for BeaconState<E>
{ {
/// Iterates across all available prior state roots of `self`, starting at the most recent and ending /// Iterates across all available prior state roots of `self`, starting at the most recent and ending
/// at genesis. /// at genesis.
fn try_iter_ancestor_roots( fn try_iter_ancestor_roots(
&self, &self,
store: Arc<HotColdDB<E, Hot, Cold>>, store: &'a HotColdDB<E, Hot, Cold>,
) -> Option<StateRootsIterator<'a, E, Hot, Cold>> { ) -> Option<StateRootsIterator<'a, E, Hot, Cold>> {
// The `self.clone()` here is wasteful. // The `self.clone()` here is wasteful.
Some(StateRootsIterator::owned(store, self.clone())) Some(StateRootsIterator::owned(store, self.clone()))
@ -64,13 +63,13 @@ impl<'a, T: EthSpec, Hot: ItemStore<T>, Cold: ItemStore<T>> Clone
} }
impl<'a, T: EthSpec, Hot: ItemStore<T>, Cold: ItemStore<T>> StateRootsIterator<'a, T, Hot, Cold> { impl<'a, T: EthSpec, Hot: ItemStore<T>, Cold: ItemStore<T>> StateRootsIterator<'a, T, Hot, Cold> {
pub fn new(store: Arc<HotColdDB<T, Hot, Cold>>, beacon_state: &'a BeaconState<T>) -> Self { pub fn new(store: &'a HotColdDB<T, Hot, Cold>, beacon_state: &'a BeaconState<T>) -> Self {
Self { Self {
inner: RootsIterator::new(store, beacon_state), inner: RootsIterator::new(store, beacon_state),
} }
} }
pub fn owned(store: Arc<HotColdDB<T, Hot, Cold>>, beacon_state: BeaconState<T>) -> Self { pub fn owned(store: &'a HotColdDB<T, Hot, Cold>, beacon_state: BeaconState<T>) -> Self {
Self { Self {
inner: RootsIterator::owned(store, beacon_state), inner: RootsIterator::owned(store, beacon_state),
} }
@ -113,21 +112,21 @@ impl<'a, T: EthSpec, Hot: ItemStore<T>, Cold: ItemStore<T>> Clone
impl<'a, T: EthSpec, Hot: ItemStore<T>, Cold: ItemStore<T>> BlockRootsIterator<'a, T, Hot, Cold> { impl<'a, T: EthSpec, Hot: ItemStore<T>, Cold: ItemStore<T>> BlockRootsIterator<'a, T, Hot, Cold> {
/// Create a new iterator over all block roots in the given `beacon_state` and prior states. /// Create a new iterator over all block roots in the given `beacon_state` and prior states.
pub fn new(store: Arc<HotColdDB<T, Hot, Cold>>, beacon_state: &'a BeaconState<T>) -> Self { pub fn new(store: &'a HotColdDB<T, Hot, Cold>, beacon_state: &'a BeaconState<T>) -> Self {
Self { Self {
inner: RootsIterator::new(store, beacon_state), inner: RootsIterator::new(store, beacon_state),
} }
} }
/// Create a new iterator over all block roots in the given `beacon_state` and prior states. /// Create a new iterator over all block roots in the given `beacon_state` and prior states.
pub fn owned(store: Arc<HotColdDB<T, Hot, Cold>>, beacon_state: BeaconState<T>) -> Self { pub fn owned(store: &'a HotColdDB<T, Hot, Cold>, beacon_state: BeaconState<T>) -> Self {
Self { Self {
inner: RootsIterator::owned(store, beacon_state), inner: RootsIterator::owned(store, beacon_state),
} }
} }
pub fn from_block( pub fn from_block(
store: Arc<HotColdDB<T, Hot, Cold>>, store: &'a HotColdDB<T, Hot, Cold>,
block_hash: Hash256, block_hash: Hash256,
) -> Result<Self, Error> { ) -> Result<Self, Error> {
Ok(Self { Ok(Self {
@ -150,7 +149,7 @@ impl<'a, T: EthSpec, Hot: ItemStore<T>, Cold: ItemStore<T>> Iterator
/// Iterator over state and block roots that backtracks using the vectors from a `BeaconState`. /// Iterator over state and block roots that backtracks using the vectors from a `BeaconState`.
pub struct RootsIterator<'a, T: EthSpec, Hot: ItemStore<T>, Cold: ItemStore<T>> { pub struct RootsIterator<'a, T: EthSpec, Hot: ItemStore<T>, Cold: ItemStore<T>> {
store: Arc<HotColdDB<T, Hot, Cold>>, store: &'a HotColdDB<T, Hot, Cold>,
beacon_state: Cow<'a, BeaconState<T>>, beacon_state: Cow<'a, BeaconState<T>>,
slot: Slot, slot: Slot,
} }
@ -160,7 +159,7 @@ impl<'a, T: EthSpec, Hot: ItemStore<T>, Cold: ItemStore<T>> Clone
{ {
fn clone(&self) -> Self { fn clone(&self) -> Self {
Self { Self {
store: self.store.clone(), store: self.store,
beacon_state: self.beacon_state.clone(), beacon_state: self.beacon_state.clone(),
slot: self.slot, slot: self.slot,
} }
@ -168,7 +167,7 @@ impl<'a, T: EthSpec, Hot: ItemStore<T>, Cold: ItemStore<T>> Clone
} }
impl<'a, T: EthSpec, Hot: ItemStore<T>, Cold: ItemStore<T>> RootsIterator<'a, T, Hot, Cold> { impl<'a, T: EthSpec, Hot: ItemStore<T>, Cold: ItemStore<T>> RootsIterator<'a, T, Hot, Cold> {
pub fn new(store: Arc<HotColdDB<T, Hot, Cold>>, beacon_state: &'a BeaconState<T>) -> Self { pub fn new(store: &'a HotColdDB<T, Hot, Cold>, beacon_state: &'a BeaconState<T>) -> Self {
Self { Self {
store, store,
slot: beacon_state.slot(), slot: beacon_state.slot(),
@ -176,7 +175,7 @@ impl<'a, T: EthSpec, Hot: ItemStore<T>, Cold: ItemStore<T>> RootsIterator<'a, T,
} }
} }
pub fn owned(store: Arc<HotColdDB<T, Hot, Cold>>, beacon_state: BeaconState<T>) -> Self { pub fn owned(store: &'a HotColdDB<T, Hot, Cold>, beacon_state: BeaconState<T>) -> Self {
Self { Self {
store, store,
slot: beacon_state.slot(), slot: beacon_state.slot(),
@ -185,7 +184,7 @@ impl<'a, T: EthSpec, Hot: ItemStore<T>, Cold: ItemStore<T>> RootsIterator<'a, T,
} }
pub fn from_block( pub fn from_block(
store: Arc<HotColdDB<T, Hot, Cold>>, store: &'a HotColdDB<T, Hot, Cold>,
block_hash: Hash256, block_hash: Hash256,
) -> Result<Self, Error> { ) -> Result<Self, Error> {
let block = store let block = store
@ -310,14 +309,14 @@ pub struct BlockIterator<'a, T: EthSpec, Hot: ItemStore<T>, Cold: ItemStore<T>>
impl<'a, T: EthSpec, Hot: ItemStore<T>, Cold: ItemStore<T>> BlockIterator<'a, T, Hot, Cold> { impl<'a, T: EthSpec, Hot: ItemStore<T>, Cold: ItemStore<T>> BlockIterator<'a, T, Hot, Cold> {
/// Create a new iterator over all blocks in the given `beacon_state` and prior states. /// Create a new iterator over all blocks in the given `beacon_state` and prior states.
pub fn new(store: Arc<HotColdDB<T, Hot, Cold>>, beacon_state: &'a BeaconState<T>) -> Self { pub fn new(store: &'a HotColdDB<T, Hot, Cold>, beacon_state: &'a BeaconState<T>) -> Self {
Self { Self {
roots: BlockRootsIterator::new(store, beacon_state), roots: BlockRootsIterator::new(store, beacon_state),
} }
} }
/// Create a new iterator over all blocks in the given `beacon_state` and prior states. /// Create a new iterator over all blocks in the given `beacon_state` and prior states.
pub fn owned(store: Arc<HotColdDB<T, Hot, Cold>>, beacon_state: BeaconState<T>) -> Self { pub fn owned(store: &'a HotColdDB<T, Hot, Cold>, beacon_state: BeaconState<T>) -> Self {
Self { Self {
roots: BlockRootsIterator::owned(store, beacon_state), roots: BlockRootsIterator::owned(store, beacon_state),
} }
@ -397,9 +396,8 @@ mod test {
#[test] #[test]
fn block_root_iter() { fn block_root_iter() {
let log = NullLoggerBuilder.build().unwrap(); let log = NullLoggerBuilder.build().unwrap();
let store = Arc::new( let store =
HotColdDB::open_ephemeral(Config::default(), ChainSpec::minimal(), log).unwrap(), HotColdDB::open_ephemeral(Config::default(), ChainSpec::minimal(), log).unwrap();
);
let slots_per_historical_root = MainnetEthSpec::slots_per_historical_root(); let slots_per_historical_root = MainnetEthSpec::slots_per_historical_root();
let mut state_a: BeaconState<MainnetEthSpec> = get_state(); let mut state_a: BeaconState<MainnetEthSpec> = get_state();
@ -422,7 +420,7 @@ mod test {
state_b.state_roots_mut()[0] = state_a_root; state_b.state_roots_mut()[0] = state_a_root;
store.put_state(&state_a_root, &state_a).unwrap(); store.put_state(&state_a_root, &state_a).unwrap();
let iter = BlockRootsIterator::new(store, &state_b); let iter = BlockRootsIterator::new(&store, &state_b);
assert!( assert!(
iter.clone() iter.clone()
@ -445,9 +443,8 @@ mod test {
#[test] #[test]
fn state_root_iter() { fn state_root_iter() {
let log = NullLoggerBuilder.build().unwrap(); let log = NullLoggerBuilder.build().unwrap();
let store = Arc::new( let store =
HotColdDB::open_ephemeral(Config::default(), ChainSpec::minimal(), log).unwrap(), HotColdDB::open_ephemeral(Config::default(), ChainSpec::minimal(), log).unwrap();
);
let slots_per_historical_root = MainnetEthSpec::slots_per_historical_root(); let slots_per_historical_root = MainnetEthSpec::slots_per_historical_root();
let mut state_a: BeaconState<MainnetEthSpec> = get_state(); let mut state_a: BeaconState<MainnetEthSpec> = get_state();
@ -475,7 +472,7 @@ mod test {
store.put_state(&state_a_root, &state_a).unwrap(); store.put_state(&state_a_root, &state_a).unwrap();
store.put_state(&state_b_root, &state_b).unwrap(); store.put_state(&state_b_root, &state_b).unwrap();
let iter = StateRootsIterator::new(store, &state_b); let iter = StateRootsIterator::new(&store, &state_b);
assert!( assert!(
iter.clone() iter.clone()

2
beacon_node/store/src/lib.rs
View File

@ -30,7 +30,7 @@ pub mod iter;
pub use self::chunk_writer::ChunkWriter; pub use self::chunk_writer::ChunkWriter;
pub use self::config::StoreConfig; pub use self::config::StoreConfig;
pub use self::hot_cold_store::{BlockReplay, HotColdDB, HotStateSummary, Split}; pub use self::hot_cold_store::{HotColdDB, HotStateSummary, Split};
pub use self::leveldb_store::LevelDB; pub use self::leveldb_store::LevelDB;
pub use self::memory_store::MemoryStore; pub use self::memory_store::MemoryStore;
pub use self::partial_beacon_state::PartialBeaconState; pub use self::partial_beacon_state::PartialBeaconState;

8
beacon_node/store/src/reconstruct.rs
View File

@ -3,7 +3,9 @@ use crate::hot_cold_store::{HotColdDB, HotColdDBError};
use crate::{Error, ItemStore, KeyValueStore}; use crate::{Error, ItemStore, KeyValueStore};
use itertools::{process_results, Itertools}; use itertools::{process_results, Itertools};
use slog::info; use slog::info;
use state_processing::{per_block_processing, per_slot_processing, BlockSignatureStrategy}; use state_processing::{
per_block_processing, per_slot_processing, BlockSignatureStrategy, VerifyBlockRoot,
};
use std::sync::Arc; use std::sync::Arc;
use types::{EthSpec, Hash256}; use types::{EthSpec, Hash256};
@ -48,8 +50,7 @@ where
// Use a dummy root, as we never read the block for the upper limit state. // Use a dummy root, as we never read the block for the upper limit state.
let upper_limit_block_root = Hash256::repeat_byte(0xff); let upper_limit_block_root = Hash256::repeat_byte(0xff);
let block_root_iter = Self::forwards_block_roots_iterator( let block_root_iter = self.forwards_block_roots_iterator(
self.clone(),
lower_limit_slot, lower_limit_slot,
upper_limit_state, upper_limit_state,
upper_limit_block_root, upper_limit_block_root,
@ -91,6 +92,7 @@ where
&block, &block,
Some(block_root), Some(block_root),
BlockSignatureStrategy::NoVerification, BlockSignatureStrategy::NoVerification,
VerifyBlockRoot::True,
&self.spec, &self.spec,
) )
.map_err(HotColdDBError::BlockReplayBlockError)?; .map_err(HotColdDBError::BlockReplayBlockError)?;

313
consensus/state_processing/src/block_replayer.rs Normal file
View File

@ -0,0 +1,313 @@
use crate::{
per_block_processing, per_epoch_processing::EpochProcessingSummary, per_slot_processing,
BlockProcessingError, BlockSignatureStrategy, SlotProcessingError, VerifyBlockRoot,
};
use std::marker::PhantomData;
use types::{BeaconState, ChainSpec, EthSpec, Hash256, SignedBeaconBlock, Slot};
type PreBlockHook<'a, E, Error> =
Box<dyn FnMut(&mut BeaconState<E>, &SignedBeaconBlock<E>) -> Result<(), Error> + 'a>;
type PostBlockHook<'a, E, Error> = PreBlockHook<'a, E, Error>;
type PreSlotHook<'a, E, Error> = Box<dyn FnMut(&mut BeaconState<E>) -> Result<(), Error> + 'a>;
type PostSlotHook<'a, E, Error> = Box<
dyn FnMut(&mut BeaconState<E>, Option<EpochProcessingSummary<E>>, bool) -> Result<(), Error>
+ 'a,
>;
type StateRootIterDefault<Error> = std::iter::Empty<Result<(Hash256, Slot), Error>>;
/// Efficiently apply blocks to a state while configuring various parameters.
///
/// Usage follows a builder pattern.
pub struct BlockReplayer<
'a,
Spec: EthSpec,
Error = BlockReplayError,
StateRootIter = StateRootIterDefault<Error>,
> {
state: BeaconState<Spec>,
spec: &'a ChainSpec,
state_root_strategy: StateRootStrategy,
block_sig_strategy: BlockSignatureStrategy,
verify_block_root: Option<VerifyBlockRoot>,
pre_block_hook: Option<PreBlockHook<'a, Spec, Error>>,
post_block_hook: Option<PostBlockHook<'a, Spec, Error>>,
pre_slot_hook: Option<PreSlotHook<'a, Spec, Error>>,
post_slot_hook: Option<PostSlotHook<'a, Spec, Error>>,
state_root_iter: Option<StateRootIter>,
state_root_miss: bool,
_phantom: PhantomData<Error>,
}
#[derive(Debug)]
pub enum BlockReplayError {
NoBlocks,
SlotProcessing(SlotProcessingError),
BlockProcessing(BlockProcessingError),
}
impl From<SlotProcessingError> for BlockReplayError {
fn from(e: SlotProcessingError) -> Self {
Self::SlotProcessing(e)
}
}
impl From<BlockProcessingError> for BlockReplayError {
fn from(e: BlockProcessingError) -> Self {
Self::BlockProcessing(e)
}
}
/// Defines how state roots should be computed during block replay.
#[derive(PartialEq)]
pub enum StateRootStrategy {
/// Perform all transitions faithfully to the specification.
Accurate,
/// Don't compute state roots, eventually computing an invalid beacon state that can only be
/// used for obtaining shuffling.
Inconsistent,
}
impl<'a, E, Error, StateRootIter> BlockReplayer<'a, E, Error, StateRootIter>
where
E: EthSpec,
StateRootIter: Iterator<Item = Result<(Hash256, Slot), Error>>,
Error: From<BlockReplayError>,
{
/// Create a new replayer that will apply blocks upon `state`.
///
/// Defaults:
///
/// - Full (bulk) signature verification
/// - Accurate state roots
/// - Full block root verification
pub fn new(state: BeaconState<E>, spec: &'a ChainSpec) -> Self {
Self {
state,
spec,
state_root_strategy: StateRootStrategy::Accurate,
block_sig_strategy: BlockSignatureStrategy::VerifyBulk,
verify_block_root: Some(VerifyBlockRoot::True),
pre_block_hook: None,
post_block_hook: None,
pre_slot_hook: None,
post_slot_hook: None,
state_root_iter: None,
state_root_miss: false,
_phantom: PhantomData,
}
}
/// Set the replayer's state root strategy different from the default.
pub fn state_root_strategy(mut self, state_root_strategy: StateRootStrategy) -> Self {
if state_root_strategy == StateRootStrategy::Inconsistent {
self.verify_block_root = None;
}
self.state_root_strategy = state_root_strategy;
self
}
/// Set the replayer's block signature verification strategy.
pub fn block_signature_strategy(mut self, block_sig_strategy: BlockSignatureStrategy) -> Self {
self.block_sig_strategy = block_sig_strategy;
self
}
/// Disable signature verification during replay.
///
/// If you are truly _replaying_ blocks then you will almost certainly want to disable
/// signature checks for performance.
pub fn no_signature_verification(self) -> Self {
self.block_signature_strategy(BlockSignatureStrategy::NoVerification)
}
/// Verify only the block roots of the initial few blocks, and trust the rest.
pub fn minimal_block_root_verification(mut self) -> Self {
self.verify_block_root = None;
self
}
/// Supply a state root iterator to accelerate slot processing.
///
/// If possible the state root iterator should return a state root for every slot from
/// `self.state.slot` to the `target_slot` supplied to `apply_blocks` (inclusive of both
/// endpoints).
pub fn state_root_iter(mut self, iter: StateRootIter) -> Self {
self.state_root_iter = Some(iter);
self
}
/// Run a function immediately before each block that is applied during `apply_blocks`.
///
/// This can be used to inspect the state as blocks are applied.
pub fn pre_block_hook(mut self, hook: PreBlockHook<'a, E, Error>) -> Self {
self.pre_block_hook = Some(hook);
self
}
/// Run a function immediately after each block that is applied during `apply_blocks`.
///
/// This can be used to inspect the state as blocks are applied.
pub fn post_block_hook(mut self, hook: PostBlockHook<'a, E, Error>) -> Self {
self.post_block_hook = Some(hook);
self
}
/// Run a function immediately before slot processing advances the state to the next slot.
pub fn pre_slot_hook(mut self, hook: PreSlotHook<'a, E, Error>) -> Self {
self.pre_slot_hook = Some(hook);
self
}
/// Run a function immediately after slot processing has advanced the state to the next slot.
///
/// The hook receives the state and a bool indicating if this state corresponds to a skipped
/// slot (i.e. it will not have a block applied).
pub fn post_slot_hook(mut self, hook: PostSlotHook<'a, E, Error>) -> Self {
self.post_slot_hook = Some(hook);
self
}
/// Compute the state root for `slot` as efficiently as possible.
///
/// The `blocks` should be the full list of blocks being applied and `i` should be the index of
/// the next block that will be applied, or `blocks.len()` if all blocks have already been
/// applied.
fn get_state_root(
&mut self,
slot: Slot,
blocks: &[SignedBeaconBlock<E>],
i: usize,
) -> Result<Option<Hash256>, Error> {
// If we don't care about state roots then return immediately.
if self.state_root_strategy == StateRootStrategy::Inconsistent {
return Ok(Some(Hash256::zero()));
}
// If a state root iterator is configured, use it to find the root.
if let Some(ref mut state_root_iter) = self.state_root_iter {
let opt_root = state_root_iter
.take_while(|res| res.as_ref().map_or(true, |(_, s)| *s <= slot))
.find(|res| res.as_ref().map_or(true, |(_, s)| *s == slot))
.transpose()?;
if let Some((root, _)) = opt_root {
return Ok(Some(root));
}
}
// Otherwise try to source a root from the previous block.
if let Some(prev_i) = i.checked_sub(1) {
if let Some(prev_block) = blocks.get(prev_i) {
if prev_block.slot() == slot {
return Ok(Some(prev_block.state_root()));
}
}
}
self.state_root_miss = true;
Ok(None)
}
/// Apply `blocks` atop `self.state`, taking care of slot processing.
///
/// If `target_slot` is provided then the state will be advanced through to `target_slot`
/// after the blocks have been applied.
pub fn apply_blocks(
mut self,
blocks: Vec<SignedBeaconBlock<E>>,
target_slot: Option<Slot>,
) -> Result<Self, Error> {
for (i, block) in blocks.iter().enumerate() {
// Allow one additional block at the start which is only used for its state root.
if i == 0 && block.slot() <= self.state.slot() {
continue;
}
while self.state.slot() < block.slot() {
if let Some(ref mut pre_slot_hook) = self.pre_slot_hook {
pre_slot_hook(&mut self.state)?;
}
let state_root = self.get_state_root(self.state.slot(), &blocks, i)?;
let summary = per_slot_processing(&mut self.state, state_root, self.spec)
.map_err(BlockReplayError::from)?;
if let Some(ref mut post_slot_hook) = self.post_slot_hook {
let is_skipped_slot = self.state.slot() < block.slot();
post_slot_hook(&mut self.state, summary, is_skipped_slot)?;
}
}
if let Some(ref mut pre_block_hook) = self.pre_block_hook {
pre_block_hook(&mut self.state, block)?;
}
let verify_block_root = self.verify_block_root.unwrap_or_else(|| {
// If no explicit policy is set, verify only the first 1 or 2 block roots if using
// accurate state roots. Inaccurate state roots require block root verification to
// be off.
if i <= 1 && self.state_root_strategy == StateRootStrategy::Accurate {
VerifyBlockRoot::True
} else {
VerifyBlockRoot::False
}
});
per_block_processing(
&mut self.state,
block,
None,
self.block_sig_strategy,
verify_block_root,
self.spec,
)
.map_err(BlockReplayError::from)?;
if let Some(ref mut post_block_hook) = self.post_block_hook {
post_block_hook(&mut self.state, block)?;
}
}
if let Some(target_slot) = target_slot {
while self.state.slot() < target_slot {
if let Some(ref mut pre_slot_hook) = self.pre_slot_hook {
pre_slot_hook(&mut self.state)?;
}
let state_root = self.get_state_root(self.state.slot(), &blocks, blocks.len())?;
let summary = per_slot_processing(&mut self.state, state_root, self.spec)
.map_err(BlockReplayError::from)?;
if let Some(ref mut post_slot_hook) = self.post_slot_hook {
// No more blocks to apply (from our perspective) so we consider these slots
// skipped.
let is_skipped_slot = true;
post_slot_hook(&mut self.state, summary, is_skipped_slot)?;
}
}
}
Ok(self)
}
/// After block application, check if a state root miss occurred.
pub fn state_root_miss(&self) -> bool {
self.state_root_miss
}
/// Convert the replayer into the state that was built.
pub fn into_state(self) -> BeaconState<E> {
self.state
}
}
impl<'a, E, Error> BlockReplayer<'a, E, Error, StateRootIterDefault<Error>>
where
E: EthSpec,
Error: From<BlockReplayError>,
{
/// If type inference fails to infer the state root iterator type you can use this method
/// to hint that no state root iterator is desired.
pub fn no_state_root_iter(self) -> Self {
self
}
}

4
consensus/state_processing/src/lib.rs
View File

@ -16,6 +16,7 @@
mod macros; mod macros;
mod metrics; mod metrics;
pub mod block_replayer;
pub mod common; pub mod common;
pub mod genesis; pub mod genesis;
pub mod per_block_processing; pub mod per_block_processing;
@ -25,13 +26,14 @@ pub mod state_advance;
pub mod upgrade; pub mod upgrade;
pub mod verify_operation; pub mod verify_operation;
pub use block_replayer::{BlockReplayError, BlockReplayer, StateRootStrategy};
pub use genesis::{ pub use genesis::{
eth2_genesis_time, initialize_beacon_state_from_eth1, is_valid_genesis_state, eth2_genesis_time, initialize_beacon_state_from_eth1, is_valid_genesis_state,
process_activations, process_activations,
}; };
pub use per_block_processing::{ pub use per_block_processing::{
block_signature_verifier, errors::BlockProcessingError, per_block_processing, signature_sets, block_signature_verifier, errors::BlockProcessingError, per_block_processing, signature_sets,
BlockSignatureStrategy, BlockSignatureVerifier, VerifySignatures, BlockSignatureStrategy, BlockSignatureVerifier, VerifyBlockRoot, VerifySignatures,
}; };
pub use per_epoch_processing::{ pub use per_epoch_processing::{
errors::EpochProcessingError, process_epoch as per_epoch_processing, errors::EpochProcessingError, process_epoch as per_epoch_processing,

14
consensus/state_processing/src/per_block_processing.rs
View File

@ -68,6 +68,14 @@ impl VerifySignatures {
} }
} }
/// Control verification of the latest block header.
#[cfg_attr(feature = "arbitrary-fuzz", derive(Arbitrary))]
#[derive(PartialEq, Clone, Copy)]
pub enum VerifyBlockRoot {
True,
False,
}
/// Updates the state for a new block, whilst validating that the block is valid, optionally /// Updates the state for a new block, whilst validating that the block is valid, optionally
/// checking the block proposer signature. /// checking the block proposer signature.
/// ///
@ -84,6 +92,7 @@ pub fn per_block_processing<T: EthSpec>(
signed_block: &SignedBeaconBlock<T>, signed_block: &SignedBeaconBlock<T>,
block_root: Option<Hash256>, block_root: Option<Hash256>,
block_signature_strategy: BlockSignatureStrategy, block_signature_strategy: BlockSignatureStrategy,
verify_block_root: VerifyBlockRoot,
spec: &ChainSpec, spec: &ChainSpec,
) -> Result<(), BlockProcessingError> { ) -> Result<(), BlockProcessingError> {
let block = signed_block.message(); let block = signed_block.message();
@ -120,7 +129,7 @@ pub fn per_block_processing<T: EthSpec>(
BlockSignatureStrategy::VerifyRandao => VerifySignatures::False, BlockSignatureStrategy::VerifyRandao => VerifySignatures::False,
}; };
let proposer_index = process_block_header(state, block, spec)?; let proposer_index = process_block_header(state, block, verify_block_root, spec)?;
if verify_signatures.is_true() { if verify_signatures.is_true() {
verify_block_signature(state, signed_block, block_root, spec)?; verify_block_signature(state, signed_block, block_root, spec)?;
@ -167,6 +176,7 @@ pub fn per_block_processing<T: EthSpec>(
pub fn process_block_header<T: EthSpec>( pub fn process_block_header<T: EthSpec>(
state: &mut BeaconState<T>, state: &mut BeaconState<T>,
block: BeaconBlockRef<'_, T>, block: BeaconBlockRef<'_, T>,
verify_block_root: VerifyBlockRoot,
spec: &ChainSpec, spec: &ChainSpec,
) -> Result<u64, BlockOperationError<HeaderInvalid>> { ) -> Result<u64, BlockOperationError<HeaderInvalid>> {
// Verify that the slots match // Verify that the slots match
@ -195,6 +205,7 @@ pub fn process_block_header<T: EthSpec>(
} }
); );
if verify_block_root == VerifyBlockRoot::True {
let expected_previous_block_root = state.latest_block_header().tree_hash_root(); let expected_previous_block_root = state.latest_block_header().tree_hash_root();
verify!( verify!(
block.parent_root() == expected_previous_block_root, block.parent_root() == expected_previous_block_root,
@ -203,6 +214,7 @@ pub fn process_block_header<T: EthSpec>(
block: block.parent_root(), block: block.parent_root(),
} }
); );
}
*state.latest_block_header_mut() = block.temporary_block_header(); *state.latest_block_header_mut() = block.temporary_block_header();

10
consensus/state_processing/src/per_block_processing/tests.rs
View File

@ -6,7 +6,10 @@ use crate::per_block_processing::errors::{
DepositInvalid, HeaderInvalid, IndexedAttestationInvalid, IntoWithIndex, DepositInvalid, HeaderInvalid, IndexedAttestationInvalid, IntoWithIndex,
ProposerSlashingInvalid, ProposerSlashingInvalid,
}; };
use crate::{per_block_processing::process_operations, BlockSignatureStrategy, VerifySignatures}; use crate::{
per_block_processing::process_operations, BlockSignatureStrategy, VerifyBlockRoot,
VerifySignatures,
};
use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType}; use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType};
use lazy_static::lazy_static; use lazy_static::lazy_static;
use ssz_types::Bitfield; use ssz_types::Bitfield;
@ -65,6 +68,7 @@ fn valid_block_ok() {
&block, &block,
None, None,
BlockSignatureStrategy::VerifyIndividual, BlockSignatureStrategy::VerifyIndividual,
VerifyBlockRoot::True,
&spec, &spec,
); );
@ -88,6 +92,7 @@ fn invalid_block_header_state_slot() {
&SignedBeaconBlock::from_block(block, signature), &SignedBeaconBlock::from_block(block, signature),
None, None,
BlockSignatureStrategy::VerifyIndividual, BlockSignatureStrategy::VerifyIndividual,
VerifyBlockRoot::True,
&spec, &spec,
); );
@ -116,6 +121,7 @@ fn invalid_parent_block_root() {
&SignedBeaconBlock::from_block(block, signature), &SignedBeaconBlock::from_block(block, signature),
None, None,
BlockSignatureStrategy::VerifyIndividual, BlockSignatureStrategy::VerifyIndividual,
VerifyBlockRoot::True,
&spec, &spec,
); );
@ -145,6 +151,7 @@ fn invalid_block_signature() {
&SignedBeaconBlock::from_block(block, Signature::empty()), &SignedBeaconBlock::from_block(block, Signature::empty()),
None, None,
BlockSignatureStrategy::VerifyIndividual, BlockSignatureStrategy::VerifyIndividual,
VerifyBlockRoot::True,
&spec, &spec,
); );
@ -174,6 +181,7 @@ fn invalid_randao_reveal_signature() {
&signed_block, &signed_block,
None, None,
BlockSignatureStrategy::VerifyIndividual, BlockSignatureStrategy::VerifyIndividual,
VerifyBlockRoot::True,
&spec, &spec,
); );

5
lcli/src/transition_blocks.rs
View File

@ -1,7 +1,9 @@
use clap::ArgMatches; use clap::ArgMatches;
use eth2_network_config::Eth2NetworkConfig; use eth2_network_config::Eth2NetworkConfig;
use ssz::Encode; use ssz::Encode;
use state_processing::{per_block_processing, per_slot_processing, BlockSignatureStrategy}; use state_processing::{
per_block_processing, per_slot_processing, BlockSignatureStrategy, VerifyBlockRoot,
};
use std::fs::File; use std::fs::File;
use std::io::prelude::*; use std::io::prelude::*;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
@ -77,6 +79,7 @@ fn do_transition<T: EthSpec>(
&block, &block,
None, None,
BlockSignatureStrategy::VerifyIndividual, BlockSignatureStrategy::VerifyIndividual,
VerifyBlockRoot::True,
spec, spec,
) )
.map_err(|e| format!("State transition failed: {:?}", e))?; .map_err(|e| format!("State transition failed: {:?}", e))?;

4
testing/ef_tests/src/cases/operations.rs
View File

@ -12,7 +12,7 @@ use state_processing::per_block_processing::{
altair, base, process_attester_slashings, process_deposits, process_exits, altair, base, process_attester_slashings, process_deposits, process_exits,
process_proposer_slashings, process_proposer_slashings,
}, },
process_sync_aggregate, VerifySignatures, process_sync_aggregate, VerifyBlockRoot, VerifySignatures,
}; };
use std::fmt::Debug; use std::fmt::Debug;
use std::path::Path; use std::path::Path;
@ -183,7 +183,7 @@ impl<E: EthSpec> Operation<E> for BeaconBlock<E> {
spec: &ChainSpec, spec: &ChainSpec,
_: &Operations<E, Self>, _: &Operations<E, Self>,
) -> Result<(), BlockProcessingError> { ) -> Result<(), BlockProcessingError> {
process_block_header(state, self.to_ref(), spec)?; process_block_header(state, self.to_ref(), VerifyBlockRoot::True, spec)?;
Ok(()) Ok(())
} }
} }

3
testing/ef_tests/src/cases/sanity_blocks.rs
View File

@ -5,6 +5,7 @@ use crate::decode::{ssz_decode_file_with, ssz_decode_state, yaml_decode_file};
use serde_derive::Deserialize; use serde_derive::Deserialize;
use state_processing::{ use state_processing::{
per_block_processing, per_slot_processing, BlockProcessingError, BlockSignatureStrategy, per_block_processing, per_slot_processing, BlockProcessingError, BlockSignatureStrategy,
VerifyBlockRoot,
}; };
use types::{BeaconState, EthSpec, ForkName, RelativeEpoch, SignedBeaconBlock}; use types::{BeaconState, EthSpec, ForkName, RelativeEpoch, SignedBeaconBlock};
@ -98,6 +99,7 @@ impl<E: EthSpec> Case for SanityBlocks<E> {
signed_block, signed_block,
None, None,
BlockSignatureStrategy::VerifyIndividual, BlockSignatureStrategy::VerifyIndividual,
VerifyBlockRoot::True,
spec, spec,
)?; )?;
@ -106,6 +108,7 @@ impl<E: EthSpec> Case for SanityBlocks<E> {
signed_block, signed_block,
None, None,
BlockSignatureStrategy::VerifyBulk, BlockSignatureStrategy::VerifyBulk,
VerifyBlockRoot::True,
spec, spec,
)?; )?;

2
testing/ef_tests/src/cases/transition.rs
View File

@ -4,6 +4,7 @@ use crate::decode::{ssz_decode_file_with, ssz_decode_state, yaml_decode_file};
use serde_derive::Deserialize; use serde_derive::Deserialize;
use state_processing::{ use state_processing::{
per_block_processing, state_advance::complete_state_advance, BlockSignatureStrategy, per_block_processing, state_advance::complete_state_advance, BlockSignatureStrategy,
VerifyBlockRoot,
}; };
use std::str::FromStr; use std::str::FromStr;
use types::{BeaconState, Epoch, ForkName, SignedBeaconBlock}; use types::{BeaconState, Epoch, ForkName, SignedBeaconBlock};
@ -97,6 +98,7 @@ impl<E: EthSpec> Case for TransitionTest<E> {
block, block,
None, None,
BlockSignatureStrategy::VerifyBulk, BlockSignatureStrategy::VerifyBulk,
VerifyBlockRoot::True,
spec, spec,
) )
.map_err(|e| format!("Block processing failed: {:?}", e))?; .map_err(|e| format!("Block processing failed: {:?}", e))?;

3
testing/state_transition_vectors/src/exit.rs
View File

@ -2,7 +2,7 @@ use super::*;
use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType}; use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType};
use state_processing::{ use state_processing::{
per_block_processing, per_block_processing::errors::ExitInvalid, BlockProcessingError, per_block_processing, per_block_processing::errors::ExitInvalid, BlockProcessingError,
BlockSignatureStrategy, BlockSignatureStrategy, VerifyBlockRoot,
}; };
use types::{BeaconBlock, BeaconState, Epoch, EthSpec, SignedBeaconBlock}; use types::{BeaconBlock, BeaconState, Epoch, EthSpec, SignedBeaconBlock};
@ -66,6 +66,7 @@ impl ExitTest {
block, block,
None, None,
BlockSignatureStrategy::VerifyIndividual, BlockSignatureStrategy::VerifyIndividual,
VerifyBlockRoot::True,
&E::default_spec(), &E::default_spec(),
) )
} }