resolve merge issues

This commit is contained in:
realbigsean 2023-07-07 10:17:04 -04:00
parent cfe2452533
commit c4da1ba450
No known key found for this signature in database
GPG Key ID: BE1B3DB104F6C788
14 changed files with 143 additions and 109 deletions

View File

@ -2784,7 +2784,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
block_root: Hash256,
unverified_block: B,
notify_execution_layer: NotifyExecutionLayer,
publish_fn: impl FnOnce() -> Result<(), BlockError<T::EthSpec>> + Send + 'static,
publish_fn: impl FnOnce() -> Result<(), BlockError<T::EthSpec>> + Send + 'static,
) -> Result<AvailabilityProcessingStatus, BlockError<T::EthSpec>> {
// Start the Prometheus timer.
let _full_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_TIMES);
@ -2801,7 +2801,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
)?;
//TODO(sean) error handling?
publish_fn()?;
publish_fn()?;
let executed_block = self
.clone()

View File

@ -812,9 +812,9 @@ where
&self,
state: BeaconState<E>,
slot: Slot,
) -> (SignedBlindedBeaconBlock<E>, BeaconState<E>) {
) -> (BlockContentsTuple<E, BlindedPayload<E>>, BeaconState<E>) {
let (unblinded, new_state) = self.make_block(state, slot).await;
(unblinded.into(), new_state)
((unblinded.0.into(), unblinded.1), new_state)
}
/// Returns a newly created block, signed by the proposer for the given slot.
@ -1839,7 +1839,9 @@ where
self.set_current_slot(slot);
let block_hash: SignedBeaconBlockHash = self
.chain
.process_block(block_root, block.into(), NotifyExecutionLayer::Yes,|| Ok(()),)
.process_block(block_root, block.into(), NotifyExecutionLayer::Yes, || {
Ok(())
})
.await?
.try_into()
.unwrap();

View File

@ -686,7 +686,7 @@ async fn run_skip_slot_test(skip_slots: u64) {
harness_a.chain.head_snapshot().beacon_block_root,
harness_a.get_head_block(),
NotifyExecutionLayer::Yes,
|| Ok(())
|| Ok(()),
)
.await
.unwrap();

View File

@ -1,7 +1,7 @@
use eth2::types::builder_bid::SignedBuilderBid;
use eth2::types::{
AbstractExecPayload, BlindedPayload, EthSpec, ExecutionBlockHash, ExecutionPayload,
ForkVersionedResponse, PublicKeyBytes, SignedBeaconBlock, SignedValidatorRegistrationData,
ForkVersionedResponse, PublicKeyBytes, SignedBlockContents, SignedValidatorRegistrationData,
Slot,
};
pub use eth2::Error;
@ -140,7 +140,7 @@ impl BuilderHttpClient {
/// `POST /eth/v1/builder/blinded_blocks`
pub async fn post_builder_blinded_blocks<E: EthSpec>(
&self,
blinded_block: &SignedBeaconBlock<E, BlindedPayload<E>>,
blinded_block: &SignedBlockContents<E, BlindedPayload<E>>,
) -> Result<ForkVersionedResponse<ExecutionPayload<E>>, Error> {
let mut path = self.server.full.clone();

View File

@ -13,6 +13,7 @@ pub use engine_api::*;
pub use engine_api::{http, http::deposit_methods, http::HttpJsonRpc};
use engines::{Engine, EngineError};
pub use engines::{EngineState, ForkchoiceState};
use eth2::types::SignedBlockContents;
use eth2::types::{builder_bid::SignedBuilderBid, ForkVersionedResponse};
use ethers_core::abi::ethereum_types::FromStrRadixErr;
use ethers_core::types::Transaction as EthersTransaction;
@ -47,10 +48,7 @@ use types::{
ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge, ForkName,
};
use types::{KzgProofs, Withdrawals};
use types::{
ProposerPreparationData, PublicKeyBytes, Signature, SignedBeaconBlock, Slot, Transaction,
Uint256,
};
use types::{ProposerPreparationData, PublicKeyBytes, Signature, Slot, Transaction, Uint256};
mod block_hash;
mod engine_api;
@ -1876,7 +1874,7 @@ impl<T: EthSpec> ExecutionLayer<T> {
pub async fn propose_blinded_beacon_block(
&self,
block_root: Hash256,
block: &SignedBeaconBlock<T, BlindedPayload<T>>,
block: &SignedBlockContents<T, BlindedPayload<T>>,
) -> Result<ExecutionPayload<T>, Error> {
debug!(
self.log(),
@ -1924,6 +1922,7 @@ impl<T: EthSpec> ExecutionLayer<T> {
"relay_response_ms" => duration.as_millis(),
"block_root" => ?block_root,
"parent_hash" => ?block
.signed_block()
.message()
.execution_payload()
.map(|payload| format!("{}", payload.parent_hash()))

View File

@ -32,8 +32,8 @@ use beacon_chain::{
pub use block_id::BlockId;
use directory::DEFAULT_ROOT_DIR;
use eth2::types::{
self as api_types, BroadcastValidation, EndpointVersion, ForkChoice, ForkChoiceNode, SignedBlockContents,
SkipRandaoVerification, ValidatorId, ValidatorStatus,
self as api_types, BroadcastValidation, EndpointVersion, ForkChoice, ForkChoiceNode,
SignedBlockContents, SkipRandaoVerification, ValidatorId, ValidatorStatus,
};
use lighthouse_network::{types::SyncState, EnrExt, NetworkGlobals, PeerId, PubsubMessage};
use lighthouse_version::version_with_platform;
@ -63,9 +63,8 @@ use types::{
Attestation, AttestationData, AttestationShufflingId, AttesterSlashing, BeaconStateError,
BlindedPayload, CommitteeCache, ConfigAndPreset, Epoch, EthSpec, ForkName, FullPayload,
ProposerPreparationData, ProposerSlashing, RelativeEpoch, SignedAggregateAndProof,
SignedBeaconBlock, SignedBlsToExecutionChange, SignedContributionAndProof,
SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, SyncCommitteeMessage,
SyncContributionData,
SignedBlsToExecutionChange, SignedContributionAndProof, SignedValidatorRegistrationData,
SignedVoluntaryExit, Slot, SyncCommitteeMessage, SyncContributionData,
};
use version::{
add_consensus_version_header, execution_optimistic_finalized_fork_versioned_response,
@ -1248,7 +1247,7 @@ pub fn serve<T: BeaconChainTypes>(
.and(log_filter.clone())
.then(
|validation_level: api_types::BroadcastValidationQuery,
block_contents: SignedBlockContents<T::EthSpec>,
block_contents: SignedBlockContents<T::EthSpec>,
chain: Arc<BeaconChain<T>>,
network_tx: UnboundedSender<NetworkMessage<T::EthSpec>>,
log: Logger| async move {
@ -1289,7 +1288,7 @@ pub fn serve<T: BeaconChainTypes>(
.and(network_tx_filter.clone())
.and(log_filter.clone())
.and_then(
|block: SignedBeaconBlock<T::EthSpec, BlindedPayload<_>>,
|block: SignedBlockContents<T::EthSpec, BlindedPayload<_>>,
chain: Arc<BeaconChain<T>>,
network_tx: UnboundedSender<NetworkMessage<T::EthSpec>>,
log: Logger| async move {

View File

@ -1,13 +1,13 @@
use crate::metrics;
use beacon_chain::blob_verification::{AsBlock, BlockWrapper};
use beacon_chain::blob_verification::BlockWrapper;
use beacon_chain::validator_monitor::{get_block_delay_ms, timestamp_now};
use beacon_chain::{
BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, GossipVerifiedBlock,
NotifyExecutionLayer, AvailabilityProcessingStatus
AvailabilityProcessingStatus, BeaconChain, BeaconChainError, BeaconChainTypes, BlockError,
GossipVerifiedBlock, NotifyExecutionLayer,
};
use eth2::types::SignedBlockContents;
use eth2::types::BroadcastValidation;
use eth2::types::SignedBlockContents;
use execution_layer::ProvenancedPayload;
use lighthouse_network::PubsubMessage;
use network::NetworkMessage;
@ -15,12 +15,11 @@ use slog::{debug, error, info, warn, Logger};
use slot_clock::SlotClock;
use std::sync::Arc;
use std::time::Duration;
use store::FixedVector;
use tokio::sync::mpsc::UnboundedSender;
use tree_hash::TreeHash;
use types::{
AbstractExecPayload, BeaconBlockRef, BlindedPayload, EthSpec, ExecPayload, ExecutionBlockHash,
FullPayload, Hash256, SignedBeaconBlock,
FullPayload, Hash256, SignedBeaconBlock, SignedBlobSidecarList, VariableList,
};
use warp::Rejection;
@ -33,11 +32,11 @@ pub enum ProvenancedBlock<T: EthSpec> {
}
impl<T: EthSpec> ProvenancedBlock<T> {
pub fn local(block: Arc<SignedBeaconBlock<T>>) -> Self {
pub fn local(block: SignedBlockContents<T>) -> Self {
Self::Local(block)
}
pub fn builder(block: Arc<SignedBeaconBlock<T>>) -> Self {
pub fn builder(block: SignedBlockContents<T>) -> Self {
Self::Builder(block)
}
}
@ -52,7 +51,8 @@ pub async fn publish_block<T: BeaconChainTypes>(
validation_level: BroadcastValidation,
) -> Result<(), Rejection> {
let seen_timestamp = timestamp_now();
let (block, maybe_blobs, is_locally_built_block) = match provenanced_block {
let (block, blobs_opt, is_locally_built_block) = match provenanced_block {
ProvenancedBlock::Local(block_contents) => {
let (block, maybe_blobs) = block_contents.deconstruct();
(Arc::new(block), maybe_blobs, true)
@ -62,15 +62,12 @@ pub async fn publish_block<T: BeaconChainTypes>(
(Arc::new(block), maybe_blobs, false)
}
};
let beacon_block = block.clone();
let delay = get_block_delay_ms(seen_timestamp, beacon_block.message(), &chain.slot_clock);
//FIXME(sean) have to move this to prior to publishing because it's included in the blobs sidecar message.
//this may skew metrics
let block_root = block_root.unwrap_or_else(|| block.canonical_root());
debug!(log, "Signed block received in HTTP API"; "slot" => beacon_block.slot());
let delay = get_block_delay_ms(seen_timestamp, block.message(), &chain.slot_clock);
debug!(log, "Signed block received in HTTP API"; "slot" => block.slot());
/* actually publish a block */
let publish_block = move |block: Arc<SignedBeaconBlock<T::EthSpec>>,
blobs_opt: Option<SignedBlobSidecarList<T::EthSpec>>,
sender,
log,
seen_timestamp| {
@ -80,43 +77,59 @@ pub async fn publish_block<T: BeaconChainTypes>(
.unwrap_or_else(|| Duration::from_secs(0));
info!(log, "Signed block published to network via HTTP API"; "slot" => block.slot(), "publish_delay" => ?publish_delay);
// Send the block, regardless of whether or not it is valid. The API
// specification is very clear that this is the desired behaviour.
let wrapped_block: BlockWrapper<T::EthSpec> = match block.as_ref() {
SignedBeaconBlock::Base(_)
| SignedBeaconBlock::Altair(_)
| SignedBeaconBlock::Merge(_)
| SignedBeaconBlock::Capella(_) => {
crate::publish_pubsub_message(network_tx, PubsubMessage::BeaconBlock(block.clone()))?;
block.into()
}
SignedBeaconBlock::Deneb(_) => {
crate::publish_pubsub_message(network_tx, PubsubMessage::BeaconBlock(block.clone()))?;
if let Some(signed_blobs) = maybe_blobs {
for (blob_index, blob) in signed_blobs.clone().into_iter().enumerate() {
crate::publish_pubsub_message(
network_tx,
PubsubMessage::BlobSidecar(Box::new((blob_index as u64, blob))),
)?;
}
let blobs = signed_blobs
.into_iter()
.map(|blob| Some(blob.message))
.collect::<Vec<_>>();
BlockWrapper::BlockAndBlobs(block, FixedVector::from(blobs))
} else {
block.into()
// Send the block, regardless of whether or not it is valid. The API
// specification is very clear that this is the desired behaviour.
match block.as_ref() {
SignedBeaconBlock::Base(_)
| SignedBeaconBlock::Altair(_)
| SignedBeaconBlock::Merge(_)
| SignedBeaconBlock::Capella(_) => {
crate::publish_pubsub_message(&sender, PubsubMessage::BeaconBlock(block.clone()))
.map_err(|_| BlockError::BeaconChainError(BeaconChainError::UnableToPublish))?;
}
}
};
let message = PubsubMessage::BeaconBlock(block);
crate::publish_pubsub_message(&sender, message)
.map_err(|_| BeaconChainError::UnableToPublish.into())
SignedBeaconBlock::Deneb(_) => {
crate::publish_pubsub_message(&sender, PubsubMessage::BeaconBlock(block.clone()))
.map_err(|_| BlockError::BeaconChainError(BeaconChainError::UnableToPublish))?;
if let Some(signed_blobs) = blobs_opt {
for (blob_index, blob) in signed_blobs.into_iter().enumerate() {
crate::publish_pubsub_message(
&sender,
PubsubMessage::BlobSidecar(Box::new((blob_index as u64, blob))),
)
.map_err(|_| {
BlockError::BeaconChainError(BeaconChainError::UnableToPublish)
})?;
}
}
}
};
Ok(())
};
let mapped_blobs = blobs_opt.clone().map(|blobs| {
VariableList::from(
blobs
.into_iter()
.map(|blob| blob.message)
.collect::<Vec<_>>(),
)
});
/* only publish if gossip- and consensus-valid and equivocation-free */
let chain_clone = chain.clone();
let slot = block.message().slot();
let block_clone = block.clone();
let proposer_index = block.message().proposer_index();
let sender_clone = network_tx.clone();
let log_clone = log.clone();
/* if we can form a `GossipVerifiedBlock`, we've passed our basic gossip checks */
let gossip_verified_block = GossipVerifiedBlock::new(block, &chain).map_err(|e| {
warn!(log, "Not publishing block, not gossip verified"; "slot" => beacon_block.slot(), "error" => ?e);
let gossip_verified_block = GossipVerifiedBlock::new(
BlockWrapper::new(block.clone(), mapped_blobs),
&chain,
)
.map_err(|e| {
warn!(log, "Not publishing block, not gossip verified"; "slot" => slot, "error" => ?e);
warp_utils::reject::custom_bad_request(e.to_string())
})?;
@ -124,25 +137,24 @@ pub async fn publish_block<T: BeaconChainTypes>(
if let BroadcastValidation::Gossip = validation_level {
publish_block(
beacon_block.clone(),
network_tx.clone(),
block.clone(),
blobs_opt.clone(),
sender_clone.clone(),
log.clone(),
seen_timestamp,
)
.map_err(|_| warp_utils::reject::custom_server_error("unable to publish".into()))?;
}
/* only publish if gossip- and consensus-valid and equivocation-free */
let chain_clone = chain.clone();
let block_clone = beacon_block.clone();
let log_clone = log.clone();
let sender_clone = network_tx.clone();
let publish_fn = move || match validation_level {
BroadcastValidation::Gossip => Ok(()),
BroadcastValidation::Consensus => {
publish_block(block_clone, sender_clone, log_clone, seen_timestamp)
}
BroadcastValidation::Consensus => publish_block(
block_clone,
blobs_opt,
sender_clone,
log_clone,
seen_timestamp,
),
BroadcastValidation::ConsensusAndEquivocation => {
if chain_clone
.observed_block_producers
@ -158,13 +170,16 @@ pub async fn publish_block<T: BeaconChainTypes>(
);
Err(BlockError::Slashable)
} else {
publish_block(block_clone, sender_clone, log_clone, seen_timestamp)
publish_block(
block_clone,
blobs_opt,
sender_clone,
log_clone,
seen_timestamp,
)
}
}
};
let block_clone = wrapped_block.block_cloned();
let slot = block_clone.message().slot();
let proposer_index = block_clone.message().proposer_index();
match chain
.process_block(
@ -188,7 +203,7 @@ pub async fn publish_block<T: BeaconChainTypes>(
// Notify the validator monitor.
chain.validator_monitor.read().register_api_block(
seen_timestamp,
block_clone.message(),
block.message(),
root,
&chain.slot_clock,
);
@ -201,14 +216,7 @@ pub async fn publish_block<T: BeaconChainTypes>(
// blocks built with builders we consider the broadcast time to be
// when the blinded block is published to the builder.
if is_locally_built_block {
late_block_logging(
&chain,
seen_timestamp,
block_clone.message(),
root,
"local",
&log,
)
late_block_logging(&chain, seen_timestamp, block.message(), root, "local", &log)
}
Ok(())
@ -255,15 +263,14 @@ pub async fn publish_block<T: BeaconChainTypes>(
/// Handles a request from the HTTP API for blinded blocks. This converts blinded blocks into full
/// blocks before publishing.
pub async fn publish_blinded_block<T: BeaconChainTypes>(
block: SignedBeaconBlock<T::EthSpec, BlindedPayload<T::EthSpec>>,
block: SignedBlockContents<T::EthSpec, BlindedPayload<T::EthSpec>>,
chain: Arc<BeaconChain<T>>,
network_tx: &UnboundedSender<NetworkMessage<T::EthSpec>>,
log: Logger,
validation_level: BroadcastValidation,
) -> Result<(), Rejection> {
let block_root = block.canonical_root();
let full_block: ProvenancedBlock<T> =
reconstruct_block(chain.clone(), block_root, block, log.clone()).await?;
let block_root = block.signed_block().canonical_root();
let full_block = reconstruct_block(chain.clone(), block_root, block, log.clone()).await?;
publish_block::<T>(
Some(block_root),
full_block,
@ -281,10 +288,12 @@ pub async fn publish_blinded_block<T: BeaconChainTypes>(
pub async fn reconstruct_block<T: BeaconChainTypes>(
chain: Arc<BeaconChain<T>>,
block_root: Hash256,
block: SignedBeaconBlock<T::EthSpec, BlindedPayload<T::EthSpec>>,
block: SignedBlockContents<T::EthSpec, BlindedPayload<T::EthSpec>>,
log: Logger,
) -> Result<ProvenancedBlock<T>, Rejection> {
let full_payload_opt = if let Ok(payload_header) = block.message().body().execution_payload() {
) -> Result<ProvenancedBlock<T::EthSpec>, Rejection> {
let full_payload_opt = if let Ok(payload_header) =
block.signed_block().message().body().execution_payload()
{
let el = chain.execution_layer.as_ref().ok_or_else(|| {
warp_utils::reject::custom_server_error("Missing execution layer".to_string())
})?;
@ -292,9 +301,12 @@ pub async fn reconstruct_block<T: BeaconChainTypes>(
// If the execution block hash is zero, use an empty payload.
let full_payload = if payload_header.block_hash() == ExecutionBlockHash::zero() {
let payload = FullPayload::default_at_fork(
chain
.spec
.fork_name_at_epoch(block.slot().epoch(T::EthSpec::slots_per_epoch())),
chain.spec.fork_name_at_epoch(
block
.signed_block()
.slot()
.epoch(T::EthSpec::slots_per_epoch()),
),
)
.map_err(|e| {
warp_utils::reject::custom_server_error(format!(
@ -319,7 +331,7 @@ pub async fn reconstruct_block<T: BeaconChainTypes>(
late_block_logging(
&chain,
timestamp_now(),
block.message(),
block.signed_block().message(),
block_root,
"builder",
&log,
@ -347,14 +359,20 @@ pub async fn reconstruct_block<T: BeaconChainTypes>(
// A block without a payload is pre-merge and we consider it locally
// built.
None => block
.deconstruct()
.0
.try_into_full_block(None)
.map(SignedBlockContents::Block)
.map(ProvenancedBlock::local),
Some(ProvenancedPayload::Local(full_payload)) => block
.deconstruct()
.0
.try_into_full_block(Some(full_payload))
.map(SignedBlockContents::Block)
.map(ProvenancedBlock::local),
Some(ProvenancedPayload::Builder(full_payload)) => block
.deconstruct()
.0
.try_into_full_block(Some(full_payload))
.map(SignedBlockContents::Block)
.map(ProvenancedBlock::builder),

View File

@ -1264,7 +1264,11 @@ impl ApiTester {
.await
.0;
assert!(self.client.post_beacon_blocks(&SignedBlockContents::from(block)).await.is_err());
assert!(self
.client
.post_beacon_blocks(&SignedBlockContents::from(block))
.await
.is_err());
assert!(
self.network_rx.network_recv.recv().await.is_some(),

View File

@ -171,6 +171,10 @@ lazy_static! {
"beacon_processor_bls_to_execution_change_imported_total",
"Total number of address changes imported to the op pool."
);
}
// Need to split up this `lazy_static!` due to recursion limits.
lazy_static! {
// Rpc blocks.
pub static ref BEACON_PROCESSOR_RPC_BLOCK_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
"beacon_processor_rpc_block_queue_total",

View File

@ -770,7 +770,8 @@ where
(parent_justified, parent_finalized)
} else {
let justification_and_finalization_state = match block {
BeaconBlockRef::Deneb(_)| BeaconBlockRef::Capella(_)
BeaconBlockRef::Deneb(_)
| BeaconBlockRef::Capella(_)
| BeaconBlockRef::Merge(_)
| BeaconBlockRef::Altair(_) => match progressive_balances_mode {
ProgressiveBalancesMode::Disabled => {

View File

@ -63,6 +63,7 @@ pub fn upgrade_to_deneb<E: EthSpec>(
historical_summaries: pre.historical_summaries.clone(),
// Caches
total_active_balance: pre.total_active_balance,
progressive_balances_cache: mem::take(&mut pre.progressive_balances_cache),
committee_caches: mem::take(&mut pre.committee_caches),
pubkey_cache: mem::take(&mut pre.pubkey_cache),
exit_cache: mem::take(&mut pre.exit_cache),

View File

@ -1207,7 +1207,7 @@ impl<T: EthSpec> BeaconState<T> {
&mut state.balances,
&mut state.progressive_balances_cache,
),
BeaconState::Deneb(state) => (
BeaconState::Deneb(state) => (
&mut state.validators,
&mut state.balances,
&mut state.progressive_balances_cache,

View File

@ -179,6 +179,9 @@ impl ProgressiveBalancesMode {
pub fn is_progressive_balances_enabled<E: EthSpec>(state: &BeaconState<E>) -> bool {
match state {
BeaconState::Base(_) => false,
BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_) => true,
BeaconState::Altair(_)
| BeaconState::Merge(_)
| BeaconState::Capella(_)
| BeaconState::Deneb(_) => true,
}
}

View File

@ -4,8 +4,8 @@ use crate::case_result::compare_beacon_state_results_without_caches;
use crate::decode::{ssz_decode_file, ssz_decode_file_with, ssz_decode_state, yaml_decode_file};
use crate::testing_spec;
use serde_derive::Deserialize;
use state_processing::common::update_progressive_balances_cache::initialize_progressive_balances_cache;
use ssz::Decode;
use state_processing::common::update_progressive_balances_cache::initialize_progressive_balances_cache;
use state_processing::{
per_block_processing::{
errors::BlockProcessingError,
@ -98,7 +98,10 @@ impl<E: EthSpec> Operation<E> for Attestation<E> {
&mut ctxt,
spec,
),
BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_)| BeaconState::Deneb(_) => {
BeaconState::Altair(_)
| BeaconState::Merge(_)
| BeaconState::Capella(_)
| BeaconState::Deneb(_) => {
initialize_progressive_balances_cache(state, None, spec)?;
altair::process_attestation(state, self, 0, &mut ctxt, VerifySignatures::True, spec)
}