cerc-io/lighthouse
11
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'blobs-beacon-api' of github.com:jimmygchen/lighthouse into blobs-beacon-api

Browse Source
This commit is contained in:
Jimmy Chen
2023-01-04 01:58:08 +11:00
parent 355cab8704 1b64cbadba
commit 7c2a0aeb58
73 changed files with 484 additions and 578 deletions
Download Patch File Download Diff File Expand all files Collapse all files
beacon_node/Cargo.toml
-1
View File
@@ -13,7 +13,6 @@ node_test_rig = { path = "../testing/node_test_rig" }
[features]
write_ssz_files = ["beacon_chain/write_ssz_files"] # Writes debugging .ssz files to /tmp during block processing.
withdrawals = ["beacon_chain/withdrawals", "types/withdrawals", "store/withdrawals", "execution_layer/withdrawals"]
withdrawals-processing = [
"beacon_chain/withdrawals-processing",
"store/withdrawals-processing",
beacon_node/beacon_chain/Cargo.toml
-1
View File
@@ -10,7 +10,6 @@ default = ["participation_metrics"]
write_ssz_files = [] # Writes debugging .ssz files to /tmp during block processing.
participation_metrics = [] # Exposes validator participation metrics to Prometheus.
fork_from_env = [] # Initialise the harness chain spec from the FORK_NAME env variable
withdrawals = ["state_processing/withdrawals", "types/withdrawals", "store/withdrawals", "execution_layer/withdrawals"]
withdrawals-processing = [
"state_processing/withdrawals-processing",
"store/withdrawals-processing",
beacon_node/beacon_chain/src/beacon_chain.rs
+3 -15
View File
@@ -62,7 +62,7 @@ use crate::{metrics, BeaconChainError, BeaconForkChoiceStore, BeaconSnapshot, Ca
use eth2::types::{EventKind, SseBlock, SyncDuty};
use execution_layer::{
BlockProposalContents, BuilderParams, ChainHealth, ExecutionLayer, FailedCondition,
PayloadAttributes, PayloadAttributesV2, PayloadStatus,
PayloadAttributes, PayloadStatus,
};
pub use fork_choice::CountUnrealized;
use fork_choice::{
@@ -80,14 +80,12 @@ use slasher::Slasher;
use slog::{crit, debug, error, info, trace, warn, Logger};
use slot_clock::SlotClock;
use ssz::Encode;
#[cfg(feature = "withdrawals")]
use state_processing::per_block_processing::get_expected_withdrawals;
use state_processing::{
common::get_attesting_indices_from_state,
per_block_processing,
per_block_processing::{
errors::AttestationValidationError, verify_attestation_for_block_inclusion,
VerifySignatures,
errors::AttestationValidationError, get_expected_withdrawals,
verify_attestation_for_block_inclusion, VerifySignatures,
},
per_slot_processing,
state_advance::{complete_state_advance, partial_state_advance},
@@ -290,7 +288,6 @@ struct PartialBeaconBlock<E: EthSpec, Payload: AbstractExecPayload<E>> {
voluntary_exits: Vec<SignedVoluntaryExit>,
sync_aggregate: Option<SyncAggregate<E>>,
prepare_payload_handle: Option<PreparePayloadHandle<E, Payload>>,
#[cfg(feature = "withdrawals")]
bls_to_execution_changes: Vec<SignedBlsToExecutionChange>,
}
@@ -4208,7 +4205,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
let eth1_data = eth1_chain.eth1_data_for_block_production(&state, &self.spec)?;
let deposits = eth1_chain.deposits_for_block_inclusion(&state, &eth1_data, &self.spec)?;
#[cfg(feature = "withdrawals")]
let bls_to_execution_changes = self
.op_pool
.get_bls_to_execution_changes(&state, &self.spec);
@@ -4371,7 +4367,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
voluntary_exits,
sync_aggregate,
prepare_payload_handle,
#[cfg(feature = "withdrawals")]
bls_to_execution_changes,
})
}
@@ -4400,7 +4395,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
// this function. We can assume that the handle has already been consumed in order to
// produce said `execution_payload`.
prepare_payload_handle: _,
#[cfg(feature = "withdrawals")]
bls_to_execution_changes,
} = partial_beacon_block;
@@ -4501,7 +4495,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
execution_payload: payload
.try_into()
.map_err(|_| BlockProductionError::InvalidPayloadFork)?,
#[cfg(feature = "withdrawals")]
bls_to_execution_changes: bls_to_execution_changes.into(),
},
}),
@@ -4533,7 +4526,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
execution_payload: payload
.try_into()
.map_err(|_| BlockProductionError::InvalidPayloadFork)?,
#[cfg(feature = "withdrawals")]
bls_to_execution_changes: bls_to_execution_changes.into(),
blob_kzg_commitments: kzg_commitments
.ok_or(BlockProductionError::InvalidPayloadFork)?,
@@ -4833,7 +4825,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
return Ok(());
}
#[cfg(feature = "withdrawals")]
let withdrawals = match self.spec.fork_name_at_slot::<T::EthSpec>(prepare_slot) {
ForkName::Base | ForkName::Altair | ForkName::Merge => None,
ForkName::Capella | ForkName::Eip4844 => {
@@ -4868,10 +4859,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
execution_layer
.get_suggested_fee_recipient(proposer as u64)
.await,
#[cfg(feature = "withdrawals")]
withdrawals,
#[cfg(not(feature = "withdrawals"))]
None,
);
debug!(
beacon_node/beacon_chain/src/blob_cache.rs
+1 -2
View File
@@ -1,7 +1,6 @@
use lru::LruCache;
use parking_lot::Mutex;
use tree_hash::TreeHash;
use types::{BlobsSidecar, EthSpec, ExecutionPayload, Hash256};
use types::{BlobsSidecar, EthSpec, Hash256};
pub const DEFAULT_BLOB_CACHE_SIZE: usize = 10;
beacon_node/beacon_chain/src/blob_verification.rs
-2
View File
@@ -2,9 +2,7 @@ use slot_clock::SlotClock;
use crate::beacon_chain::{BeaconChain, BeaconChainTypes, MAXIMUM_GOSSIP_CLOCK_DISPARITY};
use crate::{kzg_utils, BeaconChainError};
use bls::PublicKey;
use state_processing::per_block_processing::eip4844::eip4844::verify_kzg_commitments_against_transactions;
use types::consts::eip4844::BLS_MODULUS;
use types::{BeaconStateError, BlobsSidecar, Hash256, KzgCommitment, Slot, Transactions};
#[derive(Debug)]
beacon_node/beacon_chain/src/block_verification.rs
+1 -1
View File
@@ -88,12 +88,12 @@ use store::{Error as DBError, HotStateSummary, KeyValueStore, StoreOp};
use task_executor::JoinHandle;
use tree_hash::TreeHash;
use types::signed_block_and_blobs::BlockWrapper;
use types::ExecPayload;
use types::{
BeaconBlockRef, BeaconState, BeaconStateError, BlindedPayload, ChainSpec, CloneConfig, Epoch,
EthSpec, ExecutionBlockHash, Hash256, InconsistentFork, PublicKey, PublicKeyBytes,
RelativeEpoch, SignedBeaconBlock, SignedBeaconBlockHeader, Slot,
};
use types::{BlobsSidecar, ExecPayload};
pub const POS_PANDA_BANNER: &str = r#"
,,, ,,, ,,, ,,,
beacon_node/beacon_chain/src/execution_payload.rs
+5 -16
View File
@@ -17,11 +17,9 @@ use fork_choice::{InvalidationOperation, PayloadVerificationStatus};
use proto_array::{Block as ProtoBlock, ExecutionStatus};
use slog::debug;
use slot_clock::SlotClock;
#[cfg(feature = "withdrawals")]
use state_processing::per_block_processing::get_expected_withdrawals;
use state_processing::per_block_processing::{
compute_timestamp_at_slot, is_execution_enabled, is_merge_transition_complete,
partially_verify_execution_payload,
compute_timestamp_at_slot, get_expected_withdrawals, is_execution_enabled,
is_merge_transition_complete, partially_verify_execution_payload,
};
use std::sync::Arc;
use tokio::task::JoinHandle;
@@ -382,7 +380,6 @@ pub fn get_execution_payload<
let random = *state.get_randao_mix(current_epoch)?;
let latest_execution_payload_header_block_hash =
state.latest_execution_payload_header()?.block_hash();
#[cfg(feature = "withdrawals")]
let withdrawals = match state {
&BeaconState::Capella(_) | &BeaconState::Eip4844(_) => {
Some(get_expected_withdrawals(state, spec)?.into())
@@ -407,7 +404,6 @@ pub fn get_execution_payload<
proposer_index,
latest_execution_payload_header_block_hash,
builder_params,
#[cfg(feature = "withdrawals")]
withdrawals,
)
.await
@@ -442,7 +438,7 @@ pub async fn prepare_execution_payload<T, Payload>(
proposer_index: u64,
latest_execution_payload_header_block_hash: ExecutionBlockHash,
builder_params: BuilderParams,
#[cfg(feature = "withdrawals")] withdrawals: Option<Vec<Withdrawal>>,
withdrawals: Option<Vec<Withdrawal>>,
) -> Result<BlockProposalContents<T::EthSpec, Payload>, BlockProductionError>
where
T: BeaconChainTypes,
@@ -504,15 +500,8 @@ where
let suggested_fee_recipient = execution_layer
.get_suggested_fee_recipient(proposer_index)
.await;
let payload_attributes = PayloadAttributes::new(
timestamp,
random,
suggested_fee_recipient,
#[cfg(feature = "withdrawals")]
withdrawals,
#[cfg(not(feature = "withdrawals"))]
None,
);
let payload_attributes =
PayloadAttributes::new(timestamp, random, suggested_fee_recipient, withdrawals);
// Note: the suggested_fee_recipient is stored in the `execution_layer`, it will add this parameter.
//
beacon_node/execution_layer/Cargo.toml
-1
View File
@@ -5,7 +5,6 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[features]
withdrawals = ["state_processing/withdrawals", "types/withdrawals", "eth2/withdrawals"]
withdrawals-processing = ["state_processing/withdrawals-processing", "eth2/withdrawals-processing"]
[dependencies]
beacon_node/execution_layer/src/engine_api.rs
-3
View File
@@ -165,7 +165,6 @@ pub struct ExecutionBlockWithTransactions<T: EthSpec> {
#[serde(rename = "hash")]
pub block_hash: ExecutionBlockHash,
pub transactions: Vec<Transaction>,
#[cfg(feature = "withdrawals")]
#[superstruct(only(Capella, Eip4844))]
pub withdrawals: Vec<JsonWithdrawal>,
}
@@ -215,7 +214,6 @@ impl<T: EthSpec> TryFrom<ExecutionPayload<T>> for ExecutionBlockWithTransactions
.iter()
.map(|tx| Transaction::decode(&Rlp::new(tx)))
.collect::<Result<Vec<_>, _>>()?,
#[cfg(feature = "withdrawals")]
withdrawals: Vec::from(block.withdrawals)
.into_iter()
.map(|withdrawal| withdrawal.into())
@@ -243,7 +241,6 @@ impl<T: EthSpec> TryFrom<ExecutionPayload<T>> for ExecutionBlockWithTransactions
.iter()
.map(|tx| Transaction::decode(&Rlp::new(tx)))
.collect::<Result<Vec<_>, _>>()?,
#[cfg(feature = "withdrawals")]
withdrawals: Vec::from(block.withdrawals)
.into_iter()
.map(|withdrawal| withdrawal.into())
beacon_node/execution_layer/src/engine_api/http.rs
+5 -5
View File
@@ -779,7 +779,7 @@ impl HttpJsonRpc {
) -> Result<ExecutionPayload<T>, Error> {
let params = json!([JsonPayloadIdRequest::from(payload_id)]);
let payload_v2: JsonExecutionPayloadV2<T> = self
let response: JsonGetPayloadResponse<T> = self
.rpc_request(
ENGINE_GET_PAYLOAD_V2,
params,
@@ -787,7 +787,7 @@ impl HttpJsonRpc {
)
.await?;
JsonExecutionPayload::V2(payload_v2).try_into_execution_payload(fork_name)
JsonExecutionPayload::V2(response.execution_payload).try_into_execution_payload(fork_name)
}
pub async fn get_payload_v3<T: EthSpec>(
@@ -889,11 +889,11 @@ impl HttpJsonRpc {
pub async fn supported_apis_v1(&self) -> Result<SupportedApis, Error> {
Ok(SupportedApis {
new_payload_v1: true,
new_payload_v2: cfg!(all(feature = "withdrawals", not(test))),
new_payload_v2: cfg!(not(test)),
forkchoice_updated_v1: true,
forkchoice_updated_v2: cfg!(all(feature = "withdrawals", not(test))),
forkchoice_updated_v2: cfg!(not(test)),
get_payload_v1: true,
get_payload_v2: cfg!(all(feature = "withdrawals", not(test))),
get_payload_v2: cfg!(not(test)),
exchange_transition_configuration_v1: true,
})
}
beacon_node/execution_layer/src/engine_api/json_structures.rs
+9 -8
View File
@@ -164,7 +164,6 @@ impl<T: EthSpec> JsonExecutionPayload<T> {
base_fee_per_gas: v2.base_fee_per_gas,
block_hash: v2.block_hash,
transactions: v2.transactions,
#[cfg(feature = "withdrawals")]
withdrawals: v2
.withdrawals
.map(|v| {
@@ -192,7 +191,6 @@ impl<T: EthSpec> JsonExecutionPayload<T> {
excess_data_gas: v2.excess_data_gas.ok_or_else(|| Error::BadConversion("Null `excess_data_gas` field converting JsonExecutionPayloadV2 -> ExecutionPayloadEip4844".to_string()))?,
block_hash: v2.block_hash,
transactions: v2.transactions,
#[cfg(feature = "withdrawals")]
withdrawals: v2
.withdrawals
.map(|v| {
@@ -280,7 +278,6 @@ impl<T: EthSpec> TryFrom<ExecutionPayload<T>> for JsonExecutionPayloadV2<T> {
excess_data_gas: None,
block_hash: capella.block_hash,
transactions: capella.transactions,
#[cfg(feature = "withdrawals")]
withdrawals: Some(
Vec::from(capella.withdrawals)
.into_iter()
@@ -288,8 +285,6 @@ impl<T: EthSpec> TryFrom<ExecutionPayload<T>> for JsonExecutionPayloadV2<T> {
.collect::<Vec<_>>()
.into(),
),
#[cfg(not(feature = "withdrawals"))]
withdrawals: None,
}),
ExecutionPayload::Eip4844(eip4844) => Ok(JsonExecutionPayloadV2 {
parent_hash: eip4844.parent_hash,
@@ -307,7 +302,6 @@ impl<T: EthSpec> TryFrom<ExecutionPayload<T>> for JsonExecutionPayloadV2<T> {
excess_data_gas: Some(eip4844.excess_data_gas),
block_hash: eip4844.block_hash,
transactions: eip4844.transactions,
#[cfg(feature = "withdrawals")]
withdrawals: Some(
Vec::from(eip4844.withdrawals)
.into_iter()
@@ -315,13 +309,20 @@ impl<T: EthSpec> TryFrom<ExecutionPayload<T>> for JsonExecutionPayloadV2<T> {
.collect::<Vec<_>>()
.into(),
),
#[cfg(not(feature = "withdrawals"))]
withdrawals: None,
}),
}
}
}
#[derive(Debug, PartialEq, Serialize, Deserialize)]
#[serde(bound = "T: EthSpec", rename_all = "camelCase")]
pub struct JsonGetPayloadResponse<T: EthSpec> {
pub execution_payload: JsonExecutionPayloadV2<T>,
// uncomment this when geth fixes its serialization
//#[serde(with = "eth2_serde_utils::u256_hex_be")]
//pub block_value: Uint256,
}
#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct JsonWithdrawal {
beacon_node/execution_layer/src/engines.rs
+1 -1
View File
@@ -11,7 +11,7 @@ use std::sync::Arc;
use task_executor::TaskExecutor;
use tokio::sync::{watch, Mutex, RwLock};
use tokio_stream::wrappers::WatchStream;
use types::{Address, ExecutionBlockHash, ForkName, Hash256};
use types::{ExecutionBlockHash, ForkName};
/// The number of payload IDs that will be stored for each `Engine`.
///
beacon_node/execution_layer/src/lib.rs
-6
View File
@@ -1642,7 +1642,6 @@ impl<T: EthSpec> ExecutionLayer<T> {
})
}
ExecutionBlockWithTransactions::Capella(capella_block) => {
#[cfg(feature = "withdrawals")]
let withdrawals = VariableList::new(
capella_block
.withdrawals
@@ -1651,7 +1650,6 @@ impl<T: EthSpec> ExecutionLayer<T> {
.collect(),
)
.map_err(ApiError::DeserializeWithdrawals)?;
ExecutionPayload::Capella(ExecutionPayloadCapella {
parent_hash: capella_block.parent_hash,
fee_recipient: capella_block.fee_recipient,
@@ -1667,12 +1665,10 @@ impl<T: EthSpec> ExecutionLayer<T> {
base_fee_per_gas: capella_block.base_fee_per_gas,
block_hash: capella_block.block_hash,
transactions,
#[cfg(feature = "withdrawals")]
withdrawals,
})
}
ExecutionBlockWithTransactions::Eip4844(eip4844_block) => {
#[cfg(feature = "withdrawals")]
let withdrawals = VariableList::new(
eip4844_block
.withdrawals
@@ -1681,7 +1677,6 @@ impl<T: EthSpec> ExecutionLayer<T> {
.collect(),
)
.map_err(ApiError::DeserializeWithdrawals)?;
ExecutionPayload::Eip4844(ExecutionPayloadEip4844 {
parent_hash: eip4844_block.parent_hash,
fee_recipient: eip4844_block.fee_recipient,
@@ -1698,7 +1693,6 @@ impl<T: EthSpec> ExecutionLayer<T> {
excess_data_gas: eip4844_block.excess_data_gas,
block_hash: eip4844_block.block_hash,
transactions,
#[cfg(feature = "withdrawals")]
withdrawals,
})
}
beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs
-3
View File
@@ -103,10 +103,7 @@ impl<T: EthSpec> MockExecutionLayer<T> {
prev_randao,
Address::repeat_byte(42),
// FIXME: think about how to handle different forks / withdrawals here..
#[cfg(feature = "withdrawals")]
Some(vec![]),
#[cfg(not(feature = "withdrawals"))]
None,
);
// Insert a proposer to ensure the fork choice updated command works.
beacon_node/http_api/src/publish_blocks.rs
+2 -3
View File
@@ -11,9 +11,8 @@ use tokio::sync::mpsc::UnboundedSender;
use tree_hash::TreeHash;
use types::signed_block_and_blobs::BlockWrapper;
use types::{
AbstractExecPayload, BlindedPayload, BlobsSidecar, EthSpec, ExecPayload, ExecutionBlockHash,
FullPayload, Hash256, SignedBeaconBlock, SignedBeaconBlockAndBlobsSidecar,
SignedBeaconBlockEip4844,
AbstractExecPayload, BlindedPayload, EthSpec, ExecPayload, ExecutionBlockHash, FullPayload,
Hash256, SignedBeaconBlock, SignedBeaconBlockAndBlobsSidecar,
};
use warp::Rejection;
beacon_node/lighthouse_network/src/rpc/protocol.rs
+1 -1
View File
@@ -22,7 +22,7 @@ use tokio_util::{
};
use types::BlobsSidecar;
use types::{
BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockMerge, Blob, EmptyBlock, EthSpec,
BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockMerge, EmptyBlock, EthSpec,
ForkContext, ForkName, Hash256, MainnetEthSpec, Signature, SignedBeaconBlock,
};
beacon_node/lighthouse_network/src/types/pubsub.rs
+4 -8
View File
@@ -3,20 +3,16 @@
use crate::types::{GossipEncoding, GossipKind, GossipTopic};
use crate::TopicHash;
use libp2p::gossipsub::{DataTransform, GossipsubMessage, RawGossipsubMessage};
use serde_derive::{Deserialize, Serialize};
use snap::raw::{decompress_len, Decoder, Encoder};
use ssz::{Decode, Encode};
use ssz_derive::{Decode, Encode};
use std::boxed::Box;
use std::io::{Error, ErrorKind};
use std::sync::Arc;
use tree_hash_derive::TreeHash;
use types::{
Attestation, AttesterSlashing, BlobsSidecar, EthSpec, ForkContext, ForkName,
LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing,
SignedAggregateAndProof, SignedBeaconBlock, SignedBeaconBlockAltair,
SignedBeaconBlockAndBlobsSidecar, SignedBeaconBlockBase, SignedBeaconBlockCapella,
SignedBeaconBlockEip4844, SignedBeaconBlockMerge, SignedBlsToExecutionChange,
Attestation, AttesterSlashing, EthSpec, ForkContext, ForkName, LightClientFinalityUpdate,
LightClientOptimisticUpdate, ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock,
SignedBeaconBlockAltair, SignedBeaconBlockAndBlobsSidecar, SignedBeaconBlockBase,
SignedBeaconBlockCapella, SignedBeaconBlockMerge, SignedBlsToExecutionChange,
SignedContributionAndProof, SignedVoluntaryExit, SubnetId, SyncCommitteeMessage, SyncSubnetId,
};
beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs
+1 -2
View File
@@ -23,7 +23,6 @@ use slog::{crit, debug, error, warn, Logger};
use slot_clock::SlotClock;
use std::collections::{HashMap, HashSet};
use std::pin::Pin;
use std::sync::Arc;
use std::task::Context;
use std::time::Duration;
use task_executor::TaskExecutor;
@@ -31,7 +30,7 @@ use tokio::sync::mpsc::{self, Receiver, Sender};
use tokio::time::error::Error as TimeError;
use tokio_util::time::delay_queue::{DelayQueue, Key as DelayKey};
use types::signed_block_and_blobs::BlockWrapper;
use types::{Attestation, EthSpec, Hash256, SignedAggregateAndProof, SignedBeaconBlock, SubnetId};
use types::{Attestation, EthSpec, Hash256, SignedAggregateAndProof, SubnetId};
const TASK_NAME: &str = "beacon_processor_reprocess_queue";
const GOSSIP_BLOCKS: &str = "gossip_blocks";
beacon_node/network/src/beacon_processor/worker/gossip_methods.rs
+2 -4
View File
@@ -15,15 +15,13 @@ use lighthouse_network::{Client, MessageAcceptance, MessageId, PeerAction, PeerI
use slog::{crit, debug, error, info, trace, warn};
use slot_clock::SlotClock;
use ssz::Encode;
use std::sync::Arc;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use store::hot_cold_store::HotColdDBError;
use tokio::sync::mpsc;
use types::signed_block_and_blobs::BlockWrapper;
use types::{
Attestation, AttesterSlashing, BlobsSidecar, EthSpec, Hash256, IndexedAttestation,
LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing,
SignedAggregateAndProof, SignedBeaconBlock, SignedBeaconBlockAndBlobsSidecar,
Attestation, AttesterSlashing, EthSpec, Hash256, IndexedAttestation, LightClientFinalityUpdate,
LightClientOptimisticUpdate, ProposerSlashing, SignedAggregateAndProof,
SignedBlsToExecutionChange, SignedContributionAndProof, SignedVoluntaryExit, Slot, SubnetId,
SyncCommitteeMessage, SyncSubnetId,
};
beacon_node/network/src/beacon_processor/worker/rpc_methods.rs
+11 -3
View File
@@ -12,7 +12,6 @@ use lighthouse_network::rpc::*;
use lighthouse_network::{PeerId, PeerRequestId, ReportSource, Response, SyncInfo};
use slog::{debug, error};
use slot_clock::SlotClock;
use ssz_types::VariableList;
use std::sync::Arc;
use task_executor::TaskExecutor;
use types::light_client_bootstrap::LightClientBootstrap;
@@ -521,6 +520,15 @@ impl<T: BeaconChainTypes> Worker<T> {
"block_root" => ?root,
"error" => ?e
);
// send the stream terminator
self.send_error_response(
peer_id,
RPCResponseErrorCode::ServerError,
"Failed fetching blocks".into(),
request_id,
);
send_response = false;
break;
}
}
@@ -572,7 +580,7 @@ impl<T: BeaconChainTypes> Worker<T> {
/// Handle a `BlobsByRange` request from the peer.
pub fn handle_blobs_by_range_request(
self,
executor: TaskExecutor,
_executor: TaskExecutor,
send_on_drop: SendOnDrop,
peer_id: PeerId,
request_id: PeerRequestId,
@@ -647,7 +655,7 @@ impl<T: BeaconChainTypes> Worker<T> {
let block_roots = block_roots.into_iter().flatten().collect::<Vec<_>>();
let mut blobs_sent = 0;
let mut send_response = true;
let send_response = true;
for root in block_roots {
match self.chain.store.get_blobs(&root) {
beacon_node/network/src/beacon_processor/worker/sync_methods.rs
+1 -4
View File
@@ -17,10 +17,7 @@ use slog::{debug, error, info, warn};
use std::sync::Arc;
use tokio::sync::mpsc;
use types::signed_block_and_blobs::BlockWrapper;
use types::{
Epoch, Hash256, SignedBeaconBlock, SignedBeaconBlockAndBlobsSidecar,
SignedBeaconBlockAndBlobsSidecarDecode,
};
use types::{Epoch, Hash256, SignedBeaconBlock};
/// Id associated to a batch processing request, either a sync batch or a parent lookup.
#[derive(Clone, Debug, PartialEq)]
beacon_node/network/src/sync/backfill_sync/mod.rs
+1 -1
View File
@@ -33,7 +33,7 @@ use types::{Epoch, EthSpec};
/// we will negatively report peers with poor bandwidth. This can be set arbitrarily high, in which
/// case the responder will fill the response up to the max request size, assuming they have the
/// bandwidth to do so.
pub const BACKFILL_EPOCHS_PER_BATCH: u64 = 2;
pub const BACKFILL_EPOCHS_PER_BATCH: u64 = 1;
/// The maximum number of batches to queue before requesting more.
const BACKFILL_BATCH_BUFFER_SIZE: u8 = 20;
beacon_node/network/src/sync/block_lookups/mod.rs
+1 -4
View File
@@ -4,15 +4,12 @@ use std::time::Duration;
use beacon_chain::{BeaconChainTypes, BlockError};
use fnv::FnvHashMap;
use futures::StreamExt;
use itertools::{Either, Itertools};
use lighthouse_network::rpc::{RPCError, RPCResponseErrorCode};
use lighthouse_network::{PeerAction, PeerId};
use lru_cache::LRUTimeCache;
use slog::{debug, error, trace, warn, Logger};
use smallvec::SmallVec;
use std::sync::Arc;
use store::{Hash256, SignedBeaconBlock};
use store::Hash256;
use types::signed_block_and_blobs::BlockWrapper;
use crate::beacon_processor::{ChainSegmentProcessId, WorkEvent};
beacon_node/network/src/sync/block_lookups/parent_lookup.rs
+1 -2
View File
@@ -1,8 +1,7 @@
use super::RootBlockTuple;
use beacon_chain::BeaconChainTypes;
use lighthouse_network::PeerId;
use std::sync::Arc;
use store::{Hash256, SignedBeaconBlock};
use store::Hash256;
use strum::IntoStaticStr;
use types::signed_block_and_blobs::BlockWrapper;
beacon_node/network/src/sync/block_lookups/single_block_lookup.rs
+1 -2
View File
@@ -4,8 +4,7 @@ use lighthouse_network::{rpc::BlocksByRootRequest, PeerId};
use rand::seq::IteratorRandom;
use ssz_types::VariableList;
use std::collections::HashSet;
use std::sync::Arc;
use store::{EthSpec, Hash256, SignedBeaconBlock};
use store::{EthSpec, Hash256};
use strum::IntoStaticStr;
use types::signed_block_and_blobs::BlockWrapper;
beacon_node/network/src/sync/block_lookups/tests.rs
+2 -2
View File
@@ -10,11 +10,11 @@ use beacon_chain::builder::Witness;
use beacon_chain::eth1_chain::CachingEth1Backend;
use lighthouse_network::{NetworkGlobals, Request};
use slog::{Drain, Level};
use slot_clock::{SlotClock, SystemTimeSlotClock};
use slot_clock::SystemTimeSlotClock;
use store::MemoryStore;
use tokio::sync::mpsc;
use types::test_utils::{SeedableRng, TestRandom, XorShiftRng};
use types::{EthSpec, MainnetEthSpec, MinimalEthSpec as E, Slot};
use types::MinimalEthSpec as E;
type T = Witness<SystemTimeSlotClock, CachingEth1Backend<E>, E, MemoryStore<E>, MemoryStore<E>>;
beacon_node/network/src/sync/block_sidecar_coupling.rs
+79
View File
@@ -0,0 +1,79 @@
use std::{collections::VecDeque, sync::Arc};
use types::{
signed_block_and_blobs::BlockWrapper, BlobsSidecar, EthSpec, SignedBeaconBlock,
SignedBeaconBlockAndBlobsSidecar,
};
#[derive(Debug, Default)]
pub struct BlockBlobRequestInfo<T: EthSpec> {
/// Blocks we have received awaiting for their corresponding sidecar.
accumulated_blocks: VecDeque<Arc<SignedBeaconBlock<T>>>,
/// Sidecars we have received awaiting for their corresponding block.
accumulated_sidecars: VecDeque<Arc<BlobsSidecar<T>>>,
/// Whether the individual RPC request for blocks is finished or not.
is_blocks_stream_terminated: bool,
/// Whether the individual RPC request for sidecars is finished or not.
is_sidecars_stream_terminated: bool,
}
impl<T: EthSpec> BlockBlobRequestInfo<T> {
pub fn add_block_response(&mut self, maybe_block: Option<Arc<SignedBeaconBlock<T>>>) {
match maybe_block {
Some(block) => self.accumulated_blocks.push_back(block),
None => self.is_blocks_stream_terminated = true,
}
}
pub fn add_sidecar_response(&mut self, maybe_sidecar: Option<Arc<BlobsSidecar<T>>>) {
match maybe_sidecar {
Some(sidecar) => self.accumulated_sidecars.push_back(sidecar),
None => self.is_sidecars_stream_terminated = true,
}
}
pub fn into_responses(self) -> Result<Vec<BlockWrapper<T>>, &'static str> {
let BlockBlobRequestInfo {
accumulated_blocks,
mut accumulated_sidecars,
..
} = self;
// ASSUMPTION: There can't be more more blobs than blocks. i.e. sending any blob (empty
// included) for a skipped slot is not permitted.
let pairs = accumulated_blocks
.into_iter()
.map(|beacon_block| {
if accumulated_sidecars
.front()
.map(|sidecar| sidecar.beacon_block_slot == beacon_block.slot())
.unwrap_or(false)
{
let blobs_sidecar =
accumulated_sidecars.pop_front().ok_or("missing sidecar")?;
Ok(BlockWrapper::BlockAndBlob {
block_sidecar_pair: SignedBeaconBlockAndBlobsSidecar {
beacon_block,
blobs_sidecar,
},
})
} else {
Ok(BlockWrapper::Block {
block: beacon_block,
})
}
})
.collect::<Result<Vec<_>, _>>();
// if accumulated sidecars is not empty, throw an error.
if !accumulated_sidecars.is_empty() {
return Err("Received more sidecars than blocks");
}
pairs
}
pub fn is_finished(&self) -> bool {
self.is_blocks_stream_terminated && self.is_sidecars_stream_terminated
}
}
beacon_node/network/src/sync/manager.rs
+117 -84
View File
@@ -35,7 +35,7 @@
use super::backfill_sync::{BackFillSync, ProcessResult, SyncStart};
use super::block_lookups::BlockLookups;
use super::network_context::SyncNetworkContext;
use super::network_context::{BlockOrBlob, SyncNetworkContext};
use super::peer_sync_info::{remote_sync_type, PeerSyncType};
use super::range_sync::{RangeSync, RangeSyncType, EPOCHS_PER_BATCH};
use crate::beacon_processor::{ChainSegmentProcessId, WorkEvent as BeaconWorkEvent};
@@ -45,11 +45,11 @@ use crate::sync::range_sync::ExpectedBatchTy;
use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError, EngineState};
use futures::StreamExt;
use lighthouse_network::rpc::methods::MAX_REQUEST_BLOCKS;
use lighthouse_network::rpc::{RPCError, RPCResponseErrorCode};
use lighthouse_network::rpc::RPCError;
use lighthouse_network::types::{NetworkGlobals, SyncState};
use lighthouse_network::SyncInfo;
use lighthouse_network::{PeerAction, PeerId};
use slog::{crit, debug, error, info, trace, Logger};
use slog::{crit, debug, error, info, trace, warn, Logger};
use std::boxed::Box;
use std::ops::Sub;
use std::sync::Arc;
@@ -746,17 +746,17 @@ impl<T: BeaconChainTypes> SyncManager<T> {
&mut self.network,
),
RequestId::BackFillSync { id } => {
if let Some((batch_id, block)) = self.network.backfill_sync_block_response(
id,
beacon_block,
ExpectedBatchTy::OnlyBlock,
) {
let is_stream_terminator = beacon_block.is_none();
if let Some(batch_id) = self
.network
.backfill_sync_only_blocks_response(id, is_stream_terminator)
{
match self.backfill_sync.on_block_response(
&mut self.network,
batch_id,
&peer_id,
id,
block,
beacon_block.map(|block| BlockWrapper::Block { block }),
) {
Ok(ProcessResult::SyncCompleted) => self.update_sync_state(),
Ok(ProcessResult::Successful) => {}
@@ -769,61 +769,125 @@ impl<T: BeaconChainTypes> SyncManager<T> {
}
}
RequestId::RangeSync { id } => {
if let Some((chain_id, batch_id, block)) = self.network.range_sync_block_response(
id,
beacon_block,
ExpectedBatchTy::OnlyBlock,
) {
let is_stream_terminator = beacon_block.is_none();
if let Some((chain_id, batch_id)) = self
.network
.range_sync_block_response(id, is_stream_terminator)
{
self.range_sync.blocks_by_range_response(
&mut self.network,
peer_id,
chain_id,
batch_id,
id,
block,
beacon_block.map(|block| BlockWrapper::Block { block }),
);
self.update_sync_state();
}
}
RequestId::BackFillSidecarPair { id } => {
if let Some((batch_id, block)) = self.network.backfill_sync_block_response(
id,
beacon_block,
ExpectedBatchTy::OnlyBlockBlobs,
) {
match self.backfill_sync.on_block_response(
&mut self.network,
batch_id,
&peer_id,
id,
block,
) {
Ok(ProcessResult::SyncCompleted) => self.update_sync_state(),
Ok(ProcessResult::Successful) => {}
Err(_error) => {
// The backfill sync has failed, errors are reported
// within.
self.update_sync_state();
self.block_blob_backfill_response(id, peer_id, beacon_block.into())
}
RequestId::RangeSidecarPair { id } => {
self.block_blob_range_response(id, peer_id, beacon_block.into())
}
}
}
/// Handles receiving a response for a range sync request that should have both blocks and
/// blobs.
fn block_blob_range_response(
&mut self,
id: Id,
peer_id: PeerId,
block_or_blob: BlockOrBlob<T::EthSpec>,
) {
if let Some((chain_id, batch_id, block_responses)) = self
.network
.range_sync_block_and_blob_response(id, block_or_blob)
{
match block_responses {
Ok(blocks) => {
for block in blocks
.into_iter()
.map(|block| Some(block))
// chain the stream terminator
.chain(vec![None])
{
self.range_sync.blocks_by_range_response(
&mut self.network,
peer_id,
chain_id,
batch_id,
id,
block,
);
self.update_sync_state();
}
}
Err(e) => {
// inform range that the request needs to be treated as failed
// With time we will want to downgrade this log
warn!(
self.log, "Blocks and blobs request for range received invalid data";
"peer_id" => %peer_id, "batch_id" => batch_id, "error" => e
);
// TODO: penalize the peer for being a bad boy
let id = RequestId::RangeSidecarPair { id };
self.inject_error(peer_id, id, RPCError::InvalidData(e.into()))
}
}
}
}
/// Handles receiving a response for a Backfill sync request that should have both blocks and
/// blobs.
fn block_blob_backfill_response(
&mut self,
id: Id,
peer_id: PeerId,
block_or_blob: BlockOrBlob<T::EthSpec>,
) {
if let Some((batch_id, block_responses)) = self
.network
.backfill_sync_block_and_blob_response(id, block_or_blob)
{
match block_responses {
Ok(blocks) => {
for block in blocks
.into_iter()
.map(|block| Some(block))
// chain the stream terminator
.chain(vec![None])
{
match self.backfill_sync.on_block_response(
&mut self.network,
batch_id,
&peer_id,
id,
block,
) {
Ok(ProcessResult::SyncCompleted) => self.update_sync_state(),
Ok(ProcessResult::Successful) => {}
Err(_error) => {
// The backfill sync has failed, errors are reported
// within.
self.update_sync_state();
}
}
}
}
}
RequestId::RangeSidecarPair { id } => {
if let Some((chain_id, batch_id, block)) = self.network.range_sync_block_response(
id,
beacon_block,
ExpectedBatchTy::OnlyBlockBlobs,
) {
self.range_sync.blocks_by_range_response(
&mut self.network,
peer_id,
chain_id,
batch_id,
id,
block,
Err(e) => {
// inform backfill that the request needs to be treated as failed
// With time we will want to downgrade this log
warn!(
self.log, "Blocks and blobs request for backfill received invalid data";
"peer_id" => %peer_id, "batch_id" => batch_id, "error" => e
);
self.update_sync_state();
// TODO: penalize the peer for being a bad boy
let id = RequestId::BackFillSidecarPair { id };
self.inject_error(peer_id, id, RPCError::InvalidData(e.into()))
}
}
}
@@ -834,54 +898,23 @@ impl<T: BeaconChainTypes> SyncManager<T> {
request_id: RequestId,
peer_id: PeerId,
maybe_sidecar: Option<Arc<BlobsSidecar<<T>::EthSpec>>>,
seen_timestamp: Duration,
_seen_timestamp: Duration,
) {
match request_id {
RequestId::SingleBlock { id } | RequestId::ParentLookup { id } => {
RequestId::SingleBlock { .. } | RequestId::ParentLookup { .. } => {
unreachable!("There is no such thing as a singular 'by root' glob request that is not accompanied by the block")
}
RequestId::BackFillSync { .. } => {
unreachable!("An only blocks request does not receive sidecars")
}
RequestId::BackFillSidecarPair { id } => {
if let Some((batch_id, block)) = self
.network
.backfill_sync_sidecar_response(id, maybe_sidecar)
{
match self.backfill_sync.on_block_response(
&mut self.network,
batch_id,
&peer_id,
id,
block,
) {
Ok(ProcessResult::SyncCompleted) => self.update_sync_state(),
Ok(ProcessResult::Successful) => {}
Err(_error) => {
// The backfill sync has failed, errors are reported
// within.
self.update_sync_state();
}
}
}
self.block_blob_backfill_response(id, peer_id, maybe_sidecar.into())
}
RequestId::RangeSync { .. } => {
unreachable!("And only blocks range request does not receive sidecars")
unreachable!("Only-blocks range requests don't receive sidecars")
}
RequestId::RangeSidecarPair { id } => {
if let Some((chain_id, batch_id, block)) =
self.network.range_sync_sidecar_response(id, maybe_sidecar)
{
self.range_sync.blocks_by_range_response(
&mut self.network,
peer_id,
chain_id,
batch_id,
id,
block,
);
self.update_sync_state();
}
self.block_blob_range_response(id, peer_id, maybe_sidecar.into())
}
}
}
beacon_node/network/src/sync/mod.rs
+1
View File
@@ -3,6 +3,7 @@
//! Stores the various syncing methods for the beacon chain.
mod backfill_sync;
mod block_lookups;
mod block_sidecar_coupling;
pub mod manager;
mod network_context;
mod peer_sync_info;
beacon_node/network/src/sync/network_context.rs
+76 -181
View File
@@ -1,6 +1,7 @@
//! Provides network functionality for the Syncing thread. This fundamentally wraps a network
//! channel and stores a global RPC ID to perform requests.
use super::block_sidecar_coupling::BlockBlobRequestInfo;
use super::manager::{Id, RequestId as SyncRequestId};
use super::range_sync::{BatchId, ChainId, ExpectedBatchTy};
use crate::beacon_processor::WorkEvent;
@@ -13,59 +14,11 @@ use lighthouse_network::rpc::methods::BlobsByRangeRequest;
use lighthouse_network::rpc::{BlocksByRangeRequest, BlocksByRootRequest, GoodbyeReason};
use lighthouse_network::{Client, NetworkGlobals, PeerAction, PeerId, ReportSource, Request};
use slog::{debug, trace, warn};
use slot_clock::SlotClock;
use std::collections::hash_map::Entry;
use std::collections::VecDeque;
use std::sync::Arc;
use tokio::sync::mpsc;
use types::signed_block_and_blobs::BlockWrapper;
use types::{
BlobsSidecar, ChainSpec, EthSpec, SignedBeaconBlock, SignedBeaconBlockAndBlobsSidecar,
};
#[derive(Debug, Default)]
struct BlockBlobRequestInfo<T: EthSpec> {
/// Blocks we have received awaiting for their corresponding sidecar.
accumulated_blocks: VecDeque<Arc<SignedBeaconBlock<T>>>,
/// Sidecars we have received awaiting for their corresponding block.
accumulated_sidecars: VecDeque<Arc<BlobsSidecar<T>>>,
/// Whether the individual RPC request for blocks is finished or not.
is_blocks_rpc_finished: bool,
/// Whether the individual RPC request for sidecars is finished or not.
is_sidecar_rpc_finished: bool,
}
impl<T: EthSpec> BlockBlobRequestInfo<T> {
pub fn add_block_response(&mut self, maybe_block: Option<Arc<SignedBeaconBlock<T>>>) {
match maybe_block {
Some(block) => self.accumulated_blocks.push_back(block),
None => self.is_blocks_rpc_finished = true,
}
}
pub fn add_sidecar_response(&mut self, maybe_sidecar: Option<Arc<BlobsSidecar<T>>>) {
match maybe_sidecar {
Some(sidecar) => self.accumulated_sidecars.push_back(sidecar),
None => self.is_sidecar_rpc_finished = true,
}
}
pub fn pop_response(&mut self) -> Option<SignedBeaconBlockAndBlobsSidecar<T>> {
if !self.accumulated_blocks.is_empty() && !self.accumulated_sidecars.is_empty() {
let beacon_block = self.accumulated_blocks.pop_front().expect("non empty");
let blobs_sidecar = self.accumulated_sidecars.pop_front().expect("non empty");
return Some(SignedBeaconBlockAndBlobsSidecar {
beacon_block,
blobs_sidecar,
});
}
None
}
pub fn is_finished(&self) -> bool {
self.is_blocks_rpc_finished && self.is_sidecar_rpc_finished
}
}
use types::{BlobsSidecar, EthSpec, SignedBeaconBlock};
/// Wraps a Network channel to employ various RPC related network functionality for the Sync manager. This includes management of a global RPC request Id.
pub struct SyncNetworkContext<T: BeaconChainTypes> {
@@ -104,6 +57,24 @@ pub struct SyncNetworkContext<T: BeaconChainTypes> {
log: slog::Logger,
}
/// Small enumeration to make dealing with block and blob requests easier.
pub enum BlockOrBlob<T: EthSpec> {
Block(Option<Arc<SignedBeaconBlock<T>>>),
Blob(Option<Arc<BlobsSidecar<T>>>),
}
impl<T: EthSpec> From<Option<Arc<SignedBeaconBlock<T>>>> for BlockOrBlob<T> {
fn from(block: Option<Arc<SignedBeaconBlock<T>>>) -> Self {
BlockOrBlob::Block(block)
}
}
impl<T: EthSpec> From<Option<Arc<BlobsSidecar<T>>>> for BlockOrBlob<T> {
fn from(blob: Option<Arc<BlobsSidecar<T>>>) -> Self {
BlockOrBlob::Blob(blob)
}
}
impl<T: BeaconChainTypes> SyncNetworkContext<T> {
pub fn new(
network_send: mpsc::UnboundedSender<NetworkMessage<T::EthSpec>>,
@@ -300,91 +271,43 @@ impl<T: BeaconChainTypes> SyncNetworkContext<T> {
}
}
/// Received a blocks by range response.
/// Response for a request that is only for blocks.
pub fn range_sync_block_response(
&mut self,
request_id: Id,
maybe_block: Option<Arc<SignedBeaconBlock<T::EthSpec>>>,
batch_type: ExpectedBatchTy,
) -> Option<(ChainId, BatchId, Option<BlockWrapper<T::EthSpec>>)> {
match batch_type {
ExpectedBatchTy::OnlyBlockBlobs => {
match self.range_sidecar_pair_requests.entry(request_id) {
Entry::Occupied(mut entry) => {
let (chain_id, batch_id, info) = entry.get_mut();
let chain_id = chain_id.clone();
let batch_id = batch_id.clone();
let stream_terminator = maybe_block.is_none();
info.add_block_response(maybe_block);
let maybe_block_wrapped = info.pop_response().map(|block_sidecar_pair| {
BlockWrapper::BlockAndBlob { block_sidecar_pair }
});
if stream_terminator && !info.is_finished() {
return None;
}
if !stream_terminator && maybe_block_wrapped.is_none() {
return None;
}
if info.is_finished() {
entry.remove();
}
Some((chain_id, batch_id, maybe_block_wrapped))
}
Entry::Vacant(_) => None,
}
}
ExpectedBatchTy::OnlyBlock => {
// if the request is just for blocks then it can be removed on a stream termination
match maybe_block {
Some(block) => {
self.range_requests
.get(&request_id)
.cloned()
.map(|(chain_id, batch_id)| {
(chain_id, batch_id, Some(BlockWrapper::Block { block }))
})
}
None => self
.range_requests
.remove(&request_id)
.map(|(chain_id, batch_id)| (chain_id, batch_id, None)),
}
}
is_stream_terminator: bool,
) -> Option<(ChainId, BatchId)> {
if is_stream_terminator {
self.range_requests.remove(&request_id)
} else {
self.range_requests.get(&request_id).copied()
}
}
pub fn range_sync_sidecar_response(
/// Received a blocks by range response for a request that couples blocks and blobs.
pub fn range_sync_block_and_blob_response(
&mut self,
request_id: Id,
maybe_sidecar: Option<Arc<BlobsSidecar<T::EthSpec>>>,
) -> Option<(ChainId, BatchId, Option<BlockWrapper<T::EthSpec>>)> {
block_or_blob: BlockOrBlob<T::EthSpec>,
) -> Option<(
ChainId,
BatchId,
Result<Vec<BlockWrapper<T::EthSpec>>, &'static str>,
)> {
match self.range_sidecar_pair_requests.entry(request_id) {
Entry::Occupied(mut entry) => {
let (chain_id, batch_id, info) = entry.get_mut();
let chain_id = chain_id.clone();
let batch_id = batch_id.clone();
let stream_terminator = maybe_sidecar.is_none();
info.add_sidecar_response(maybe_sidecar);
let maybe_block = info
.pop_response()
.map(|block_sidecar_pair| BlockWrapper::BlockAndBlob { block_sidecar_pair });
if stream_terminator && !info.is_finished() {
return None;
let (_, _, info) = entry.get_mut();
match block_or_blob {
BlockOrBlob::Block(maybe_block) => info.add_block_response(maybe_block),
BlockOrBlob::Blob(maybe_sidecar) => info.add_sidecar_response(maybe_sidecar),
}
if !stream_terminator && maybe_block.is_none() {
return None;
}
if info.is_finished() {
entry.remove();
// If the request is finished, dequeue everything
let (chain_id, batch_id, info) = entry.remove();
Some((chain_id, batch_id, info.into_responses()))
} else {
None
}
Some((chain_id, batch_id, maybe_block))
}
Entry::Vacant(_) => None,
}
@@ -418,65 +341,41 @@ impl<T: BeaconChainTypes> SyncNetworkContext<T> {
}
}
/// Received a blocks by range response.
pub fn backfill_sync_block_response(
/// Response for a request that is only for blocks.
pub fn backfill_sync_only_blocks_response(
&mut self,
request_id: Id,
maybe_block: Option<Arc<SignedBeaconBlock<T::EthSpec>>>,
batch_type: ExpectedBatchTy,
) -> Option<(BatchId, Option<BlockWrapper<T::EthSpec>>)> {
match batch_type {
ExpectedBatchTy::OnlyBlockBlobs => {
match self.backfill_sidecar_pair_requests.entry(request_id) {
Entry::Occupied(mut entry) => {
let (batch_id, info) = entry.get_mut();
let batch_id = batch_id.clone();
info.add_block_response(maybe_block);
let maybe_block = info.pop_response().map(|block_sidecar_pair| {
BlockWrapper::BlockAndBlob { block_sidecar_pair }
});
if info.is_finished() {
entry.remove();
}
Some((batch_id, maybe_block))
}
Entry::Vacant(_) => None,
}
}
ExpectedBatchTy::OnlyBlock => {
// if the request is just for blocks then it can be removed on a stream termination
match maybe_block {
Some(block) => self
.backfill_requests
.get(&request_id)
.cloned()
.map(|batch_id| (batch_id, Some(BlockWrapper::Block { block }))),
None => self
.backfill_requests
.remove(&request_id)
.map(|batch_id| (batch_id, None)),
}
}
is_stream_terminator: bool,
) -> Option<BatchId> {
if is_stream_terminator {
self.backfill_requests
.remove(&request_id)
.map(|batch_id| batch_id)
} else {
self.backfill_requests.get(&request_id).copied()
}
}
pub fn backfill_sync_sidecar_response(
/// Received a blocks by range response for a request that couples blocks and blobs.
pub fn backfill_sync_block_and_blob_response(
&mut self,
request_id: Id,
maybe_sidecar: Option<Arc<BlobsSidecar<T::EthSpec>>>,
) -> Option<(BatchId, Option<BlockWrapper<T::EthSpec>>)> {
block_or_blob: BlockOrBlob<T::EthSpec>,
) -> Option<(BatchId, Result<Vec<BlockWrapper<T::EthSpec>>, &'static str>)> {
match self.backfill_sidecar_pair_requests.entry(request_id) {
Entry::Occupied(mut entry) => {
let (batch_id, info) = entry.get_mut();
let batch_id = batch_id.clone();
info.add_sidecar_response(maybe_sidecar);
let maybe_block = info
.pop_response()
.map(|block_sidecar_pair| BlockWrapper::BlockAndBlob { block_sidecar_pair });
let (_, info) = entry.get_mut();
match block_or_blob {
BlockOrBlob::Block(maybe_block) => info.add_block_response(maybe_block),
BlockOrBlob::Blob(maybe_sidecar) => info.add_sidecar_response(maybe_sidecar),
}
if info.is_finished() {
entry.remove();
// If the request is finished, dequeue everything
let (batch_id, info) = entry.remove();
Some((batch_id, info.into_responses()))
} else {
None
}
Some((batch_id, maybe_block))
}
Entry::Vacant(_) => None,
}
@@ -632,25 +531,21 @@ impl<T: BeaconChainTypes> SyncNetworkContext<T> {
id
}
/// Check whether a batch for this epoch (and only this epoch) should request just blocks or
/// blocks and blobs.
pub fn batch_type(&self, epoch: types::Epoch) -> ExpectedBatchTy {
// Keep tests only for blocks.
const _: () = assert!(
super::backfill_sync::BACKFILL_EPOCHS_PER_BATCH == 1
&& super::range_sync::EPOCHS_PER_BATCH == 1,
"To deal with alignment with 4844 boundaries, batches need to be of just one epoch"
);
#[cfg(test)]
{
// Keep tests only for blocks.
return ExpectedBatchTy::OnlyBlock;
}
#[cfg(not(test))]
{
use super::range_sync::EPOCHS_PER_BATCH;
assert_eq!(
EPOCHS_PER_BATCH, 1,
"If this is not one, everything will fail horribly"
);
// Here we need access to the beacon chain, check the fork boundary, the current epoch, the
// blob period to serve and check with that if the batch is a blob batch or not.
// NOTE: This would carelessly assume batch sizes are always 1 epoch, to avoid needing to
// align with the batch boundary.
if let Some(data_availability_boundary) = self.chain.data_availability_boundary() {
if epoch >= data_availability_boundary {
ExpectedBatchTy::OnlyBlockBlobs
beacon_node/network/src/sync/range_sync/batch.rs
+5 -2
View File
@@ -5,6 +5,7 @@ use std::collections::HashSet;
use std::hash::{Hash, Hasher};
use std::ops::Sub;
use std::sync::Arc;
use strum::Display;
use types::signed_block_and_blobs::BlockWrapper;
use types::{Epoch, EthSpec, SignedBeaconBlock, SignedBeaconBlockAndBlobsSidecar, Slot};
@@ -40,7 +41,8 @@ impl<T: EthSpec> BatchTy<T> {
pub struct MixedBlockTyErr;
/// Type of expected batch.
#[derive(Debug, Clone)]
#[derive(Debug, Copy, Clone, Display)]
#[strum(serialize_all = "snake_case")]
pub enum ExpectedBatchTy {
OnlyBlockBlobs,
OnlyBlock,
@@ -247,7 +249,7 @@ impl<T: EthSpec, B: BatchConfig> BatchInfo<T, B> {
start_slot: self.start_slot.into(),
count: self.end_slot.sub(self.start_slot).into(),
},
self.batch_type.clone(),
self.batch_type,
)
}
@@ -557,6 +559,7 @@ impl<T: EthSpec, B: BatchConfig> slog::KV for BatchInfo<T, B> {
serializer.emit_usize("processed", self.failed_processing_attempts.len())?;
serializer.emit_u8("processed_no_penalty", self.non_faulty_processing_attempts)?;
serializer.emit_arguments("state", &format_args!("{:?}", self.state))?;
serializer.emit_arguments("batch_ty", &format_args!("{}", self.batch_type));
slog::Result::Ok(())
}
}
beacon_node/network/src/sync/range_sync/chain.rs
-1
View File
@@ -1,5 +1,4 @@
use super::batch::{BatchInfo, BatchProcessingResult, BatchState};
use super::BatchTy;
use crate::beacon_processor::{ChainSegmentProcessId, WorkEvent as BeaconWorkEvent};
use crate::sync::{
manager::Id, network_context::SyncNetworkContext, BatchOperationOutcome, BatchProcessResult,
beacon_node/network/src/sync/range_sync/range.rs
+10 -17
View File
@@ -388,12 +388,11 @@ mod tests {
use slog::{o, Drain};
use tokio::sync::mpsc;
use slot_clock::{SlotClock, SystemTimeSlotClock};
use slot_clock::SystemTimeSlotClock;
use std::collections::HashSet;
use std::sync::Arc;
use std::time::Duration;
use store::MemoryStore;
use types::{Hash256, MainnetEthSpec, MinimalEthSpec as E};
use types::{Hash256, MinimalEthSpec as E};
#[derive(Debug)]
struct FakeStorage {
@@ -686,13 +685,10 @@ mod tests {
// add some peers
let (peer1, local_info, head_info) = rig.head_peer();
range.add_peer(&mut rig.cx, local_info, peer1, head_info);
let ((chain1, batch1, _), id1) = match rig.grab_request(&peer1).0 {
RequestId::Sync(crate::sync::manager::RequestId::RangeSync { id }) => (
rig.cx
.range_sync_block_response(id, None, ExpectedBatchTy::OnlyBlock)
.unwrap(),
id,
),
let ((chain1, batch1), id1) = match rig.grab_request(&peer1).0 {
RequestId::Sync(crate::sync::manager::RequestId::RangeSync { id }) => {
(rig.cx.range_sync_response(id, true).unwrap(), id)
}
other => panic!("unexpected request {:?}", other),
};
@@ -708,13 +704,10 @@ mod tests {
// while the ee is offline, more peers might arrive. Add a new finalized peer.
let (peer2, local_info, finalized_info) = rig.finalized_peer();
range.add_peer(&mut rig.cx, local_info, peer2, finalized_info);
let ((chain2, batch2, _), id2) = match rig.grab_request(&peer2).0 {
RequestId::Sync(crate::sync::manager::RequestId::RangeSync { id }) => (
rig.cx
.range_sync_block_response(id, None, ExpectedBatchTy::OnlyBlock)
.unwrap(),
id,
),
let ((chain2, batch2), id2) = match rig.grab_request(&peer2).0 {
RequestId::Sync(crate::sync::manager::RequestId::RangeSync { id }) => {
(rig.cx.range_sync_response(id, true).unwrap(), id)
}
other => panic!("unexpected request {:?}", other),
};
beacon_node/store/Cargo.toml
-1
View File
@@ -28,5 +28,4 @@ directory = { path = "../../common/directory" }
strum = { version = "0.24.0", features = ["derive"] }
[features]
withdrawals = ["state_processing/withdrawals", "types/withdrawals"]
withdrawals-processing = ["state_processing/withdrawals-processing"]
beacon_node/store/src/lib.rs
+1
View File
@@ -7,6 +7,7 @@
//!
//! Provides a simple API for storing/retrieving all types that sometimes needs type-hints. See
//! tests for implementation examples.
#![allow(dead_code)]
#[macro_use]
extern crate lazy_static;
beacon_node/store/src/partial_beacon_state.rs
-64
View File
@@ -105,10 +105,8 @@ where
pub latest_execution_payload_header: ExecutionPayloadHeaderEip4844<T>,
// Withdrawals
#[cfg(feature = "withdrawals")]
#[superstruct(only(Capella, Eip4844))]
pub next_withdrawal_index: u64,
#[cfg(feature = "withdrawals")]
#[superstruct(only(Capella, Eip4844))]
pub next_withdrawal_validator_index: u64,
}
@@ -199,7 +197,6 @@ impl<T: EthSpec> PartialBeaconState<T> {
latest_execution_payload_header
]
),
#[cfg(feature = "withdrawals")]
BeaconState::Capella(s) => impl_from_state_forgetful!(
s,
outer,
@@ -216,22 +213,6 @@ impl<T: EthSpec> PartialBeaconState<T> {
next_withdrawal_validator_index
]
),
#[cfg(not(feature = "withdrawals"))]
BeaconState::Capella(s) => impl_from_state_forgetful!(
s,
outer,
Capella,
PartialBeaconStateCapella,
[
previous_epoch_participation,
current_epoch_participation,
current_sync_committee,
next_sync_committee,
inactivity_scores,
latest_execution_payload_header
]
),
#[cfg(feature = "withdrawals")]
BeaconState::Eip4844(s) => impl_from_state_forgetful!(
s,
outer,
@@ -248,21 +229,6 @@ impl<T: EthSpec> PartialBeaconState<T> {
next_withdrawal_validator_index
]
),
#[cfg(not(feature = "withdrawals"))]
BeaconState::Eip4844(s) => impl_from_state_forgetful!(
s,
outer,
Eip4844,
PartialBeaconStateEip4844,
[
previous_epoch_participation,
current_epoch_participation,
current_sync_committee,
next_sync_committee,
inactivity_scores,
latest_execution_payload_header
]
),
}
}
@@ -450,7 +416,6 @@ impl<E: EthSpec> TryInto<BeaconState<E>> for PartialBeaconState<E> {
latest_execution_payload_header
]
),
#[cfg(feature = "withdrawals")]
PartialBeaconState::Capella(inner) => impl_try_into_beacon_state!(
inner,
Capella,
@@ -466,21 +431,6 @@ impl<E: EthSpec> TryInto<BeaconState<E>> for PartialBeaconState<E> {
next_withdrawal_validator_index
]
),
#[cfg(not(feature = "withdrawals"))]
PartialBeaconState::Capella(inner) => impl_try_into_beacon_state!(
inner,
Capella,
BeaconStateCapella,
[
previous_epoch_participation,
current_epoch_participation,
current_sync_committee,
next_sync_committee,
inactivity_scores,
latest_execution_payload_header
]
),
#[cfg(feature = "withdrawals")]
PartialBeaconState::Eip4844(inner) => impl_try_into_beacon_state!(
inner,
Eip4844,
@@ -496,20 +446,6 @@ impl<E: EthSpec> TryInto<BeaconState<E>> for PartialBeaconState<E> {
next_withdrawal_validator_index
]
),
#[cfg(not(feature = "withdrawals"))]
PartialBeaconState::Eip4844(inner) => impl_try_into_beacon_state!(
inner,
Eip4844,
BeaconStateEip4844,
[
previous_epoch_participation,
current_epoch_participation,
current_sync_committee,
next_sync_committee,
inactivity_scores,
latest_execution_payload_header
]
),
};
Ok(state)
}