cerc-io/lighthouse
16
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

fix rpc types to free the blobs (#4059)

Browse Source 
* rename to follow name in spec

* use roots and indexes

* wip

* fix req/resp types

* move blob identifier to consensus types
This commit is contained in:
Divma 2023-03-07 16:28:45 -05:00 committed by GitHub
parent 63011c5bca
commit 545532a883
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 51 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs
View File

@ -15,11 +15,11 @@ use std::io::{Read, Write};
use std::marker::PhantomData;
use std::sync::Arc;
use tokio_util::codec::{Decoder, Encoder};
use types::light_client_bootstrap::LightClientBootstrap;
use types::{light_client_bootstrap::LightClientBootstrap, BlobSidecar};
use types::{
BlobsSidecar, EthSpec, ForkContext, ForkName, Hash256, SignedBeaconBlock,
SignedBeaconBlockAltair, SignedBeaconBlockAndBlobsSidecar, SignedBeaconBlockBase,
SignedBeaconBlockCapella, SignedBeaconBlockEip4844, SignedBeaconBlockMerge,
EthSpec, ForkContext, ForkName, Hash256, SignedBeaconBlock, SignedBeaconBlockAltair,
SignedBeaconBlockBase, SignedBeaconBlockCapella, SignedBeaconBlockEip4844,
SignedBeaconBlockMerge,
};
use unsigned_varint::codec::Uvi;
@ -73,7 +73,7 @@ impl<TSpec: EthSpec> Encoder<RPCCodedResponse<TSpec>> for SSZSnappyInboundCodec<
RPCResponse::BlocksByRange(res) => res.as_ssz_bytes(),
RPCResponse::BlocksByRoot(res) => res.as_ssz_bytes(),
RPCResponse::BlobsByRange(res) => res.as_ssz_bytes(),
RPCResponse::BlockAndBlobsByRoot(res) => res.as_ssz_bytes(),
RPCResponse::SidecarByRoot(res) => res.as_ssz_bytes(),
RPCResponse::LightClientBootstrap(res) => res.as_ssz_bytes(),
RPCResponse::Pong(res) => res.data.as_ssz_bytes(),
RPCResponse::MetaData(res) =>
@ -234,7 +234,7 @@ impl<TSpec: EthSpec> Encoder<OutboundRequest<TSpec>> for SSZSnappyOutboundCodec<
OutboundRequest::BlocksByRange(req) => req.as_ssz_bytes(),
OutboundRequest::BlocksByRoot(req) => req.block_roots.as_ssz_bytes(),
OutboundRequest::BlobsByRange(req) => req.as_ssz_bytes(),
OutboundRequest::BlobsByRoot(req) => req.block_roots.as_ssz_bytes(),
OutboundRequest::BlobsByRoot(req) => req.blob_ids.as_ssz_bytes(),
OutboundRequest::Ping(req) => req.as_ssz_bytes(),
OutboundRequest::MetaData(_) => return Ok(()), // no metadata to encode
OutboundRequest::LightClientBootstrap(req) => req.as_ssz_bytes(),
@ -439,8 +439,7 @@ fn context_bytes<T: EthSpec>(
SignedBeaconBlock::Base { .. } => Some(fork_context.genesis_context_bytes()),
};
}
if let RPCResponse::BlobsByRange(_) | RPCResponse::BlockAndBlobsByRoot(_) = rpc_variant
{
if let RPCResponse::BlobsByRange(_) | RPCResponse::SidecarByRoot(_) = rpc_variant {
return fork_context.to_context_bytes(ForkName::Eip4844);
}
}
@ -497,7 +496,7 @@ fn handle_v1_request<T: EthSpec>(
BlobsByRangeRequest::from_ssz_bytes(decoded_buffer)?,
))),
Protocol::BlobsByRoot => Ok(Some(InboundRequest::BlobsByRoot(BlobsByRootRequest {
block_roots: VariableList::from_ssz_bytes(decoded_buffer)?,
blob_ids: VariableList::from_ssz_bytes(decoded_buffer)?,
}))),
Protocol::Ping => Ok(Some(InboundRequest::Ping(Ping {
data: u64::from_ssz_bytes(decoded_buffer)?,
@ -582,7 +581,7 @@ fn handle_v1_response<T: EthSpec>(
})?;
match fork_name {
ForkName::Eip4844 => Ok(Some(RPCResponse::BlobsByRange(Arc::new(
BlobsSidecar::from_ssz_bytes(decoded_buffer)?,
BlobSidecar::from_ssz_bytes(decoded_buffer)?,
)))),
_ => Err(RPCError::ErrorResponse(
RPCResponseErrorCode::InvalidRequest,
@ -598,8 +597,8 @@ fn handle_v1_response<T: EthSpec>(
)
})?;
match fork_name {
ForkName::Eip4844 => Ok(Some(RPCResponse::BlockAndBlobsByRoot(
SignedBeaconBlockAndBlobsSidecar::from_ssz_bytes(decoded_buffer)?,
ForkName::Eip4844 => Ok(Some(RPCResponse::SidecarByRoot(
BlobSidecar::from_ssz_bytes(decoded_buffer)?,
))),
_ => Err(RPCError::ErrorResponse(
RPCResponseErrorCode::InvalidRequest,
@ -738,8 +737,9 @@ mod tests {
};
use std::sync::Arc;
use types::{
BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockMerge, EmptyBlock, Epoch,
ForkContext, FullPayload, Hash256, Signature, SignedBeaconBlock, Slot,
blob_sidecar::BlobIdentifier, BeaconBlock, BeaconBlockAltair, BeaconBlockBase,
BeaconBlockMerge, EmptyBlock, Epoch, ForkContext, FullPayload, Hash256, Signature,
SignedBeaconBlock, Slot,
};
use snap::write::FrameEncoder;
@ -846,7 +846,10 @@ mod tests {
fn blbroot_request() -> BlobsByRootRequest {
BlobsByRootRequest {
block_roots: VariableList::from(vec![Hash256::zero()]),
blob_ids: VariableList::from(vec![BlobIdentifier {
block_root: Hash256::zero(),
index: 0,
}]),
}
}

37
beacon_node/lighthouse_network/src/rpc/methods.rs
View File

@ -12,9 +12,9 @@ use std::ops::Deref;
use std::sync::Arc;
use strum::IntoStaticStr;
use superstruct::superstruct;
use types::SignedBeaconBlockAndBlobsSidecar;
use types::blob_sidecar::BlobIdentifier;
use types::{
blobs_sidecar::BlobsSidecar, light_client_bootstrap::LightClientBootstrap, Epoch, EthSpec,
blob_sidecar::BlobSidecar, light_client_bootstrap::LightClientBootstrap, Epoch, EthSpec,
Hash256, SignedBeaconBlock, Slot,
};
@ -26,6 +26,8 @@ pub const MAX_REQUEST_BLOCKS: u64 = 1024;
pub type MaxErrorLen = U256;
pub const MAX_ERROR_LEN: u64 = 256;
pub const MAX_REQUEST_BLOCKS_DENEB: u64 = 128;
// TODO: this is calculated as MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK and
// MAX_BLOBS_PER_BLOCK comes from the spec.
// MAX_REQUEST_BLOCKS_DENEB = 128
@ -243,7 +245,7 @@ pub struct OldBlocksByRangeRequest {
}
/// Request a number of beacon block bodies from a peer.
#[derive(Clone, Debug, PartialEq)]
#[derive(Encode, Decode, Clone, Debug, PartialEq)]
pub struct BlocksByRootRequest {
/// The list of beacon block bodies being requested.
pub block_roots: VariableList<Hash256, MaxRequestBlocks>,
@ -253,14 +255,7 @@ pub struct BlocksByRootRequest {
#[derive(Clone, Debug, PartialEq)]
pub struct BlobsByRootRequest {
/// The list of beacon block roots being requested.
pub block_roots: VariableList<Hash256, MaxRequestBlocks>,
}
impl From<BlocksByRootRequest> for BlobsByRootRequest {
fn from(r: BlocksByRootRequest) -> Self {
let BlocksByRootRequest { block_roots } = r;
Self { block_roots }
}
pub blob_ids: VariableList<BlobIdentifier, MaxRequestBlobSidecars>,
}
/* RPC Handling and Grouping */
@ -279,13 +274,13 @@ pub enum RPCResponse<T: EthSpec> {
BlocksByRoot(Arc<SignedBeaconBlock<T>>),
/// A response to a get BLOBS_BY_RANGE request
BlobsByRange(Arc<BlobsSidecar<T>>),
BlobsByRange(Arc<BlobSidecar<T>>),
/// A response to a get LIGHTCLIENT_BOOTSTRAP request.
LightClientBootstrap(LightClientBootstrap<T>),
/// A response to a get BLOBS_BY_ROOT request.
BlockAndBlobsByRoot(SignedBeaconBlockAndBlobsSidecar<T>),
SidecarByRoot(BlobSidecar<T>),
/// A PONG response to a PING request.
Pong(Ping),
@ -378,7 +373,7 @@ impl<T: EthSpec> RPCCodedResponse<T> {
RPCResponse::BlocksByRange(_) => true,
RPCResponse::BlocksByRoot(_) => true,
RPCResponse::BlobsByRange(_) => true,
RPCResponse::BlockAndBlobsByRoot(_) => true,
RPCResponse::SidecarByRoot(_) => true,
RPCResponse::Pong(_) => false,
RPCResponse::MetaData(_) => false,
RPCResponse::LightClientBootstrap(_) => false,
@ -416,7 +411,7 @@ impl<T: EthSpec> RPCResponse<T> {
RPCResponse::BlocksByRange(_) => Protocol::BlocksByRange,
RPCResponse::BlocksByRoot(_) => Protocol::BlocksByRoot,
RPCResponse::BlobsByRange(_) => Protocol::BlobsByRange,
RPCResponse::BlockAndBlobsByRoot(_) => Protocol::BlobsByRoot,
RPCResponse::SidecarByRoot(_) => Protocol::BlobsByRoot,
RPCResponse::Pong(_) => Protocol::Ping,
RPCResponse::MetaData(_) => Protocol::MetaData,
RPCResponse::LightClientBootstrap(_) => Protocol::LightClientBootstrap,
@ -455,14 +450,10 @@ impl<T: EthSpec> std::fmt::Display for RPCResponse<T> {
write!(f, "BlocksByRoot: Block slot: {}", block.slot())
}
RPCResponse::BlobsByRange(blob) => {
write!(f, "BlobsByRange: Blob slot: {}", blob.beacon_block_slot)
write!(f, "BlobsByRange: Blob slot: {}", blob.slot)
}
RPCResponse::BlockAndBlobsByRoot(blob) => {
write!(
f,
"BlobsByRoot: Blob slot: {}",
blob.blobs_sidecar.beacon_block_slot
)
RPCResponse::SidecarByRoot(sidecar) => {
write!(f, "BlobsByRoot: Blob slot: {}", sidecar.slot)
}
RPCResponse::Pong(ping) => write!(f, "Pong: {}", ping.data),
RPCResponse::MetaData(metadata) => write!(f, "Metadata: {}", metadata.seq_number()),
@ -520,7 +511,7 @@ impl std::fmt::Display for BlobsByRootRequest {
write!(
f,
"Request: BlobsByRoot: Number of Requested Roots: {}",
self.block_roots.len()
self.blob_ids.len()
)
}
}

2
beacon_node/lighthouse_network/src/rpc/outbound.rs
View File

@ -113,7 +113,7 @@ impl<TSpec: EthSpec> OutboundRequest<TSpec> {
OutboundRequest::BlocksByRange(req) => req.count,
OutboundRequest::BlocksByRoot(req) => req.block_roots.len() as u64,
OutboundRequest::BlobsByRange(req) => req.count,
OutboundRequest::BlobsByRoot(req) => req.block_roots.len() as u64,
OutboundRequest::BlobsByRoot(req) => req.blob_ids.len() as u64,
OutboundRequest::Ping(_) => 1,
OutboundRequest::MetaData(_) => 1,
OutboundRequest::LightClientBootstrap(_) => 1,

6
beacon_node/lighthouse_network/src/rpc/protocol.rs
View File

@ -194,7 +194,7 @@ pub enum Protocol {
#[strum(serialize = "blobs_sidecars_by_range")]
BlobsByRange,
/// The `BlobsByRoot` protocol name.
#[strum(serialize = "beacon_block_and_blobs_sidecar_by_root")]
#[strum(serialize = "blob_sidecars_by_root")]
BlobsByRoot,
/// The `Ping` protocol name.
Ping,
@ -360,6 +360,7 @@ impl ProtocolId {
<BlobsByRangeRequest as Encode>::ssz_fixed_len(),
),
Protocol::BlobsByRoot => {
// TODO: This looks wrong to me
RpcLimits::new(*BLOCKS_BY_ROOT_REQUEST_MIN, *BLOCKS_BY_ROOT_REQUEST_MAX)
}
Protocol::Ping => RpcLimits::new(
@ -386,6 +387,7 @@ impl ProtocolId {
Protocol::BlocksByRoot => rpc_block_limits_by_fork(fork_context.current_fork()),
Protocol::BlobsByRange => RpcLimits::new(*BLOBS_SIDECAR_MIN, *BLOBS_SIDECAR_MAX),
Protocol::BlobsByRoot => {
// TODO: wrong too
RpcLimits::new(*SIGNED_BLOCK_AND_BLOBS_MIN, *SIGNED_BLOCK_AND_BLOBS_MAX)
}
Protocol::Ping => RpcLimits::new(
@ -524,7 +526,7 @@ impl<TSpec: EthSpec> InboundRequest<TSpec> {
InboundRequest::BlocksByRange(req) => req.count,
InboundRequest::BlocksByRoot(req) => req.block_roots.len() as u64,
InboundRequest::BlobsByRange(req) => req.count,
InboundRequest::BlobsByRoot(req) => req.block_roots.len() as u64,
InboundRequest::BlobsByRoot(req) => req.blob_ids.len() as u64,
InboundRequest::Ping(_) => 1,
InboundRequest::MetaData(_) => 1,
InboundRequest::LightClientBootstrap(_) => 1,

9
beacon_node/lighthouse_network/src/service/api_types.rs
View File

@ -2,7 +2,7 @@ use std::sync::Arc;
use libp2p::core::connection::ConnectionId;
use types::light_client_bootstrap::LightClientBootstrap;
use types::{BlobsSidecar, EthSpec, SignedBeaconBlock};
use types::{BlobSidecar, EthSpec, SignedBeaconBlock};
use crate::rpc::methods::{BlobsByRangeRequest, BlobsByRootRequest};
use crate::rpc::{
@ -12,7 +12,6 @@ use crate::rpc::{
},
OutboundRequest, SubstreamId,
};
use types::SignedBeaconBlockAndBlobsSidecar;
/// Identifier of requests sent by a peer.
pub type PeerRequestId = (ConnectionId, SubstreamId);
@ -77,13 +76,13 @@ pub enum Response<TSpec: EthSpec> {
/// A response to a get BLOCKS_BY_RANGE request. A None response signals the end of the batch.
BlocksByRange(Option<Arc<SignedBeaconBlock<TSpec>>>),
/// A response to a get BLOBS_BY_RANGE request. A None response signals the end of the batch.
BlobsByRange(Option<Arc<BlobsSidecar<TSpec>>>),
BlobsByRange(Option<Arc<BlobSidecar<TSpec>>>),
/// A response to a get BLOCKS_BY_ROOT request.
BlocksByRoot(Option<Arc<SignedBeaconBlock<TSpec>>>),
/// A response to a LightClientUpdate request.
LightClientBootstrap(LightClientBootstrap<TSpec>),
/// A response to a get BLOBS_BY_ROOT request.
BlobsByRoot(Option<SignedBeaconBlockAndBlobsSidecar<TSpec>>),
BlobsByRoot(Option<BlobSidecar<TSpec>>),
}
impl<TSpec: EthSpec> std::convert::From<Response<TSpec>> for RPCCodedResponse<TSpec> {
@ -98,7 +97,7 @@ impl<TSpec: EthSpec> std::convert::From<Response<TSpec>> for RPCCodedResponse<TS
None => RPCCodedResponse::StreamTermination(ResponseTermination::BlocksByRange),
},
Response::BlobsByRoot(r) => match r {
Some(b) => RPCCodedResponse::Success(RPCResponse::BlockAndBlobsByRoot(b)),
Some(b) => RPCCodedResponse::Success(RPCResponse::SidecarByRoot(b)),
None => RPCCodedResponse::StreamTermination(ResponseTermination::BlobsByRoot),
},
Response::BlobsByRange(r) => match r {

2
beacon_node/lighthouse_network/src/service/mod.rs
View File

@ -1328,7 +1328,7 @@ impl<AppReqId: ReqId, TSpec: EthSpec> Network<AppReqId, TSpec> {
RPCResponse::BlocksByRoot(resp) => {
self.build_response(id, peer_id, Response::BlocksByRoot(Some(resp)))
}
RPCResponse::BlockAndBlobsByRoot(resp) => {
RPCResponse::SidecarByRoot(resp) => {
self.build_response(id, peer_id, Response::BlobsByRoot(Some(resp)))
}
// Should never be reached

4
beacon_node/network/src/beacon_processor/worker/rpc_methods.rs
View File

@ -224,7 +224,7 @@ impl<T: BeaconChainTypes> Worker<T> {
async move {
let mut send_block_count = 0;
let mut send_response = true;
for root in request.block_roots.iter() {
for root in request.blob_ids.iter() {
match self
.chain
.get_block_and_blobs_checking_early_attester_cache(root)
@ -329,7 +329,7 @@ impl<T: BeaconChainTypes> Worker<T> {
self.log,
"Received BlobsByRoot Request";
"peer" => %peer_id,
"requested" => request.block_roots.len(),
"requested" => request.blob_ids.len(),
"returned" => send_block_count
);

7
consensus/types/src/blob_sidecar.rs
View File

@ -8,6 +8,13 @@ use ssz_derive::{Decode, Encode};
use test_random_derive::TestRandom;
use tree_hash_derive::TreeHash;
/// Container of the data that identifies an individual blob.
#[derive(Encode, Decode, Clone, Debug, PartialEq)]
pub struct BlobIdentifier {
pub block_root: Hash256,
pub index: u64,
}
#[derive(
Debug,
Clone,