Update BlindedBlobsBundle SSZ list max length and update builder tests (#4710)

* Update mev-rs and ethereum-consensus

* Fix mock buidler open bid to return fork versioned response

* Update `mev-rs` and `ethereum-consensus`

* Remove BuilderKzgCommitments and use BlockBodyKzgCommitments everywhere.

* Update testnet scripts to support builder testing and update README.md.

* Add comment on `mev-rs` version.

* Add `BN_ARGS` config to `./scripts/tests/vars.env`

* Update builder testing command in README.md

* Reject zero block hash payloads after Bellatrix.

* Update scripts/local_testnet/README.md

Co-authored-by: realbigsean <seananderson33@GMAIL.com>

---------

Co-authored-by: realbigsean <seananderson33@GMAIL.com>
This commit is contained in:
Jimmy Chen 2023-09-09 16:10:15 +10:00 committed by GitHub
parent 50bf40b4bc
commit 1db739490e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 87 additions and 78 deletions

6
Cargo.lock generated
View File

@ -528,7 +528,7 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b"
[[package]]
name = "beacon-api-client"
version = "0.1.0"
source = "git+https://github.com/ralexstokes/beacon-api-client?rev=56a290c#56a290ca9d2c67086917a0929cdf2fe35e5f917f"
source = "git+https://github.com/ralexstokes/beacon-api-client?rev=7f28993615fde52d563dd601a0511c34fe9b7c38#7f28993615fde52d563dd601a0511c34fe9b7c38"
dependencies = [
"clap 4.3.21",
"ethereum-consensus",
@ -2413,7 +2413,7 @@ dependencies = [
[[package]]
name = "ethereum-consensus"
version = "0.1.1"
source = "git+https://github.com/jimmygchen/ethereum-consensus?rev=2354493#2354493fd631b736c189868b7dc1b415a160f0f7"
source = "git+https://github.com/ralexstokes/ethereum-consensus?rev=12508c1f9b0c8f4bf4c5e9b6d441e840c1b37fd9#12508c1f9b0c8f4bf4c5e9b6d441e840c1b37fd9"
dependencies = [
"async-stream",
"blst",
@ -4906,7 +4906,7 @@ dependencies = [
[[package]]
name = "mev-rs"
version = "0.3.0"
source = "git+https://github.com/ralexstokes/mev-rs?rev=9d88a2386b58c2948fa850f0dd4b3dfe18bd4962#9d88a2386b58c2948fa850f0dd4b3dfe18bd4962"
source = "git+https://github.com/jimmygchen/mev-rs?rev=dedc77a#dedc77a796986603fb3376c5f353863d09e0dbf2"
dependencies = [
"anvil-rpc",
"async-trait",

View File

@ -94,10 +94,6 @@ resolver = "2"
[patch.crates-io]
warp = { git = "https://github.com/macladson/warp", rev="7e75acc368229a46a236a8c991bf251fe7fe50ef" }
# PR: https://github.com/ralexstokes/ethereum-consensus/pull/213
[patch."https://github.com/ralexstokes/ethereum-consensus"]
ethereum-consensus = { git = "https://github.com/jimmygchen/ethereum-consensus", rev = "2354493" }
[profile.maxperf]
inherits = "release"
lto = "fat"

View File

@ -118,7 +118,6 @@ use store::{
use task_executor::{ShutdownReason, TaskExecutor};
use tokio_stream::Stream;
use tree_hash::TreeHash;
use types::beacon_block_body::from_block_kzg_commitments;
use types::beacon_state::CloneConfig;
use types::blob_sidecar::{BlobSidecarList, FixedBlobSidecarList};
use types::sidecar::BlobItems;
@ -4994,11 +4993,8 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
metrics::start_timer(&metrics::BLOCK_PRODUCTION_BLOBS_VERIFICATION_TIMES);
let maybe_sidecar_list = match (blobs_opt, proofs_opt) {
(Some(blobs_or_blobs_roots), Some(proofs)) => {
let expected_kzg_commitments = block
.body()
.blob_kzg_commitments()
.map(from_block_kzg_commitments::<T::EthSpec>)
.map_err(|_| {
let expected_kzg_commitments =
block.body().blob_kzg_commitments().map_err(|_| {
BlockProductionError::InvalidBlockVariant(
"deneb block does not contain kzg commitments".to_string(),
)
@ -5022,7 +5018,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.ok_or(BlockProductionError::TrustedSetupNotInitialized)?;
kzg_utils::validate_blobs::<T::EthSpec>(
kzg,
&expected_kzg_commitments,
expected_kzg_commitments,
blobs,
&kzg_proofs,
)
@ -5033,7 +5029,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
Sidecar::build_sidecar(
blobs_or_blobs_roots,
&block,
&expected_kzg_commitments,
expected_kzg_commitments,
kzg_proofs,
)
.map_err(BlockProductionError::FailedToBuildBlobSidecars)?,

View File

@ -42,8 +42,9 @@ lazy_static = "1.4.0"
ethers-core = "1.0.2"
builder_client = { path = "../builder_client" }
fork_choice = { path = "../../consensus/fork_choice" }
mev-rs = { git = "https://github.com/ralexstokes/mev-rs", rev = "9d88a2386b58c2948fa850f0dd4b3dfe18bd4962" }
ethereum-consensus = { git = "https://github.com/ralexstokes/ethereum-consensus", rev = "56418ea" }
#PR: https://github.com/ralexstokes/mev-rs/pull/124
mev-rs = { git = "https://github.com/jimmygchen/mev-rs", rev = "dedc77a" }
ethereum-consensus = { git = "https://github.com/ralexstokes/ethereum-consensus", rev = "12508c1f9b0c8f4bf4c5e9b6d441e840c1b37fd9" }
ssz_rs = "0.9.0"
tokio-stream = { version = "0.1.9", features = [ "sync" ] }
strum = "0.24.0"

View File

@ -2,7 +2,7 @@ use super::*;
use serde::{Deserialize, Serialize};
use strum::EnumString;
use superstruct::superstruct;
use types::beacon_block_body::BuilderKzgCommitments;
use types::beacon_block_body::KzgCommitments;
use types::blob_sidecar::BlobsList;
use types::{
EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadDeneb,
@ -438,7 +438,7 @@ impl From<JsonPayloadAttributes> for PayloadAttributes {
#[derive(Debug, PartialEq, Serialize, Deserialize)]
#[serde(bound = "E: EthSpec", rename_all = "camelCase")]
pub struct JsonBlobsBundleV1<E: EthSpec> {
pub commitments: BuilderKzgCommitments<E>,
pub commitments: KzgCommitments<E>,
pub proofs: KzgProofs<E>,
#[serde(with = "ssz_types::serde_utils::list_of_hex_fixed_vec")]
pub blobs: BlobsList<E>,

View File

@ -39,7 +39,7 @@ use tokio::{
};
use tokio_stream::wrappers::WatchStream;
use tree_hash::TreeHash;
use types::beacon_block_body::{to_block_kzg_commitments, BlockBodyKzgCommitments};
use types::beacon_block_body::KzgCommitments;
use types::builder_bid::BuilderBid;
use types::sidecar::{BlobItems, Sidecar};
use types::KzgProofs;
@ -110,9 +110,7 @@ impl<E: EthSpec, Payload: AbstractExecPayload<E>> TryFrom<BuilderBid<E>>
.try_into()
.map_err(|_| Error::InvalidPayloadConversion)?,
block_value: builder_bid.value,
kzg_commitments: to_block_kzg_commitments::<E>(
builder_bid.blinded_blobs_bundle.commitments,
),
kzg_commitments: builder_bid.blinded_blobs_bundle.commitments,
blobs: BlobItems::try_from_blob_roots(builder_bid.blinded_blobs_bundle.blob_roots)
.map_err(Error::InvalidBlobConversion)?,
proofs: builder_bid.blinded_blobs_bundle.proofs,
@ -168,7 +166,7 @@ pub enum BlockProposalContents<T: EthSpec, Payload: AbstractExecPayload<T>> {
PayloadAndBlobs {
payload: Payload,
block_value: Uint256,
kzg_commitments: BlockBodyKzgCommitments<T>,
kzg_commitments: KzgCommitments<T>,
blobs: <Payload::Sidecar as Sidecar<T>>::BlobItems,
proofs: KzgProofs<T>,
},
@ -185,7 +183,7 @@ impl<E: EthSpec, Payload: AbstractExecPayload<E>> TryFrom<GetPayloadResponse<E>>
Some(bundle) => Ok(Self::PayloadAndBlobs {
payload: execution_payload.into(),
block_value,
kzg_commitments: to_block_kzg_commitments::<E>(bundle.commitments),
kzg_commitments: bundle.commitments,
blobs: BlobItems::try_from_blobs(bundle.blobs)
.map_err(Error::InvalidBlobConversion)?,
proofs: bundle.proofs,
@ -204,7 +202,7 @@ impl<T: EthSpec, Payload: AbstractExecPayload<T>> BlockProposalContents<T, Paylo
self,
) -> (
Payload,
Option<BlockBodyKzgCommitments<T>>,
Option<KzgCommitments<T>>,
Option<<Payload::Sidecar as Sidecar<T>>::BlobItems>,
Option<KzgProofs<T>>,
) {

View File

@ -39,7 +39,7 @@ use tree_hash::TreeHash;
use types::builder_bid::BlindedBlobsBundle;
use types::{
Address, BeaconState, ChainSpec, EthSpec, ExecPayload, ExecutionPayload,
ExecutionPayloadHeader, ForkName, Hash256, Slot, Uint256,
ExecutionPayloadHeader, ForkName, ForkVersionedResponse, Hash256, Slot, Uint256,
};
#[derive(Clone)]
@ -533,7 +533,13 @@ impl<E: EthSpec> mev_rs::BlindedBlockProvider for MockBuilder<E> {
.get_payload_by_root(&from_ssz_rs(&node)?)
.ok_or_else(|| convert_err("missing payload for tx root"))?;
let json_payload = serde_json::to_string(&payload).map_err(convert_err)?;
let fork = payload.payload_ref().fork_name();
let resp = ForkVersionedResponse {
version: Some(fork),
data: payload,
};
let json_payload = serde_json::to_string(&resp).map_err(convert_err)?;
serde_json::from_str(json_payload.as_str()).map_err(convert_err)
}
}

View File

@ -20,7 +20,7 @@ use tokio::sync::mpsc::UnboundedSender;
use tree_hash::TreeHash;
use types::{
AbstractExecPayload, BeaconBlockRef, BlindedPayload, EthSpec, ExecPayload, ExecutionBlockHash,
FullPayload, Hash256, SignedBeaconBlock, SignedBlobSidecarList,
ForkName, FullPayload, FullPayloadMerge, Hash256, SignedBeaconBlock, SignedBlobSidecarList,
};
use warp::Rejection;
@ -308,18 +308,17 @@ pub async fn reconstruct_block<T: BeaconChainTypes>(
// If the execution block hash is zero, use an empty payload.
let full_payload_contents = if payload_header.block_hash() == ExecutionBlockHash::zero() {
let payload = FullPayload::default_at_fork(
chain
.spec
.fork_name_at_epoch(block.slot().epoch(T::EthSpec::slots_per_epoch())),
)
.map_err(|e| {
warp_utils::reject::custom_server_error(format!(
"Default payload construction error: {e:?}"
))
})?
.into();
ProvenancedPayload::Local(FullPayloadContents::Payload(payload))
let fork_name = chain
.spec
.fork_name_at_epoch(block.slot().epoch(T::EthSpec::slots_per_epoch()));
if fork_name == ForkName::Merge {
let payload: FullPayload<T::EthSpec> = FullPayloadMerge::default().into();
ProvenancedPayload::Local(FullPayloadContents::Payload(payload.into()))
} else {
Err(warp_utils::reject::custom_server_error(
"Failed to construct full payload - block hash must be non-zero after Bellatrix.".to_string()
))?
}
// If we already have an execution payload with this transactions root cached, use it.
} else if let Some(cached_payload) =
el.get_payload_by_root(&payload_header.tree_hash_root())

View File

@ -17,7 +17,6 @@ use lighthouse_network::{NetworkGlobals, Request};
use slot_clock::{ManualSlotClock, SlotClock, TestingSlotClock};
use store::MemoryStore;
use tokio::sync::mpsc;
use types::beacon_block_body::to_block_kzg_commitments;
use types::{
map_fork_name, map_fork_name_with,
test_utils::{SeedableRng, TestRandom, XorShiftRng},
@ -124,8 +123,7 @@ impl TestRig {
for tx in Vec::from(transactions) {
payload.execution_payload.transactions.push(tx).unwrap();
}
message.body.blob_kzg_commitments =
to_block_kzg_commitments::<E>(bundle.commitments.clone());
message.body.blob_kzg_commitments = bundle.commitments.clone();
let eth2::types::BlobsBundle {
commitments,

View File

@ -12,7 +12,7 @@ use std::fmt::{self, Display};
use std::str::{from_utf8, FromStr};
use std::time::Duration;
use tree_hash::TreeHash;
use types::beacon_block_body::BuilderKzgCommitments;
use types::beacon_block_body::KzgCommitments;
use types::builder_bid::BlindedBlobsBundle;
pub use types::*;
@ -1793,7 +1793,7 @@ pub struct ExecutionPayloadAndBlobs<E: EthSpec> {
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, Encode)]
#[serde(bound = "E: EthSpec")]
pub struct BlobsBundle<E: EthSpec> {
pub commitments: BuilderKzgCommitments<E>,
pub commitments: KzgCommitments<E>,
pub proofs: KzgProofs<E>,
#[serde(with = "ssz_types::serde_utils::list_of_hex_fixed_vec")]
pub blobs: BlobsList<E>,

View File

@ -9,24 +9,9 @@ use superstruct::superstruct;
use test_random_derive::TestRandom;
use tree_hash_derive::TreeHash;
//TODO: Remove this type and use `BlockBodyKzgCommitments` everywhere when this PR is merged:
// https://github.com/ethereum/builder-specs/pull/87
pub type BuilderKzgCommitments<T> = VariableList<KzgCommitment, <T as EthSpec>::MaxBlobsPerBlock>;
pub type BlockBodyKzgCommitments<T> =
pub type KzgCommitments<T> =
VariableList<KzgCommitment, <T as EthSpec>::MaxBlobCommitmentsPerBlock>;
pub fn to_block_kzg_commitments<E: EthSpec>(
commitments: BuilderKzgCommitments<E>,
) -> BlockBodyKzgCommitments<E> {
commitments.to_vec().into()
}
pub fn from_block_kzg_commitments<E: EthSpec>(
commitments: &BlockBodyKzgCommitments<E>,
) -> BuilderKzgCommitments<E> {
commitments.to_vec().into()
}
/// The body of a `BeaconChain` block, containing operations.
///
/// This *superstruct* abstracts over the hard-fork.
@ -87,7 +72,7 @@ pub struct BeaconBlockBody<T: EthSpec, Payload: AbstractExecPayload<T> = FullPay
pub bls_to_execution_changes:
VariableList<SignedBlsToExecutionChange, T::MaxBlsToExecutionChanges>,
#[superstruct(only(Deneb))]
pub blob_kzg_commitments: BlockBodyKzgCommitments<T>,
pub blob_kzg_commitments: KzgCommitments<T>,
#[superstruct(only(Base, Altair))]
#[ssz(skip_serializing, skip_deserializing)]
#[tree_hash(skip_hashing)]

View File

@ -233,5 +233,5 @@ pub type BlindedBlobSidecarList<T> = SidecarList<T, BlindedBlobSidecar>;
pub type FixedBlobSidecarList<T> =
FixedVector<Option<Arc<BlobSidecar<T>>>, <T as EthSpec>::MaxBlobsPerBlock>;
pub type BlobsList<T> = VariableList<Blob<T>, <T as EthSpec>::MaxBlobsPerBlock>;
pub type BlobRootsList<T> = VariableList<Hash256, <T as EthSpec>::MaxBlobsPerBlock>;
pub type BlobsList<T> = VariableList<Blob<T>, <T as EthSpec>::MaxBlobCommitmentsPerBlock>;
pub type BlobRootsList<T> = VariableList<Hash256, <T as EthSpec>::MaxBlobCommitmentsPerBlock>;

View File

@ -1,4 +1,4 @@
use crate::beacon_block_body::BuilderKzgCommitments;
use crate::beacon_block_body::KzgCommitments;
use crate::{
BlobRootsList, ChainSpec, EthSpec, ExecutionPayloadHeaderCapella, ExecutionPayloadHeaderDeneb,
ExecutionPayloadHeaderMerge, ExecutionPayloadHeaderRef, ForkName, ForkVersionDeserialize,
@ -15,7 +15,7 @@ use tree_hash_derive::TreeHash;
#[derive(PartialEq, Debug, Default, Serialize, Deserialize, TreeHash, Clone, Encode)]
#[serde(bound = "E: EthSpec")]
pub struct BlindedBlobsBundle<E: EthSpec> {
pub commitments: BuilderKzgCommitments<E>,
pub commitments: KzgCommitments<E>,
pub proofs: KzgProofs<E>,
pub blob_roots: BlobRootsList<E>,
}

View File

@ -208,7 +208,7 @@ pub type Address = H160;
pub type ForkVersion = [u8; 4];
pub type BLSFieldElement = Uint256;
pub type Blob<T> = FixedVector<u8, <T as EthSpec>::BytesPerBlob>;
pub type KzgProofs<T> = VariableList<KzgProof, <T as EthSpec>::MaxBlobsPerBlock>;
pub type KzgProofs<T> = VariableList<KzgProof, <T as EthSpec>::MaxBlobCommitmentsPerBlock>;
pub type VersionedHash = Hash256;
pub type Hash64 = ethereum_types::H64;

View File

@ -398,8 +398,13 @@ impl<T: EthSpec> AbstractExecPayload<T> for FullPayload<T> {
ForkName::Deneb => Ok(FullPayloadDeneb::default().into()),
}
}
fn default_blobs_at_fork(_fork_name: ForkName) -> Result<BlobsList<T>, Error> {
Ok(VariableList::default())
fn default_blobs_at_fork(fork_name: ForkName) -> Result<BlobsList<T>, Error> {
match fork_name {
ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => {
Err(Error::IncorrectStateVariant)
}
ForkName::Deneb => Ok(VariableList::default()),
}
}
}
@ -916,8 +921,13 @@ impl<T: EthSpec> AbstractExecPayload<T> for BlindedPayload<T> {
ForkName::Deneb => Ok(BlindedPayloadDeneb::default().into()),
}
}
fn default_blobs_at_fork(_fork_name: ForkName) -> Result<BlobRootsList<T>, Error> {
Ok(VariableList::default())
fn default_blobs_at_fork(fork_name: ForkName) -> Result<BlobRootsList<T>, Error> {
match fork_name {
ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => {
Err(Error::IncorrectStateVariant)
}
ForkName::Deneb => Ok(VariableList::default()),
}
}
}

View File

@ -1,4 +1,4 @@
use crate::beacon_block_body::BuilderKzgCommitments;
use crate::beacon_block_body::KzgCommitments;
use crate::test_utils::TestRandom;
use crate::{
AbstractExecPayload, BeaconBlock, BlindedBlobSidecar, BlindedBlobSidecarList, BlobRootsList,
@ -33,7 +33,7 @@ pub trait Sidecar<E: EthSpec>:
fn build_sidecar<Payload: AbstractExecPayload<E>>(
blob_items: Self::BlobItems,
block: &BeaconBlock<E, Payload>,
expected_kzg_commitments: &BuilderKzgCommitments<E>,
expected_kzg_commitments: &KzgCommitments<E>,
kzg_proofs: Vec<KzgProof>,
) -> Result<SidecarList<E, Self>, String>;
}
@ -106,7 +106,7 @@ impl<E: EthSpec> Sidecar<E> for BlobSidecar<E> {
fn build_sidecar<Payload: AbstractExecPayload<E>>(
blobs: BlobsList<E>,
block: &BeaconBlock<E, Payload>,
expected_kzg_commitments: &BuilderKzgCommitments<E>,
expected_kzg_commitments: &KzgCommitments<E>,
kzg_proofs: Vec<KzgProof>,
) -> Result<SidecarList<E, Self>, String> {
let beacon_block_root = block.canonical_root();
@ -152,7 +152,7 @@ impl<E: EthSpec> Sidecar<E> for BlindedBlobSidecar {
fn build_sidecar<Payload: AbstractExecPayload<E>>(
blob_roots: BlobRootsList<E>,
block: &BeaconBlock<E, Payload>,
expected_kzg_commitments: &BuilderKzgCommitments<E>,
expected_kzg_commitments: &KzgCommitments<E>,
kzg_proofs: Vec<KzgProof>,
) -> Result<SidecarList<E, BlindedBlobSidecar>, String> {
let beacon_block_root = block.canonical_root();

View File

@ -128,3 +128,16 @@ Update the genesis time to now using:
> Note: you probably want to just rerun `./start_local_testnet.sh` to start over
> but this is another option.
### Testing builder flow
1. Add builder URL to `BN_ARGS` in `./var.env`, e.g. `--builder http://localhost:8650`. Some mock builder server options:
- [`mock-relay`](https://github.com/realbigsean/mock-relay)
- [`dummy-builder`](https://github.com/michaelsproul/dummy_builder)
2. (Optional) Add `--always-prefer-builder-payload` to `BN_ARGS`.
3. The above mock builders do not support non-mainnet presets as of now, and will require setting `SECONDS_PER_SLOT` and `SECONDS_PER_ETH1_BLOCK` to `12` in `./vars.env`.
4. Start the testnet with the following command (the `-p` flag enables the validator client `--builder-proposals` flag:
```bash
./start_local_testnet.sh -p genesis.json
```
5. Block production using builder flow will start at epoch 4.

View File

@ -63,4 +63,5 @@ exec $lighthouse_binary \
--disable-packet-filter \
--target-peers $((BN_COUNT - 1)) \
--execution-endpoint $execution_endpoint \
--execution-jwt $execution_jwt
--execution-jwt $execution_jwt \
$BN_ARGS

View File

@ -47,6 +47,6 @@ lcli \
insecure-validators \
--count $VALIDATOR_COUNT \
--base-dir $DATADIR \
--node-count $BN_COUNT
--node-count $VC_COUNT
echo Validators generated with keystore passwords at $DATADIR.

View File

@ -61,5 +61,8 @@ SECONDS_PER_ETH1_BLOCK=1
# Proposer score boost percentage
PROPOSER_SCORE_BOOST=40
# Command line arguments for beacon node client
BN_ARGS=""
# Command line arguments for validator client
VC_ARGS=""

View File

@ -58,5 +58,8 @@ SECONDS_PER_ETH1_BLOCK=1
# Proposer score boost percentage
PROPOSER_SCORE_BOOST=70
# Command line arguments for beacon node client
BN_ARGS=""
# Enable doppelganger detection
VC_ARGS=" --enable-doppelganger-protection "