Merge branch 'eip4844-devnet-v3' of https://github.com/realbigsean/lighthouse into eip4844-devnet-v3

Cargo.lock

@@ -712,7 +712,7 @@ dependencies = [
 [[package]]
 name = "c-kzg"
 version = "0.1.0"
-source = "git+https://github.com/pawanjay176/c-kzg-4844?rev=cb3745d26b728ee526dc41912e3e1bc6f17a5eeb#cb3745d26b728ee526dc41912e3e1bc6f17a5eeb"
+source = "git+https://github.com/pawanjay176/c-kzg-4844?rev=669a13800a8a0d094c5387db58e06936ef194a25#669a13800a8a0d094c5387db58e06936ef194a25"
 dependencies = [
  "hex",
  "libc",

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -53,13 +53,13 @@ use crate::validator_monitor::{
     HISTORIC_EPOCHS as VALIDATOR_MONITOR_HISTORIC_EPOCHS,
 };
 use crate::validator_pubkey_cache::ValidatorPubkeyCache;
-use crate::BeaconForkChoiceStore;
 use crate::BeaconSnapshot;
+use crate::{kzg_utils, BeaconForkChoiceStore};
 use crate::{metrics, BeaconChainError};
 use eth2::types::{EventKind, SseBlock, SyncDuty};
 use execution_layer::{
     BlockProposalContents, BuilderParams, ChainHealth, ExecutionLayer, FailedCondition,
-    PayloadAttributes, PayloadAttributesV1, PayloadAttributesV2, PayloadStatus,
+    PayloadAttributes, PayloadAttributesV2, PayloadStatus,
 };
 pub use fork_choice::CountUnrealized;
 use fork_choice::{
@@ -399,6 +399,7 @@ pub struct BeaconChain<T: BeaconChainTypes> {
     /// Provides monitoring of a set of explicitly defined validators.
     pub validator_monitor: RwLock<ValidatorMonitor<T::EthSpec>>,
     pub blob_cache: BlobCache<T::EthSpec>,
+    pub kzg: Option<Arc<kzg::Kzg>>,
 }
 
 type BeaconBlockAndState<T, Payload> = (BeaconBlock<T, Payload>, BeaconState<T>);
@@ -3937,17 +3938,38 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         *block.state_root_mut() = state_root;
 
         //FIXME(sean)
-        // - generate kzg proof
-        // - validate blobs then cache them
         // - add a new timer for processing here
         if let Some(blobs) = blobs_opt {
+            let kzg = if let Some(kzg) = &self.kzg {
+                kzg
+            } else {
+                return Err(BlockProductionError::KzgError(
+                    "Trusted setup not initialized".to_string(),
+                ));
+            };
+            let kzg_aggregated_proof =
+                kzg_utils::compute_aggregate_kzg_proof::<T::EthSpec>(&kzg, &blobs)
+                    .map_err(|e| BlockProductionError::KzgError(e))?;
             let beacon_block_root = block.canonical_root();
+            let expected_kzg_commitments = block.body().blob_kzg_commitments().map_err(|_| {
+                BlockProductionError::KzgError(
+                    "EIP4844 block does not contain kzg commitments".to_string(),
+                )
+            })?;
             let blobs_sidecar = BlobsSidecar {
                 beacon_block_slot: slot,
                 beacon_block_root,
                 blobs,
-                kzg_aggregated_proof: KzgProof::default(),
+                kzg_aggregated_proof,
             };
+            kzg_utils::validate_blobs_sidecar(
+                &kzg,
+                slot,
+                beacon_block_root,
+                expected_kzg_commitments,
+                &blobs_sidecar,
+            )
+            .map_err(BlockProductionError::KzgError)?;
             self.blob_cache.put(beacon_block_root, blobs_sidecar);
         }
 

beacon_node/beacon_chain/src/blob_verification.rs

@@ -1,11 +1,9 @@
-use derivative::Derivative;
 use slot_clock::SlotClock;
-use std::sync::Arc;
 
 use crate::beacon_chain::{BeaconChain, BeaconChainTypes, MAXIMUM_GOSSIP_CLOCK_DISPARITY};
-use crate::BeaconChainError;
+use crate::{kzg_utils, BeaconChainError};
-use bls::PublicKey;
+use state_processing::per_block_processing::eip4844::eip4844::verify_kzg_commitments_against_transactions;
-use types::{consts::eip4844::BLS_MODULUS, BeaconStateError, BlobsSidecar, Hash256, Slot};
+use types::{BeaconStateError, BlobsSidecar, Hash256, KzgCommitment, Slot, Transactions};
 
 #[derive(Debug)]
 pub enum BlobError {
@@ -36,7 +34,9 @@ pub enum BlobError {
     /// ## Peer scoring
     ///
     /// The peer has sent an invalid message.
-    BlobOutOfRange { blob_index: usize },
+    BlobOutOfRange {
+        blob_index: usize,
+    },
 
     /// The blob sidecar contains a KZGCommitment that is not a valid G1 point on
     /// the bls curve.
@@ -52,13 +52,31 @@ pub enum BlobError {
     /// The signature on the blob sidecar invalid and the peer is faulty.
     ProposalSignatureInvalid,
 
+    /// No kzg ccommitment associated with blob sidecar.
+    KzgCommitmentMissing,
+
+    /// No transactions in block
+    TransactionsMissing,
+
+    /// Blob transactions in the block do not correspond to the kzg commitments.
+    TransactionCommitmentMismatch,
+
+    TrustedSetupNotInitialized,
+
+    InvalidKzgProof,
+
+    KzgError(String),
+
     /// A blob sidecar for this proposer and slot has already been observed.
     ///
     /// ## Peer scoring
     ///
     /// The `proposer` has already proposed a sidecar at this slot. The existing sidecar may or may not
     /// be equal to the given sidecar.
-    RepeatSidecar { proposer: u64, slot: Slot },
+    RepeatSidecar {
+        proposer: u64,
+        slot: Slot,
+    },
 
     /// There was an error whilst processing the sync contribution. It is not known if it is valid or invalid.
     ///
@@ -83,6 +101,10 @@ impl From<BeaconStateError> for BlobError {
 
 pub fn validate_blob_for_gossip<T: BeaconChainTypes>(
     blob_sidecar: &BlobsSidecar<T::EthSpec>,
+    kzg_commitments: &[KzgCommitment],
+    transactions: &Transactions<T::EthSpec>,
+    block_slot: Slot,
+    block_root: Hash256,
     chain: &BeaconChain<T>,
 ) -> Result<(), BlobError> {
     let blob_slot = blob_sidecar.beacon_block_slot;
@@ -109,19 +131,46 @@ pub fn validate_blob_for_gossip<T: BeaconChainTypes>(
         });
     }
 
-    // Verify that blobs are properly formatted
+    // Verify that kzg commitments in the block are valid BLS g1 points
-    //TODO: add the check while constructing a Blob type from bytes instead of after
+    for commitment in kzg_commitments {
-    // for (i, blob) in blob_sidecar.blobs.iter().enumerate() {
+        if kzg::bytes_to_g1(&commitment.0).is_err() {
-    //     if blob.iter().any(|b| *b >= *BLS_MODULUS) {
+            return Err(BlobError::InvalidKZGCommitment);
-    //         return Err(BlobError::BlobOutOfRange { blob_index: i });
+        }
-    //     }
-    // }
-
-    // Verify that the KZG proof is a valid G1 point
-    if PublicKey::deserialize(&blob_sidecar.kzg_aggregated_proof.0).is_err() {
-        return Err(BlobError::InvalidKZGCommitment);
     }
 
-    // TODO: `validate_blobs_sidecar`
+    // Validate commitments agains transactions in the block.
+    if verify_kzg_commitments_against_transactions::<T::EthSpec>(transactions, kzg_commitments)
+        .is_err()
+    {
+        return Err(BlobError::TransactionCommitmentMismatch);
+    }
+
+    // Check that blobs are < BLS_MODULUS
+    // TODO(pawan): Add this check after there's some resolution of this
+    // issue https://github.com/ethereum/c-kzg-4844/issues/11
+    // As of now, `bytes_to_bls_field` does not fail in the c-kzg library if blob >= BLS_MODULUS
+
+    // Validate that kzg proof is a valid g1 point
+    if kzg::bytes_to_g1(&blob_sidecar.kzg_aggregated_proof.0).is_err() {
+        return Err(BlobError::InvalidKzgProof);
+    }
+
+    // Validatate that the kzg proof is valid against the commitments and blobs
+    let kzg = chain
+        .kzg
+        .as_ref()
+        .ok_or(BlobError::TrustedSetupNotInitialized)?;
+
+    if !kzg_utils::validate_blobs_sidecar(
+        kzg,
+        block_slot,
+        block_root,
+        kzg_commitments,
+        blob_sidecar,
+    )
+    .map_err(BlobError::KzgError)?
+    {
+        return Err(BlobError::InvalidKzgProof);
+    }
     Ok(())
 }

beacon_node/beacon_chain/src/block_verification.rs

@@ -90,7 +90,7 @@ use types::{
     EthSpec, ExecutionBlockHash, Hash256, InconsistentFork, PublicKey, PublicKeyBytes,
     RelativeEpoch, SignedBeaconBlock, SignedBeaconBlockHeader, Slot,
 };
-use types::{BlobsSidecar, ExecPayload, SignedBeaconBlockAndBlobsSidecar};
+use types::{BlobsSidecar, ExecPayload};
 
 pub const POS_PANDA_BANNER: &str = r#"
     ,,,         ,,,                                               ,,,         ,,,
@@ -905,7 +905,27 @@ impl<T: BeaconChainTypes> GossipVerifiedBlock<T> {
         validate_execution_payload_for_gossip(&parent_block, block.message(), chain)?;
 
         if let Some(blobs_sidecar) = blobs.as_ref() {
-            validate_blob_for_gossip(blobs_sidecar, chain).map_err(BlobValidation)?;
+            let kzg_commitments = block
+                .message()
+                .body()
+                .blob_kzg_commitments()
+                .map_err(|_| BlockError::BlobValidation(BlobError::KzgCommitmentMissing))?;
+            let transactions = block
+                .message()
+                .body()
+                .execution_payload_eip4844()
+                .map(|payload| payload.transactions())
+                .map_err(|_| BlockError::BlobValidation(BlobError::TransactionsMissing))?
+                .ok_or(BlockError::BlobValidation(BlobError::TransactionsMissing))?;
+            validate_blob_for_gossip(
+                blobs_sidecar,
+                kzg_commitments,
+                transactions,
+                block.slot(),
+                block_root,
+                chain,
+            )
+            .map_err(BlobValidation)?;
             //FIXME(sean) validate blobs sidecar
         }
 

beacon_node/beacon_chain/src/builder.rs

@@ -21,12 +21,14 @@ use eth1::Config as Eth1Config;
 use execution_layer::ExecutionLayer;
 use fork_choice::{ForkChoice, ResetPayloadStatuses};
 use futures::channel::mpsc::Sender;
+use kzg::Kzg;
 use operation_pool::{OperationPool, PersistedOperationPool};
 use parking_lot::RwLock;
 use slasher::Slasher;
 use slog::{crit, error, info, Logger};
 use slot_clock::{SlotClock, TestingSlotClock};
 use std::marker::PhantomData;
+use std::path::PathBuf;
 use std::sync::Arc;
 use std::time::Duration;
 use store::{Error as StoreError, HotColdDB, ItemStore, KeyValueStoreOp};
@@ -94,6 +96,7 @@ pub struct BeaconChainBuilder<T: BeaconChainTypes> {
     // Pending I/O batch that is constructed during building and should be executed atomically
     // alongside `PersistedBeaconChain` storage when `BeaconChainBuilder::build` is called.
     pending_io_batch: Vec<KeyValueStoreOp>,
+    trusted_setup_path: Option<PathBuf>,
     task_executor: Option<TaskExecutor>,
 }
 
@@ -133,6 +136,7 @@ where
             slasher: None,
             validator_monitor: None,
             pending_io_batch: vec![],
+            trusted_setup_path: None,
             task_executor: None,
         }
     }
@@ -572,6 +576,11 @@ where
         self
     }
 
+    pub fn trusted_setup(mut self, trusted_setup_file_path: PathBuf) -> Self {
+        self.trusted_setup_path = Some(trusted_setup_file_path);
+        self
+    }
+
     /// Consumes `self`, returning a `BeaconChain` if all required parameters have been supplied.
     ///
     /// An error will be returned at runtime if all required parameters have not been configured.
@@ -613,6 +622,14 @@ where
             slot_clock.now().ok_or("Unable to read slot")?
         };
 
+        let kzg = if let Some(trusted_setup_file) = self.trusted_setup_path {
+            let kzg = Kzg::new_from_file(trusted_setup_file)
+                .map_err(|e| format!("Failed to load trusted setup: {:?}", e))?;
+            Some(Arc::new(kzg))
+        } else {
+            None
+        };
+
         let initial_head_block_root = fork_choice
             .get_head(current_slot, &self.spec)
             .map_err(|e| format!("Unable to get fork choice head: {:?}", e))?;
@@ -814,6 +831,7 @@ where
             slasher: self.slasher.clone(),
             validator_monitor: RwLock::new(validator_monitor),
             blob_cache: BlobCache::default(),
+            kzg,
         };
 
         let head = beacon_chain.head_snapshot();

beacon_node/beacon_chain/src/errors.rs

@@ -267,6 +267,7 @@ pub enum BlockProductionError {
     TokioJoin(tokio::task::JoinError),
     BeaconChain(BeaconChainError),
     InvalidPayloadFork,
+    KzgError(String),
 }
 
 easy_from_to!(BlockProcessingError, BlockProductionError);

beacon_node/beacon_chain/src/kzg_utils.rs

@@ -17,7 +17,7 @@ pub fn validate_blobs_sidecar<T: EthSpec>(
     slot: Slot,
     beacon_block_root: Hash256,
     expected_kzg_commitments: &[KzgCommitment],
-    blobs_sidecar: BlobsSidecar<T>,
+    blobs_sidecar: &BlobsSidecar<T>,
 ) -> Result<bool, String> {
     if slot != blobs_sidecar.beacon_block_slot
         || beacon_block_root != blobs_sidecar.beacon_block_root

beacon_node/client/src/builder.rs

@@ -188,6 +188,12 @@ where
             builder
         };
 
+        let builder = if let Some(trusted_setup_file) = config.trusted_setup_file {
+            builder.trusted_setup(trusted_setup_file)
+        } else {
+            builder
+        };
+
         let chain_exists = builder.store_contains_beacon_chain().unwrap_or(false);
 
         // If the client is expect to resume but there's no beacon chain in the database,

beacon_node/client/src/config.rs

@@ -68,6 +68,7 @@ pub struct Config {
     pub chain: beacon_chain::ChainConfig,
     pub eth1: eth1::Config,
     pub execution_layer: Option<execution_layer::Config>,
+    pub trusted_setup_file: Option<PathBuf>,
     pub http_api: http_api::Config,
     pub http_metrics: http_metrics::Config,
     pub monitoring_api: Option<monitoring_api::Config>,
@@ -90,6 +91,7 @@ impl Default for Config {
             sync_eth1_chain: false,
             eth1: <_>::default(),
             execution_layer: None,
+            trusted_setup_file: None,
             graffiti: Graffiti::default(),
             http_api: <_>::default(),
             http_metrics: <_>::default(),

beacon_node/src/cli.rs

@@ -511,6 +511,17 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .default_value("1")
                 .takes_value(true)
         )
+        /* 4844 settings */
+        .arg(
+            Arg::with_name("trusted-setup-file")
+                .long("trusted-setup-file")
+                .value_name("FILE")
+                .help("File containing the trusted setup parameters. \
+                      NOTE: This is only for the devnet, the trusted setup params \
+                      must be embedded into the ethspec once parameter loading \
+                      is supported in the ckzg library")
+                .takes_value(true)
+        )
         /*
          * Database purging and compaction.
          */

beacon_node/src/config.rs

@@ -360,6 +360,11 @@ pub fn get_config<E: EthSpec>(
         client_config.execution_layer = Some(el_config);
     }
 
+    // 4844 params
+    if let Some(trusted_setup_file) = cli_args.value_of("trusted-setup-file") {
+        client_config.trusted_setup_file = Some(PathBuf::from(trusted_setup_file));
+    }
+
     if let Some(freezer_dir) = cli_args.value_of("freezer-dir") {
         client_config.freezer_db_path = Some(PathBuf::from(freezer_dir));
     }

consensus/state_processing/src/per_block_processing/eip4844/eip4844.rs

@@ -3,7 +3,6 @@ use eth2_hashing::hash_fixed;
 use itertools::{EitherOrBoth, Itertools};
 use safe_arith::SafeArith;
 use ssz::Decode;
-use ssz_types::VariableList;
 use types::consts::eip4844::{BLOB_TX_TYPE, VERSIONED_HASH_VERSION_KZG};
 use types::{
     AbstractExecPayload, BeaconBlockBodyRef, EthSpec, ExecPayload, KzgCommitment, Transaction,
@@ -30,7 +29,7 @@ pub fn process_blob_kzg_commitments<T: EthSpec, Payload: AbstractExecPayload<T>>
 
 pub fn verify_kzg_commitments_against_transactions<T: EthSpec>(
     transactions: &Transactions<T>,
-    kzg_commitments: &VariableList<KzgCommitment, T::MaxBlobsPerBlock>,
+    kzg_commitments: &[KzgCommitment],
 ) -> Result<bool, BlockProcessingError> {
     let nested_iter = transactions
         .into_iter()

crypto/kzg/Cargo.toml

@@ -18,4 +18,5 @@ eth2_serde_utils = "0.1.1"
 hex = "0.4.2"
 eth2_hashing = "0.3.0"
 ethereum-types = "0.12.1"
-c-kzg = {git = "https://github.com/pawanjay176/c-kzg-4844", rev = "cb3745d26b728ee526dc41912e3e1bc6f17a5eeb" }
+c-kzg = {git = "https://github.com/pawanjay176/c-kzg-4844", rev = "669a13800a8a0d094c5387db58e06936ef194a25" }
+

crypto/kzg/src/lib.rs

@@ -2,6 +2,7 @@ mod kzg_commitment;
 mod kzg_proof;
 
 pub use crate::{kzg_commitment::KzgCommitment, kzg_proof::KzgProof};
+pub use c_kzg::bytes_to_g1;
 use c_kzg::{Error as CKzgError, KZGSettings, BYTES_PER_FIELD_ELEMENT, FIELD_ELEMENTS_PER_BLOB};
 use std::path::PathBuf;