Merge branch 'eip4844' into empty-blobs

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -62,7 +62,7 @@ use crate::{metrics, BeaconChainError, BeaconForkChoiceStore, BeaconSnapshot, Ca
 use eth2::types::{EventKind, SseBlock, SyncDuty};
 use execution_layer::{
     BlockProposalContents, BuilderParams, ChainHealth, ExecutionLayer, FailedCondition,
-    PayloadAttributes, PayloadAttributesV2, PayloadStatus,
+    PayloadAttributes, PayloadStatus,
 };
 pub use fork_choice::CountUnrealized;
 use fork_choice::{

beacon_node/beacon_chain/src/blob_cache.rs

@@ -1,7 +1,6 @@
 use lru::LruCache;
 use parking_lot::Mutex;
-use tree_hash::TreeHash;
+use types::{BlobsSidecar, EthSpec, Hash256};
-use types::{BlobsSidecar, EthSpec, ExecutionPayload, Hash256};
 
 pub const DEFAULT_BLOB_CACHE_SIZE: usize = 10;
 

beacon_node/beacon_chain/src/blob_verification.rs

@@ -2,9 +2,7 @@ use slot_clock::SlotClock;
 
 use crate::beacon_chain::{BeaconChain, BeaconChainTypes, MAXIMUM_GOSSIP_CLOCK_DISPARITY};
 use crate::{kzg_utils, BeaconChainError};
-use bls::PublicKey;
 use state_processing::per_block_processing::eip4844::eip4844::verify_kzg_commitments_against_transactions;
-use types::consts::eip4844::BLS_MODULUS;
 use types::signed_beacon_block::BlobReconstructionError;
 use types::{BeaconStateError, BlobsSidecar, Hash256, KzgCommitment, Slot, Transactions};
 

beacon_node/beacon_chain/src/block_verification.rs

@@ -89,12 +89,12 @@ use task_executor::JoinHandle;
 use tree_hash::TreeHash;
 use types::signed_beacon_block::BlobReconstructionError;
 use types::signed_block_and_blobs::BlockWrapper;
+use types::ExecPayload;
 use types::{
     BeaconBlockRef, BeaconState, BeaconStateError, BlindedPayload, ChainSpec, CloneConfig, Epoch,
     EthSpec, ExecutionBlockHash, Hash256, InconsistentFork, PublicKey, PublicKeyBytes,
     RelativeEpoch, SignedBeaconBlock, SignedBeaconBlockHeader, Slot,
 };
-use types::{BlobsSidecar, ExecPayload};
 
 pub const POS_PANDA_BANNER: &str = r#"
     ,,,         ,,,                                               ,,,         ,,,

beacon_node/execution_layer/src/engines.rs

@@ -11,7 +11,7 @@ use std::sync::Arc;
 use task_executor::TaskExecutor;
 use tokio::sync::{watch, Mutex, RwLock};
 use tokio_stream::wrappers::WatchStream;
-use types::{Address, ExecutionBlockHash, ForkName, Hash256};
+use types::{ExecutionBlockHash, ForkName};
 
 /// The number of payload IDs that will be stored for each `Engine`.
 ///

beacon_node/http_api/src/publish_blocks.rs

@@ -11,9 +11,8 @@ use tokio::sync::mpsc::UnboundedSender;
 use tree_hash::TreeHash;
 use types::signed_block_and_blobs::BlockWrapper;
 use types::{
-    AbstractExecPayload, BlindedPayload, BlobsSidecar, EthSpec, ExecPayload, ExecutionBlockHash,
+    AbstractExecPayload, BlindedPayload, EthSpec, ExecPayload, ExecutionBlockHash, FullPayload,
-    FullPayload, Hash256, SignedBeaconBlock, SignedBeaconBlockAndBlobsSidecar,
+    Hash256, SignedBeaconBlock, SignedBeaconBlockAndBlobsSidecar,
-    SignedBeaconBlockEip4844,
 };
 use warp::Rejection;
 

beacon_node/lighthouse_network/src/rpc/protocol.rs

@@ -22,7 +22,7 @@ use tokio_util::{
 };
 use types::BlobsSidecar;
 use types::{
-    BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockMerge, Blob, EmptyBlock, EthSpec,
+    BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockMerge, EmptyBlock, EthSpec,
     ForkContext, ForkName, Hash256, MainnetEthSpec, Signature, SignedBeaconBlock,
 };
 

beacon_node/lighthouse_network/src/types/pubsub.rs

@@ -3,20 +3,16 @@
 use crate::types::{GossipEncoding, GossipKind, GossipTopic};
 use crate::TopicHash;
 use libp2p::gossipsub::{DataTransform, GossipsubMessage, RawGossipsubMessage};
-use serde_derive::{Deserialize, Serialize};
 use snap::raw::{decompress_len, Decoder, Encoder};
 use ssz::{Decode, Encode};
-use ssz_derive::{Decode, Encode};
 use std::boxed::Box;
 use std::io::{Error, ErrorKind};
 use std::sync::Arc;
-use tree_hash_derive::TreeHash;
 use types::{
-    Attestation, AttesterSlashing, BlobsSidecar, EthSpec, ForkContext, ForkName,
+    Attestation, AttesterSlashing, EthSpec, ForkContext, ForkName, LightClientFinalityUpdate,
-    LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing,
+    LightClientOptimisticUpdate, ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock,
-    SignedAggregateAndProof, SignedBeaconBlock, SignedBeaconBlockAltair,
+    SignedBeaconBlockAltair, SignedBeaconBlockAndBlobsSidecar, SignedBeaconBlockBase,
-    SignedBeaconBlockAndBlobsSidecar, SignedBeaconBlockBase, SignedBeaconBlockCapella,
+    SignedBeaconBlockCapella, SignedBeaconBlockMerge, SignedBlsToExecutionChange,
-    SignedBeaconBlockEip4844, SignedBeaconBlockMerge, SignedBlsToExecutionChange,
     SignedContributionAndProof, SignedVoluntaryExit, SubnetId, SyncCommitteeMessage, SyncSubnetId,
 };
 

beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs

@@ -23,7 +23,6 @@ use slog::{crit, debug, error, warn, Logger};
 use slot_clock::SlotClock;
 use std::collections::{HashMap, HashSet};
 use std::pin::Pin;
-use std::sync::Arc;
 use std::task::Context;
 use std::time::Duration;
 use task_executor::TaskExecutor;
@@ -31,7 +30,7 @@ use tokio::sync::mpsc::{self, Receiver, Sender};
 use tokio::time::error::Error as TimeError;
 use tokio_util::time::delay_queue::{DelayQueue, Key as DelayKey};
 use types::signed_block_and_blobs::BlockWrapper;
-use types::{Attestation, EthSpec, Hash256, SignedAggregateAndProof, SignedBeaconBlock, SubnetId};
+use types::{Attestation, EthSpec, Hash256, SignedAggregateAndProof, SubnetId};
 
 const TASK_NAME: &str = "beacon_processor_reprocess_queue";
 const GOSSIP_BLOCKS: &str = "gossip_blocks";

beacon_node/network/src/beacon_processor/worker/gossip_methods.rs

@@ -15,15 +15,13 @@ use lighthouse_network::{Client, MessageAcceptance, MessageId, PeerAction, PeerI
 use slog::{crit, debug, error, info, trace, warn};
 use slot_clock::SlotClock;
 use ssz::Encode;
-use std::sync::Arc;
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 use store::hot_cold_store::HotColdDBError;
 use tokio::sync::mpsc;
 use types::signed_block_and_blobs::BlockWrapper;
 use types::{
-    Attestation, AttesterSlashing, BlobsSidecar, EthSpec, Hash256, IndexedAttestation,
+    Attestation, AttesterSlashing, EthSpec, Hash256, IndexedAttestation, LightClientFinalityUpdate,
-    LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing,
+    LightClientOptimisticUpdate, ProposerSlashing, SignedAggregateAndProof,
-    SignedAggregateAndProof, SignedBeaconBlock, SignedBeaconBlockAndBlobsSidecar,
     SignedBlsToExecutionChange, SignedContributionAndProof, SignedVoluntaryExit, Slot, SubnetId,
     SyncCommitteeMessage, SyncSubnetId,
 };

beacon_node/network/src/beacon_processor/worker/rpc_methods.rs

@@ -12,7 +12,6 @@ use lighthouse_network::rpc::*;
 use lighthouse_network::{PeerId, PeerRequestId, ReportSource, Response, SyncInfo};
 use slog::{debug, error};
 use slot_clock::SlotClock;
-use ssz_types::VariableList;
 use std::sync::Arc;
 use task_executor::TaskExecutor;
 use types::light_client_bootstrap::LightClientBootstrap;
@@ -581,7 +580,7 @@ impl<T: BeaconChainTypes> Worker<T> {
     /// Handle a `BlobsByRange` request from the peer.
     pub fn handle_blobs_by_range_request(
         self,
-        executor: TaskExecutor,
+        _executor: TaskExecutor,
         send_on_drop: SendOnDrop,
         peer_id: PeerId,
         request_id: PeerRequestId,
@@ -656,7 +655,7 @@ impl<T: BeaconChainTypes> Worker<T> {
         let block_roots = block_roots.into_iter().flatten().collect::<Vec<_>>();
 
         let mut blobs_sent = 0;
-        let mut send_response = true;
+        let send_response = true;
 
         for root in block_roots {
             match self.chain.store.get_blobs(&root) {

beacon_node/network/src/beacon_processor/worker/sync_methods.rs

@@ -17,10 +17,7 @@ use slog::{debug, error, info, warn};
 use std::sync::Arc;
 use tokio::sync::mpsc;
 use types::signed_block_and_blobs::BlockWrapper;
-use types::{
+use types::{Epoch, Hash256, SignedBeaconBlock};
-    Epoch, Hash256, SignedBeaconBlock, SignedBeaconBlockAndBlobsSidecar,
-    SignedBeaconBlockAndBlobsSidecarDecode,
-};
 
 /// Id associated to a batch processing request, either a sync batch or a parent lookup.
 #[derive(Clone, Debug, PartialEq)]

beacon_node/network/src/sync/backfill_sync/mod.rs

@@ -33,7 +33,7 @@ use types::{Epoch, EthSpec};
 /// we will negatively report peers with poor bandwidth. This can be set arbitrarily high, in which
 /// case the responder will fill the response up to the max request size, assuming they have the
 /// bandwidth to do so.
-pub const BACKFILL_EPOCHS_PER_BATCH: u64 = 2;
+pub const BACKFILL_EPOCHS_PER_BATCH: u64 = 1;
 
 /// The maximum number of batches to queue before requesting more.
 const BACKFILL_BATCH_BUFFER_SIZE: u8 = 20;

beacon_node/network/src/sync/block_lookups/mod.rs

@@ -4,15 +4,12 @@ use std::time::Duration;
 
 use beacon_chain::{BeaconChainTypes, BlockError};
 use fnv::FnvHashMap;
-use futures::StreamExt;
-use itertools::{Either, Itertools};
 use lighthouse_network::rpc::{RPCError, RPCResponseErrorCode};
 use lighthouse_network::{PeerAction, PeerId};
 use lru_cache::LRUTimeCache;
 use slog::{debug, error, trace, warn, Logger};
 use smallvec::SmallVec;
-use std::sync::Arc;
+use store::Hash256;
-use store::{Hash256, SignedBeaconBlock};
 use types::signed_block_and_blobs::BlockWrapper;
 
 use crate::beacon_processor::{ChainSegmentProcessId, WorkEvent};

beacon_node/network/src/sync/block_lookups/parent_lookup.rs

@@ -1,8 +1,7 @@
 use super::RootBlockTuple;
 use beacon_chain::BeaconChainTypes;
 use lighthouse_network::PeerId;
-use std::sync::Arc;
+use store::Hash256;
-use store::{Hash256, SignedBeaconBlock};
 use strum::IntoStaticStr;
 use types::signed_block_and_blobs::BlockWrapper;
 

beacon_node/network/src/sync/block_lookups/single_block_lookup.rs

@@ -4,8 +4,7 @@ use lighthouse_network::{rpc::BlocksByRootRequest, PeerId};
 use rand::seq::IteratorRandom;
 use ssz_types::VariableList;
 use std::collections::HashSet;
-use std::sync::Arc;
+use store::{EthSpec, Hash256};
-use store::{EthSpec, Hash256, SignedBeaconBlock};
 use strum::IntoStaticStr;
 use types::signed_block_and_blobs::BlockWrapper;
 

beacon_node/network/src/sync/block_lookups/tests.rs

@@ -10,11 +10,11 @@ use beacon_chain::builder::Witness;
 use beacon_chain::eth1_chain::CachingEth1Backend;
 use lighthouse_network::{NetworkGlobals, Request};
 use slog::{Drain, Level};
-use slot_clock::{SlotClock, SystemTimeSlotClock};
+use slot_clock::SystemTimeSlotClock;
 use store::MemoryStore;
 use tokio::sync::mpsc;
 use types::test_utils::{SeedableRng, TestRandom, XorShiftRng};
-use types::{EthSpec, MainnetEthSpec, MinimalEthSpec as E, Slot};
+use types::MinimalEthSpec as E;
 
 type T = Witness<SystemTimeSlotClock, CachingEth1Backend<E>, E, MemoryStore<E>, MemoryStore<E>>;
 

beacon_node/network/src/sync/manager.rs

@@ -898,10 +898,10 @@ impl<T: BeaconChainTypes> SyncManager<T> {
         request_id: RequestId,
         peer_id: PeerId,
         maybe_sidecar: Option<Arc<BlobsSidecar<<T>::EthSpec>>>,
-        seen_timestamp: Duration,
+        _seen_timestamp: Duration,
     ) {
         match request_id {
-            RequestId::SingleBlock { id } | RequestId::ParentLookup { id } => {
+            RequestId::SingleBlock { .. } | RequestId::ParentLookup { .. } => {
                 unreachable!("There is no such thing as a singular 'by root' glob request that is not accompanied by the block")
             }
             RequestId::BackFillSync { .. } => {

beacon_node/network/src/sync/network_context.rs

@@ -531,25 +531,21 @@ impl<T: BeaconChainTypes> SyncNetworkContext<T> {
         id
     }
 
+    /// Check whether a batch for this epoch (and only this epoch) should request just blocks or
+    /// blocks and blobs.
     pub fn batch_type(&self, epoch: types::Epoch) -> ExpectedBatchTy {
-        // Keep tests only for blocks.
+        const _: () = assert!(
+            super::backfill_sync::BACKFILL_EPOCHS_PER_BATCH == 1
+                && super::range_sync::EPOCHS_PER_BATCH == 1,
+            "To deal with alignment with 4844 boundaries, batches need to be of just one epoch"
+        );
         #[cfg(test)]
         {
+            // Keep tests only for blocks.
             return ExpectedBatchTy::OnlyBlock;
         }
         #[cfg(not(test))]
         {
-            use super::range_sync::EPOCHS_PER_BATCH;
-            assert_eq!(
-                EPOCHS_PER_BATCH, 1,
-                "If this is not one, everything will fail horribly"
-            );
-
-            // Here we need access to the beacon chain, check the fork boundary, the current epoch, the
-            // blob period to serve and check with that if the batch is a blob batch or not.
-            // NOTE: This would carelessly assume batch sizes are always 1 epoch, to avoid needing to
-            // align with the batch boundary.
-
             if let Some(data_availability_boundary) = self.chain.data_availability_boundary() {
                 if epoch >= data_availability_boundary {
                     ExpectedBatchTy::OnlyBlockBlobs

beacon_node/network/src/sync/range_sync/batch.rs

@@ -5,6 +5,7 @@ use std::collections::HashSet;
 use std::hash::{Hash, Hasher};
 use std::ops::Sub;
 use std::sync::Arc;
+use strum::Display;
 use types::signed_block_and_blobs::BlockWrapper;
 use types::{Epoch, EthSpec, SignedBeaconBlock, SignedBeaconBlockAndBlobsSidecar, Slot};
 
@@ -40,7 +41,8 @@ impl<T: EthSpec> BatchTy<T> {
 pub struct MixedBlockTyErr;
 
 /// Type of expected batch.
-#[derive(Debug, Clone)]
+#[derive(Debug, Copy, Clone, Display)]
+#[strum(serialize_all = "snake_case")]
 pub enum ExpectedBatchTy {
     OnlyBlockBlobs,
     OnlyBlock,
@@ -247,7 +249,7 @@ impl<T: EthSpec, B: BatchConfig> BatchInfo<T, B> {
                 start_slot: self.start_slot.into(),
                 count: self.end_slot.sub(self.start_slot).into(),
             },
-            self.batch_type.clone(),
+            self.batch_type,
         )
     }
 
@@ -557,6 +559,7 @@ impl<T: EthSpec, B: BatchConfig> slog::KV for BatchInfo<T, B> {
         serializer.emit_usize("processed", self.failed_processing_attempts.len())?;
         serializer.emit_u8("processed_no_penalty", self.non_faulty_processing_attempts)?;
         serializer.emit_arguments("state", &format_args!("{:?}", self.state))?;
+        serializer.emit_arguments("batch_ty", &format_args!("{}", self.batch_type));
         slog::Result::Ok(())
     }
 }

beacon_node/network/src/sync/range_sync/chain.rs

@@ -1,5 +1,4 @@
 use super::batch::{BatchInfo, BatchProcessingResult, BatchState};
-use super::BatchTy;
 use crate::beacon_processor::{ChainSegmentProcessId, WorkEvent as BeaconWorkEvent};
 use crate::sync::{
     manager::Id, network_context::SyncNetworkContext, BatchOperationOutcome, BatchProcessResult,

beacon_node/network/src/sync/range_sync/range.rs

@@ -388,12 +388,11 @@ mod tests {
     use slog::{o, Drain};
     use tokio::sync::mpsc;
 
-    use slot_clock::{SlotClock, SystemTimeSlotClock};
+    use slot_clock::SystemTimeSlotClock;
     use std::collections::HashSet;
     use std::sync::Arc;
-    use std::time::Duration;
     use store::MemoryStore;
-    use types::{Hash256, MainnetEthSpec, MinimalEthSpec as E};
+    use types::{Hash256, MinimalEthSpec as E};
 
     #[derive(Debug)]
     struct FakeStorage {

beacon_node/store/src/lib.rs

@@ -7,6 +7,7 @@
 //!
 //! Provides a simple API for storing/retrieving all types that sometimes needs type-hints. See
 //! tests for implementation examples.
+#![allow(dead_code)]
 #[macro_use]
 extern crate lazy_static;
 

consensus/state_processing/src/consensus_context.rs

@@ -1,13 +1,10 @@
 use crate::common::get_indexed_attestation;
 use crate::per_block_processing::errors::{AttestationInvalid, BlockOperationError};
 use std::collections::{hash_map::Entry, HashMap};
-use std::marker::PhantomData;
-use std::sync::Arc;
 use tree_hash::TreeHash;
 use types::{
     AbstractExecPayload, Attestation, AttestationData, BeaconState, BeaconStateError, BitList,
-    BlobsSidecar, ChainSpec, Epoch, EthSpec, ExecPayload, Hash256, IndexedAttestation,
+    ChainSpec, Epoch, EthSpec, Hash256, IndexedAttestation, SignedBeaconBlock, Slot,
-    SignedBeaconBlock, Slot,
 };
 
 #[derive(Debug)]

consensus/state_processing/src/per_block_processing.rs

@@ -42,8 +42,6 @@ mod verify_deposit;
 mod verify_exit;
 mod verify_proposer_slashing;
 
-use crate::common::decrease_balance;
-
 #[cfg(feature = "arbitrary-fuzz")]
 use arbitrary::Arbitrary;
 

consensus/types/src/blobs_sidecar.rs

@@ -1,6 +1,6 @@
 use derivative::Derivative;
 use crate::test_utils::TestRandom;
-use crate::{Blob, EthSpec, Hash256, SignedBeaconBlock, SignedRoot, Slot};
+use crate::{Blob, EthSpec, Hash256, SignedRoot, Slot};
 use kzg::KzgProof;
 use serde_derive::{Deserialize, Serialize};
 use ssz::Encode;

lcli/src/new_testnet.rs

@@ -2,24 +2,21 @@ use clap::ArgMatches;
 use clap_utils::{parse_optional, parse_required, parse_ssz_optional};
 use eth2_hashing::hash;
 use eth2_network_config::Eth2NetworkConfig;
-use genesis::interop_genesis_state;
 use ssz::Decode;
 use ssz::Encode;
 use state_processing::process_activations;
-use state_processing::upgrade::{
+use state_processing::upgrade::{upgrade_to_altair, upgrade_to_bellatrix};
-    upgrade_to_altair, upgrade_to_bellatrix, upgrade_to_capella, upgrade_to_eip4844,
-};
 use std::fs::File;
 use std::io::Read;
 use std::path::PathBuf;
 use std::str::FromStr;
 use std::time::{SystemTime, UNIX_EPOCH};
+use types::ExecutionBlockHash;
 use types::{
     test_utils::generate_deterministic_keypairs, Address, BeaconState, ChainSpec, Config, Eth1Data,
     EthSpec, ExecutionPayloadHeader, ExecutionPayloadHeaderMerge, Hash256, Keypair, PublicKey,
     Validator,
 };
-use types::{BeaconStateMerge, ExecutionBlockHash};
 
 pub fn run<T: EthSpec>(testnet_dir_path: PathBuf, matches: &ArgMatches) -> Result<(), String> {
     let deposit_contract_address: Address = parse_required(matches, "deposit-contract-address")?;