Fix consensus, SSZ, tree hash & run merge EF tests (#2622)

* Update to v1.1.0-beta.4 (squash of #2548)

* SSZ, cached tree hash, EF tests

consensus/ssz_types/src/serde_utils/hex_fixed_vec.rs

@@ -0,0 +1,25 @@
+use crate::FixedVector;
+use eth2_serde_utils::hex::{self, PrefixedHexVisitor};
+use serde::{Deserializer, Serializer};
+use typenum::Unsigned;
+
+pub fn serialize<S, U>(bytes: &FixedVector<u8, U>, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+    U: Unsigned,
+{
+    let mut hex_string: String = "0x".to_string();
+    hex_string.push_str(&hex::encode(&bytes[..]));
+
+    serializer.serialize_str(&hex_string)
+}
+
+pub fn deserialize<'de, D, U>(deserializer: D) -> Result<FixedVector<u8, U>, D::Error>
+where
+    D: Deserializer<'de>,
+    U: Unsigned,
+{
+    let vec = deserializer.deserialize_string(PrefixedHexVisitor)?;
+    FixedVector::new(vec)
+        .map_err(|e| serde::de::Error::custom(format!("invalid fixed vector: {:?}", e)))
+}

consensus/ssz_types/src/serde_utils/hex_var_list.rs

@@ -0,0 +1,26 @@
+//! Serialize `VariableList<u8, N>` as 0x-prefixed hex string.
+use crate::VariableList;
+use eth2_serde_utils::hex::{self, PrefixedHexVisitor};
+use serde::{Deserializer, Serializer};
+use typenum::Unsigned;
+
+pub fn serialize<S, N>(bytes: &VariableList<u8, N>, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+    N: Unsigned,
+{
+    let mut hex_string: String = "0x".to_string();
+    hex_string.push_str(&hex::encode(&**bytes));
+
+    serializer.serialize_str(&hex_string)
+}
+
+pub fn deserialize<'de, D, N>(deserializer: D) -> Result<VariableList<u8, N>, D::Error>
+where
+    D: Deserializer<'de>,
+    N: Unsigned,
+{
+    let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?;
+    VariableList::new(bytes)
+        .map_err(|e| serde::de::Error::custom(format!("invalid variable list: {:?}", e)))
+}

consensus/ssz_types/src/serde_utils/mod.rs

@@ -1,2 +1,4 @@
+pub mod hex_fixed_vec;
+pub mod hex_var_list;
 pub mod quoted_u64_fixed_vec;
 pub mod quoted_u64_var_list;

consensus/state_processing/src/per_block_processing.rs

@@ -139,10 +139,10 @@ pub fn per_block_processing<T: EthSpec>(
     process_eth1_data(state, block.body().eth1_data())?;
     process_operations(state, block.body(), proposer_index, verify_signatures, spec)?;
 
-    if let BeaconBlockRef::Altair(inner) = block {
+    if let Some(sync_aggregate) = block.body().sync_aggregate() {
         process_sync_aggregate(
             state,
-            &inner.body.sync_aggregate,
+            sync_aggregate,
             proposer_index,
             verify_signatures,
             spec,
@@ -150,7 +150,11 @@ pub fn per_block_processing<T: EthSpec>(
     }
 
     if is_execution_enabled(state, block.body()) {
-        process_execution_payload(state, block.body().execution_payload().unwrap(), spec)?
+        let payload = block
+            .body()
+            .execution_payload()
+            .ok_or(BlockProcessingError::IncorrectStateType)?;
+        process_execution_payload(state, payload, spec)?;
     }
 
     Ok(())

consensus/state_processing/src/per_block_processing/errors.rs

@@ -77,6 +77,7 @@ pub enum BlockProcessingError {
         expected: u64,
         found: u64,
     },
+    ExecutionInvalid,
 }
 
 impl From<BeaconStateError> for BlockProcessingError {

consensus/state_processing/src/per_block_processing/process_operations.rs

@@ -353,15 +353,15 @@ pub fn process_deposit<T: EthSpec>(
         state.validators_mut().push(validator)?;
         state.balances_mut().push(deposit.data.amount)?;
 
-        // Altair-specific initializations.
+        // Altair or later initializations.
-        if let BeaconState::Altair(altair_state) = state {
+        if let Ok(previous_epoch_participation) = state.previous_epoch_participation_mut() {
-            altair_state
+            previous_epoch_participation.push(ParticipationFlags::default())?;
-                .previous_epoch_participation
+        }
-                .push(ParticipationFlags::default())?;
+        if let Ok(current_epoch_participation) = state.current_epoch_participation_mut() {
-            altair_state
+            current_epoch_participation.push(ParticipationFlags::default())?;
-                .current_epoch_participation
+        }
-                .push(ParticipationFlags::default())?;
+        if let Ok(inactivity_scores) = state.inactivity_scores_mut() {
-            altair_state.inactivity_scores.push(0)?;
+            inactivity_scores.push(0)?;
         }
     }
 

consensus/types/src/beacon_block.rs

@@ -101,9 +101,13 @@ impl<T: EthSpec> BeaconBlock<T> {
     /// Usually it's better to prefer `from_ssz_bytes` which will decode the correct variant based
     /// on the fork slot.
     pub fn any_from_ssz_bytes(bytes: &[u8]) -> Result<Self, ssz::DecodeError> {
+        BeaconBlockMerge::from_ssz_bytes(bytes)
+            .map(BeaconBlock::Merge)
+            .or_else(|_| {
                 BeaconBlockAltair::from_ssz_bytes(bytes)
                     .map(BeaconBlock::Altair)
                     .or_else(|_| BeaconBlockBase::from_ssz_bytes(bytes).map(BeaconBlock::Base))
+            })
     }
 
     /// Convenience accessor for the `body` as a `BeaconBlockBodyRef`.

consensus/types/src/beacon_state.rs

@@ -1684,7 +1684,8 @@ impl<T: EthSpec> CompareFields for BeaconState<T> {
         match (self, other) {
             (BeaconState::Base(x), BeaconState::Base(y)) => x.compare_fields(y),
             (BeaconState::Altair(x), BeaconState::Altair(y)) => x.compare_fields(y),
-            _ => panic!("compare_fields: mismatched state variants"),
+            (BeaconState::Merge(x), BeaconState::Merge(y)) => x.compare_fields(y),
+            _ => panic!("compare_fields: mismatched state variants",),
         }
     }
 }

consensus/types/src/beacon_state/tree_hash_cache.rs

@@ -341,16 +341,26 @@ impl<T: EthSpec> BeaconTreeHashCacheInner<T> {
         )?;
         hasher.write(state.finalized_checkpoint().tree_hash_root().as_bytes())?;
 
-        // Inactivity & light-client sync committees
+        // Inactivity & light-client sync committees (Altair and later).
-        if let BeaconState::Altair(ref state) = state {
+        if let Ok(inactivity_scores) = state.inactivity_scores() {
             hasher.write(
                 self.inactivity_scores
-                    .recalculate_tree_hash_root(&state.inactivity_scores)?
+                    .recalculate_tree_hash_root(inactivity_scores)?
                     .as_bytes(),
             )?;
+        }
 
-            hasher.write(state.current_sync_committee.tree_hash_root().as_bytes())?;
+        if let Ok(current_sync_committee) = state.current_sync_committee() {
-            hasher.write(state.next_sync_committee.tree_hash_root().as_bytes())?;
+            hasher.write(current_sync_committee.tree_hash_root().as_bytes())?;
+        }
+
+        if let Ok(next_sync_committee) = state.next_sync_committee() {
+            hasher.write(next_sync_committee.tree_hash_root().as_bytes())?;
+        }
+
+        // Execution payload (merge and later).
+        if let Ok(payload_header) = state.latest_execution_payload_header() {
+            hasher.write(payload_header.tree_hash_root().as_bytes())?;
         }
 
         let root = hasher.finish()?;

consensus/types/src/execution_payload.rs

@@ -8,8 +8,15 @@ use tree_hash_derive::TreeHash;
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash)]
 #[ssz(enum_behaviour = "union")]
 #[tree_hash(enum_behaviour = "union")]
+#[serde(tag = "selector", content = "value")]
+#[serde(bound = "T: EthSpec")]
 pub enum Transaction<T: EthSpec> {
-    OpaqueTransaction(VariableList<u8, T::MaxBytesPerOpaqueTransaction>),
+    // FIXME(merge): renaming this enum variant to 0 is a bit of a hack...
+    #[serde(rename = "0")]
+    OpaqueTransaction(
+        #[serde(with = "ssz_types::serde_utils::hex_var_list")]
+        VariableList<u8, T::MaxBytesPerOpaqueTransaction>,
+    ),
 }
 
 impl<T: EthSpec, I: SliceIndex<[u8]>> Index<I> for Transaction<T> {
@@ -33,12 +40,13 @@ impl<T: EthSpec> From<VariableList<u8, T::MaxBytesPerOpaqueTransaction>> for Tra
 #[derive(
     Default, Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom,
 )]
+#[serde(bound = "T: EthSpec")]
 pub struct ExecutionPayload<T: EthSpec> {
     pub parent_hash: Hash256,
     pub coinbase: Address,
     pub state_root: Hash256,
     pub receipt_root: Hash256,
-    #[serde(with = "serde_logs_bloom")]
+    #[serde(with = "ssz_types::serde_utils::hex_fixed_vec")]
     pub logs_bloom: FixedVector<u8, T::BytesPerLogsBloom>,
     pub random: Hash256,
     #[serde(with = "eth2_serde_utils::quoted_u64")]
@@ -51,7 +59,6 @@ pub struct ExecutionPayload<T: EthSpec> {
     pub timestamp: u64,
     pub base_fee_per_gas: Hash256,
     pub block_hash: Hash256,
-    #[serde(with = "serde_transactions")]
     #[test_random(default)]
     pub transactions: VariableList<Transaction<T>, T::MaxTransactionsPerPayload>,
 }
@@ -76,99 +83,3 @@ impl<T: EthSpec> ExecutionPayload<T> {
         }
     }
 }
-
-/// Serializes the `logs_bloom` field.
-pub mod serde_logs_bloom {
-    use super::*;
-    use eth2_serde_utils::hex::PrefixedHexVisitor;
-    use serde::{Deserializer, Serializer};
-
-    pub fn serialize<S, U>(bytes: &FixedVector<u8, U>, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-        U: Unsigned,
-    {
-        let mut hex_string: String = "0x".to_string();
-        hex_string.push_str(&hex::encode(&bytes[..]));
-
-        serializer.serialize_str(&hex_string)
-    }
-
-    pub fn deserialize<'de, D, U>(deserializer: D) -> Result<FixedVector<u8, U>, D::Error>
-    where
-        D: Deserializer<'de>,
-        U: Unsigned,
-    {
-        let vec = deserializer.deserialize_string(PrefixedHexVisitor)?;
-
-        FixedVector::new(vec)
-            .map_err(|e| serde::de::Error::custom(format!("invalid logs bloom: {:?}", e)))
-    }
-}
-
-/// Serializes the `transactions` field.
-pub mod serde_transactions {
-    use super::*;
-    use eth2_serde_utils::hex;
-    use serde::ser::SerializeSeq;
-    use serde::{de, Deserializer, Serializer};
-    use std::marker::PhantomData;
-
-    pub struct ListOfBytesListVisitor<T: EthSpec> {
-        _t: PhantomData<T>,
-    }
-    impl<'a, T> serde::de::Visitor<'a> for ListOfBytesListVisitor<T>
-    where
-        T: EthSpec,
-    {
-        type Value = VariableList<Transaction<T>, T::MaxTransactionsPerPayload>;
-
-        fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
-            write!(formatter, "a list of 0x-prefixed byte lists")
-        }
-
-        fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
-        where
-            A: serde::de::SeqAccess<'a>,
-        {
-            let mut outer = VariableList::default();
-
-            while let Some(val) = seq.next_element::<String>()? {
-                let inner_vec = hex::decode(&val).map_err(de::Error::custom)?;
-                let inner = VariableList::new(inner_vec).map_err(|e| {
-                    serde::de::Error::custom(format!("invalid transaction: {:?}", e))
-                })?;
-                outer.push(inner.into()).map_err(|e| {
-                    serde::de::Error::custom(format!("too many transactions: {:?}", e))
-                })?;
-            }
-
-            Ok(outer)
-        }
-    }
-
-    pub fn serialize<S, T>(
-        value: &VariableList<Transaction<T>, T::MaxTransactionsPerPayload>,
-        serializer: S,
-    ) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-        T: EthSpec,
-    {
-        let mut seq = serializer.serialize_seq(Some(value.len()))?;
-        for val in value {
-            seq.serialize_element(&hex::encode(&val[..]))?;
-        }
-        seq.end()
-    }
-
-    pub fn deserialize<'de, D, T>(
-        deserializer: D,
-    ) -> Result<VariableList<Transaction<T>, T::MaxTransactionsPerPayload>, D::Error>
-    where
-        D: Deserializer<'de>,
-        T: EthSpec,
-    {
-        deserializer.deserialize_any(ListOfBytesListVisitor { _t: PhantomData })
-    }
-}

consensus/types/src/execution_payload_header.rs

@@ -1,4 +1,4 @@
-use crate::{execution_payload::serde_logs_bloom, test_utils::TestRandom, *};
+use crate::{test_utils::TestRandom, *};
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
@@ -13,7 +13,7 @@ pub struct ExecutionPayloadHeader<T: EthSpec> {
     pub coinbase: Address,
     pub state_root: Hash256,
     pub receipt_root: Hash256,
-    #[serde(with = "serde_logs_bloom")]
+    #[serde(with = "ssz_types::serde_utils::hex_fixed_vec")]
     pub logs_bloom: FixedVector<u8, T::BytesPerLogsBloom>,
     pub random: Hash256,
     #[serde(with = "eth2_serde_utils::quoted_u64")]

consensus/types/src/fork_name.rs

@@ -15,7 +15,7 @@ pub enum ForkName {
 
 impl ForkName {
     pub fn list_all() -> Vec<ForkName> {
-        vec![ForkName::Base, ForkName::Altair]
+        vec![ForkName::Base, ForkName::Altair, ForkName::Merge]
     }
 
     /// Set the activation slots in the given `ChainSpec` so that the fork named by `self`

testing/ef_tests/check_all_files_accessed.py

@@ -7,6 +7,7 @@
 # The ultimate goal is to detect any accidentally-missed spec tests.
 
 import os
+import re
 import sys
 
 # First argument should the path to a file which contains a list of accessed file names.
@@ -16,25 +17,17 @@ accessed_files_filename = sys.argv[1]
 tests_dir_filename = sys.argv[2]
 
 # If any of the file names found in the consensus-spec-tests directory *starts with* one of the
-# following strings, we will assume they are to be ignored (i.e., we are purposefully *not* running
+# following regular expressions, we will assume they are to be ignored (i.e., we are purposefully
-# the spec tests).
+# *not* running the spec tests).
 excluded_paths = [
-    # Merge tests
-    "tests/minimal/merge",
-    "tests/mainnet/merge",
     # Eth1Block
     #
     # Intentionally omitted, as per https://github.com/sigp/lighthouse/issues/1835
-    "tests/minimal/phase0/ssz_static/Eth1Block/",
+    "tests/.*/.*/ssz_static/Eth1Block/",
-    "tests/mainnet/phase0/ssz_static/Eth1Block/",
-    "tests/minimal/altair/ssz_static/Eth1Block/",
-    "tests/mainnet/altair/ssz_static/Eth1Block/",
     # LightClientStore
-    "tests/minimal/altair/ssz_static/LightClientStore",
+    "tests/.*/.*/ssz_static/LightClientStore",
-    "tests/mainnet/altair/ssz_static/LightClientStore",
     # LightClientUpdate
-    "tests/minimal/altair/ssz_static/LightClientUpdate",
+    "tests/.*/.*/ssz_static/LightClientUpdate",
-    "tests/mainnet/altair/ssz_static/LightClientUpdate",
     # LightClientSnapshot
     "tests/minimal/altair/ssz_static/LightClientSnapshot",
     "tests/mainnet/altair/ssz_static/LightClientSnapshot",
@@ -44,7 +37,7 @@ excluded_paths = [
 ]
 
 def normalize_path(path):
-	return path.split("consensus-spec-tests/", )[1]
+    return path.split("consensus-spec-tests/")[1]
 
 # Determine the list of filenames which were accessed during tests.
 passed = set()
@@ -63,8 +56,8 @@ for root, dirs, files in os.walk(tests_dir_filename):
         name = normalize_path(os.path.join(root, name))
         if name not in passed:
             excluded = False
-          for excluded_path in excluded_paths:
+            for excluded_path_regex in excluded_paths:
-              if name.startswith(excluded_path):
+                if re.match(excluded_path_regex, name):
                     excluded = True
                     break
             if excluded:

testing/ef_tests/src/cases/fork.rs

@@ -49,7 +49,10 @@ impl<E: EthSpec> Case for ForkTest<E> {
     fn is_enabled_for_fork(fork_name: ForkName) -> bool {
         // Upgrades exist targeting all forks except phase0/base.
         // Fork tests also need BLS.
-        cfg!(not(feature = "fake_crypto")) && fork_name != ForkName::Base
+        // FIXME(merge): enable merge tests once available
+        cfg!(not(feature = "fake_crypto"))
+            && fork_name != ForkName::Base
+            && fork_name != ForkName::Merge
     }
 
     fn result(&self, _case_index: usize, fork_name: ForkName) -> Result<(), Error> {

testing/ef_tests/src/cases/genesis_initialization.rs

@@ -56,7 +56,9 @@ impl<E: EthSpec> LoadCase for GenesisInitialization<E> {
 impl<E: EthSpec> Case for GenesisInitialization<E> {
     fn is_enabled_for_fork(fork_name: ForkName) -> bool {
         // Altair genesis and later requires real crypto.
-        fork_name == ForkName::Base || cfg!(not(feature = "fake_crypto"))
+        // FIXME(merge): enable merge tests once available
+        fork_name == ForkName::Base
+            || cfg!(not(feature = "fake_crypto")) && fork_name != ForkName::Merge
     }
 
     fn result(&self, _case_index: usize, fork_name: ForkName) -> Result<(), Error> {

testing/ef_tests/src/cases/operations.rs

@@ -7,7 +7,7 @@ use crate::type_name::TypeName;
 use serde_derive::Deserialize;
 use state_processing::per_block_processing::{
     errors::BlockProcessingError,
-    process_block_header,
+    process_block_header, process_execution_payload,
     process_operations::{
         altair, base, process_attester_slashings, process_deposits, process_exits,
         process_proposer_slashings,
@@ -17,8 +17,8 @@ use state_processing::per_block_processing::{
 use std::fmt::Debug;
 use std::path::Path;
 use types::{
-    Attestation, AttesterSlashing, BeaconBlock, BeaconState, ChainSpec, Deposit, EthSpec, ForkName,
+    Attestation, AttesterSlashing, BeaconBlock, BeaconState, ChainSpec, Deposit, EthSpec,
-    ProposerSlashing, SignedVoluntaryExit, SyncAggregate,
+    ExecutionPayload, ForkName, ProposerSlashing, SignedVoluntaryExit, SyncAggregate,
 };
 
 #[derive(Debug, Clone, Default, Deserialize)]
@@ -27,9 +27,15 @@ struct Metadata {
     bls_setting: Option<BlsSetting>,
 }
 
+#[derive(Debug, Clone, Deserialize)]
+struct ExecutionMetadata {
+    execution_valid: bool,
+}
+
 #[derive(Debug, Clone)]
 pub struct Operations<E: EthSpec, O: Operation<E>> {
     metadata: Metadata,
+    execution_metadata: Option<ExecutionMetadata>,
     pub pre: BeaconState<E>,
     pub operation: Option<O>,
     pub post: Option<BeaconState<E>>,
@@ -54,6 +60,7 @@ pub trait Operation<E: EthSpec>: TypeName + Debug + Sync + Sized {
         &self,
         state: &mut BeaconState<E>,
         spec: &ChainSpec,
+        _: &Operations<E, Self>,
     ) -> Result<(), BlockProcessingError>;
 }
 
@@ -66,6 +73,7 @@ impl<E: EthSpec> Operation<E> for Attestation<E> {
         &self,
         state: &mut BeaconState<E>,
         spec: &ChainSpec,
+        _: &Operations<E, Self>,
     ) -> Result<(), BlockProcessingError> {
         let proposer_index = state.get_beacon_proposer_index(state.slot(), spec)? as u64;
         match state {
@@ -97,6 +105,7 @@ impl<E: EthSpec> Operation<E> for AttesterSlashing<E> {
         &self,
         state: &mut BeaconState<E>,
         spec: &ChainSpec,
+        _: &Operations<E, Self>,
     ) -> Result<(), BlockProcessingError> {
         process_attester_slashings(state, &[self.clone()], VerifySignatures::True, spec)
     }
@@ -111,6 +120,7 @@ impl<E: EthSpec> Operation<E> for Deposit {
         &self,
         state: &mut BeaconState<E>,
         spec: &ChainSpec,
+        _: &Operations<E, Self>,
     ) -> Result<(), BlockProcessingError> {
         process_deposits(state, &[self.clone()], spec)
     }
@@ -129,6 +139,7 @@ impl<E: EthSpec> Operation<E> for ProposerSlashing {
         &self,
         state: &mut BeaconState<E>,
         spec: &ChainSpec,
+        _: &Operations<E, Self>,
     ) -> Result<(), BlockProcessingError> {
         process_proposer_slashings(state, &[self.clone()], VerifySignatures::True, spec)
     }
@@ -147,6 +158,7 @@ impl<E: EthSpec> Operation<E> for SignedVoluntaryExit {
         &self,
         state: &mut BeaconState<E>,
         spec: &ChainSpec,
+        _: &Operations<E, Self>,
     ) -> Result<(), BlockProcessingError> {
         process_exits(state, &[self.clone()], VerifySignatures::True, spec)
     }
@@ -169,6 +181,7 @@ impl<E: EthSpec> Operation<E> for BeaconBlock<E> {
         &self,
         state: &mut BeaconState<E>,
         spec: &ChainSpec,
+        _: &Operations<E, Self>,
     ) -> Result<(), BlockProcessingError> {
         process_block_header(state, self.to_ref(), spec)?;
         Ok(())
@@ -196,12 +209,49 @@ impl<E: EthSpec> Operation<E> for SyncAggregate<E> {
         &self,
         state: &mut BeaconState<E>,
         spec: &ChainSpec,
+        _: &Operations<E, Self>,
     ) -> Result<(), BlockProcessingError> {
         let proposer_index = state.get_beacon_proposer_index(state.slot(), spec)? as u64;
         process_sync_aggregate(state, self, proposer_index, VerifySignatures::True, spec)
     }
 }
 
+impl<E: EthSpec> Operation<E> for ExecutionPayload<E> {
+    fn handler_name() -> String {
+        "execution_payload".into()
+    }
+
+    fn filename() -> String {
+        "execution_payload.ssz_snappy".into()
+    }
+
+    fn is_enabled_for_fork(fork_name: ForkName) -> bool {
+        fork_name != ForkName::Base && fork_name != ForkName::Altair
+    }
+
+    fn decode(path: &Path, _spec: &ChainSpec) -> Result<Self, Error> {
+        ssz_decode_file(path)
+    }
+
+    fn apply_to(
+        &self,
+        state: &mut BeaconState<E>,
+        spec: &ChainSpec,
+        extra: &Operations<E, Self>,
+    ) -> Result<(), BlockProcessingError> {
+        // FIXME(merge): we may want to plumb the validity bool into state processing
+        let valid = extra
+            .execution_metadata
+            .as_ref()
+            .map_or(false, |e| e.execution_valid);
+        if valid {
+            process_execution_payload(state, self, spec)
+        } else {
+            Err(BlockProcessingError::ExecutionInvalid)
+        }
+    }
+}
+
 impl<E: EthSpec, O: Operation<E>> LoadCase for Operations<E, O> {
     fn load_from_dir(path: &Path, fork_name: ForkName) -> Result<Self, Error> {
         let spec = &testing_spec::<E>(fork_name);
@@ -212,6 +262,14 @@ impl<E: EthSpec, O: Operation<E>> LoadCase for Operations<E, O> {
             Metadata::default()
         };
 
+        // For execution payloads only.
+        let execution_yaml_path = path.join("execution.yaml");
+        let execution_metadata = if execution_yaml_path.is_file() {
+            Some(yaml_decode_file(&execution_yaml_path)?)
+        } else {
+            None
+        };
+
         let pre = ssz_decode_state(&path.join("pre.ssz_snappy"), spec)?;
 
         // Check BLS setting here before SSZ deserialization, as most types require signatures
@@ -237,6 +295,7 @@ impl<E: EthSpec, O: Operation<E>> LoadCase for Operations<E, O> {
 
         Ok(Self {
             metadata,
+            execution_metadata,
             pre,
             operation,
             post,
@@ -270,7 +329,7 @@ impl<E: EthSpec, O: Operation<E>> Case for Operations<E, O> {
             .operation
             .as_ref()
             .ok_or(Error::SkippedBls)?
-            .apply_to(&mut state, spec)
+            .apply_to(&mut state, spec, self)
             .map(|()| state);
 
         compare_beacon_state_results_without_caches(&mut result, &mut expected)

testing/ef_tests/src/cases/rewards.rs

@@ -3,6 +3,7 @@ use crate::case_result::compare_result_detailed;
 use crate::decode::{ssz_decode_file, ssz_decode_state, yaml_decode_file};
 use compare_fields_derive::CompareFields;
 use serde_derive::Deserialize;
+use ssz::four_byte_option_impl;
 use ssz_derive::{Decode, Encode};
 use state_processing::{
     per_epoch_processing::{
@@ -26,11 +27,16 @@ pub struct Deltas {
     penalties: Vec<u64>,
 }
 
-#[derive(Debug, Clone, PartialEq, CompareFields)]
+// Define "legacy" implementations of `Option<Epoch>`, `Option<NonZeroUsize>` which use four bytes
+// for encoding the union selector.
+four_byte_option_impl!(four_byte_option_deltas, Deltas);
+
+#[derive(Debug, Clone, PartialEq, Decode, Encode, CompareFields)]
 pub struct AllDeltas {
     source_deltas: Deltas,
     target_deltas: Deltas,
     head_deltas: Deltas,
+    #[ssz(with = "four_byte_option_deltas")]
     inclusion_delay_deltas: Option<Deltas>,
     inactivity_penalty_deltas: Deltas,
 }

testing/ef_tests/src/cases/transition.rs

@@ -72,7 +72,10 @@ impl<E: EthSpec> Case for TransitionTest<E> {
     fn is_enabled_for_fork(fork_name: ForkName) -> bool {
         // Upgrades exist targeting all forks except phase0/base.
         // Transition tests also need BLS.
-        cfg!(not(feature = "fake_crypto")) && fork_name != ForkName::Base
+        // FIXME(merge): enable merge tests once available
+        cfg!(not(feature = "fake_crypto"))
+            && fork_name != ForkName::Base
+            && fork_name != ForkName::Merge
     }
 
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {

testing/ef_tests/src/handler.rs

@@ -34,7 +34,7 @@ pub trait Handler {
         let fork_name_str = match fork_name {
             ForkName::Base => "phase0",
             ForkName::Altair => "altair",
-            ForkName::Merge => "merge", // TODO: check this
+            ForkName::Merge => "merge",
         };
 
         let handler_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
@@ -145,6 +145,18 @@ impl<T, E> SszStaticHandler<T, E> {
     pub fn altair_only() -> Self {
         Self::for_forks(vec![ForkName::Altair])
     }
+
+    pub fn altair_and_later() -> Self {
+        Self::for_forks(ForkName::list_all()[1..].to_vec())
+    }
+
+    pub fn merge_only() -> Self {
+        Self::for_forks(vec![ForkName::Merge])
+    }
+
+    pub fn merge_and_later() -> Self {
+        Self::for_forks(ForkName::list_all()[2..].to_vec())
+    }
 }
 
 /// Handler for SSZ types that implement `CachedTreeHash`.
@@ -298,6 +310,11 @@ pub struct RandomHandler<E>(PhantomData<E>);
 impl<E: EthSpec + TypeName> Handler for RandomHandler<E> {
     type Case = cases::SanityBlocks<E>;
 
+    // FIXME(merge): enable merge tests once available
+    fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool {
+        fork_name != ForkName::Merge
+    }
+
     fn config_name() -> &'static str {
         E::name()
     }
@@ -481,6 +498,11 @@ pub struct GenesisValidityHandler<E>(PhantomData<E>);
 impl<E: EthSpec + TypeName> Handler for GenesisValidityHandler<E> {
     type Case = cases::GenesisValidity<E>;
 
+    // FIXME(merge): enable merge test once available
+    fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool {
+        fork_name != ForkName::Merge
+    }
+
     fn config_name() -> &'static str {
         E::name()
     }

testing/ef_tests/src/type_name.rs

@@ -41,21 +41,20 @@ type_name_generic!(Attestation);
 type_name!(AttestationData);
 type_name_generic!(AttesterSlashing);
 type_name_generic!(BeaconBlock);
-type_name_generic!(BeaconBlockBase, "BeaconBlock");
-type_name_generic!(BeaconBlockAltair, "BeaconBlock");
 type_name_generic!(BeaconBlockBody);
 type_name_generic!(BeaconBlockBodyBase, "BeaconBlockBody");
 type_name_generic!(BeaconBlockBodyAltair, "BeaconBlockBody");
+type_name_generic!(BeaconBlockBodyMerge, "BeaconBlockBody");
 type_name!(BeaconBlockHeader);
 type_name_generic!(BeaconState);
-type_name_generic!(BeaconStateBase, "BeaconState");
-type_name_generic!(BeaconStateAltair, "BeaconState");
 type_name!(Checkpoint);
 type_name_generic!(ContributionAndProof);
 type_name!(Deposit);
 type_name!(DepositData);
 type_name!(DepositMessage);
 type_name!(Eth1Data);
+type_name_generic!(ExecutionPayload);
+type_name_generic!(ExecutionPayloadHeader);
 type_name!(Fork);
 type_name!(ForkData);
 type_name_generic!(HistoricalBatch);

testing/ef_tests/tests/tests.rs

@@ -70,6 +70,12 @@ fn operations_sync_aggregate() {
     OperationsHandler::<MainnetEthSpec, SyncAggregate<_>>::default().run();
 }
 
+#[test]
+fn operations_execution_payload() {
+    OperationsHandler::<MinimalEthSpec, ExecutionPayload<_>>::default().run();
+    OperationsHandler::<MainnetEthSpec, ExecutionPayload<_>>::default().run();
+}
+
 #[test]
 fn sanity_blocks() {
     SanityBlocksHandler::<MinimalEthSpec>::default().run();
@@ -228,55 +234,74 @@ mod ssz_static {
             .run();
         SszStaticHandler::<BeaconBlockBodyAltair<MainnetEthSpec>, MainnetEthSpec>::altair_only()
             .run();
+        SszStaticHandler::<BeaconBlockBodyMerge<MinimalEthSpec>, MinimalEthSpec>::merge_only()
+            .run();
+        SszStaticHandler::<BeaconBlockBodyMerge<MainnetEthSpec>, MainnetEthSpec>::merge_only()
+            .run();
     }
 
-    // Altair-only
+    // Altair and later
     #[test]
     fn contribution_and_proof() {
-        SszStaticHandler::<ContributionAndProof<MinimalEthSpec>, MinimalEthSpec>::altair_only()
+        SszStaticHandler::<ContributionAndProof<MinimalEthSpec>, MinimalEthSpec>::altair_and_later(
+        )
         .run();
-        SszStaticHandler::<ContributionAndProof<MainnetEthSpec>, MainnetEthSpec>::altair_only()
+        SszStaticHandler::<ContributionAndProof<MainnetEthSpec>, MainnetEthSpec>::altair_and_later(
+        )
         .run();
     }
 
     #[test]
     fn signed_contribution_and_proof() {
-        SszStaticHandler::<SignedContributionAndProof<MinimalEthSpec>, MinimalEthSpec>::altair_only().run();
+        SszStaticHandler::<SignedContributionAndProof<MinimalEthSpec>, MinimalEthSpec>::altair_and_later().run();
-        SszStaticHandler::<SignedContributionAndProof<MainnetEthSpec>, MainnetEthSpec>::altair_only().run();
+        SszStaticHandler::<SignedContributionAndProof<MainnetEthSpec>, MainnetEthSpec>::altair_and_later().run();
     }
 
     #[test]
     fn sync_aggregate() {
-        SszStaticHandler::<SyncAggregate<MinimalEthSpec>, MinimalEthSpec>::altair_only().run();
+        SszStaticHandler::<SyncAggregate<MinimalEthSpec>, MinimalEthSpec>::altair_and_later().run();
-        SszStaticHandler::<SyncAggregate<MainnetEthSpec>, MainnetEthSpec>::altair_only().run();
+        SszStaticHandler::<SyncAggregate<MainnetEthSpec>, MainnetEthSpec>::altair_and_later().run();
     }
 
     #[test]
     fn sync_committee() {
-        SszStaticHandler::<SyncCommittee<MinimalEthSpec>, MinimalEthSpec>::altair_only().run();
+        SszStaticHandler::<SyncCommittee<MinimalEthSpec>, MinimalEthSpec>::altair_and_later().run();
-        SszStaticHandler::<SyncCommittee<MainnetEthSpec>, MainnetEthSpec>::altair_only().run();
+        SszStaticHandler::<SyncCommittee<MainnetEthSpec>, MainnetEthSpec>::altair_and_later().run();
     }
 
     #[test]
     fn sync_committee_contribution() {
-        SszStaticHandler::<SyncCommitteeContribution<MinimalEthSpec>, MinimalEthSpec>::altair_only(
+        SszStaticHandler::<SyncCommitteeContribution<MinimalEthSpec>, MinimalEthSpec>::altair_and_later().run();
-        )
+        SszStaticHandler::<SyncCommitteeContribution<MainnetEthSpec>, MainnetEthSpec>::altair_and_later().run();
-        .run();
-        SszStaticHandler::<SyncCommitteeContribution<MainnetEthSpec>, MainnetEthSpec>::altair_only(
-        )
-        .run();
     }
 
     #[test]
     fn sync_committee_message() {
-        SszStaticHandler::<SyncCommitteeMessage, MinimalEthSpec>::altair_only().run();
+        SszStaticHandler::<SyncCommitteeMessage, MinimalEthSpec>::altair_and_later().run();
-        SszStaticHandler::<SyncCommitteeMessage, MainnetEthSpec>::altair_only().run();
+        SszStaticHandler::<SyncCommitteeMessage, MainnetEthSpec>::altair_and_later().run();
     }
 
     #[test]
     fn sync_aggregator_selection_data() {
-        SszStaticHandler::<SyncAggregatorSelectionData, MinimalEthSpec>::altair_only().run();
+        SszStaticHandler::<SyncAggregatorSelectionData, MinimalEthSpec>::altair_and_later().run();
-        SszStaticHandler::<SyncAggregatorSelectionData, MainnetEthSpec>::altair_only().run();
+        SszStaticHandler::<SyncAggregatorSelectionData, MainnetEthSpec>::altair_and_later().run();
+    }
+
+    // Merge and later
+    #[test]
+    fn execution_payload() {
+        SszStaticHandler::<ExecutionPayload<MinimalEthSpec>, MinimalEthSpec>::merge_and_later()
+            .run();
+        SszStaticHandler::<ExecutionPayload<MainnetEthSpec>, MainnetEthSpec>::merge_and_later()
+            .run();
+    }
+
+    #[test]
+    fn execution_payload_header() {
+        SszStaticHandler::<ExecutionPayloadHeader<MinimalEthSpec>, MinimalEthSpec>::merge_and_later()
+            .run();
+        SszStaticHandler::<ExecutionPayloadHeader<MainnetEthSpec>, MainnetEthSpec>::merge_and_later()
+            .run();
     }
 }