Add v0.8 genesis tests (#466)

Closes #452

eth2/state_processing/src/common/get_compact_committees_root.rs

@@ -14,15 +14,22 @@ pub fn get_compact_committees_root<T: EthSpec>(
     // FIXME: this is a spec bug, whereby the start shard for the epoch after the next epoch
     // is mistakenly used. The start shard from the cache SHOULD work.
     // Waiting on a release to fix https://github.com/ethereum/eth2.0-specs/issues/1315
-    // let start_shard = state.get_epoch_start_shard(relative_epoch)?;
-    let start_shard = state.next_epoch_start_shard(spec)?;
+    let start_shard = if relative_epoch == RelativeEpoch::Next {
+        state.next_epoch_start_shard(spec)?
+    } else {
+        state.get_epoch_start_shard(relative_epoch)?
+    };
 
     for committee_number in 0..state.get_committee_count(relative_epoch)? {
         let shard = (start_shard + committee_number) % T::ShardCount::to_u64();
         // FIXME: this is a partial workaround for the above, but it only works in the case
         // where there's a committee for every shard in every epoch. It works for the minimal
         // tests but not the mainnet ones.
-        let fake_shard = (shard + 1) % T::ShardCount::to_u64();
+        let fake_shard = if relative_epoch == RelativeEpoch::Next {
+            (shard + 1) % T::ShardCount::to_u64()
+        } else {
+            shard
+        };
 
         for &index in state
             .get_crosslink_committee_for_shard(fake_shard, relative_epoch)?

eth2/state_processing/src/genesis.rs

@@ -1,4 +1,4 @@
-use super::per_block_processing::{errors::BlockProcessingError, process_deposits};
+use super::per_block_processing::{errors::BlockProcessingError, process_deposit};
 use crate::common::get_compact_committees_root;
 use tree_hash::TreeHash;
 use types::typenum::U4294967296;
@@ -32,7 +32,7 @@ pub fn initialize_beacon_state_from_eth1<T: EthSpec>(
     for (index, deposit) in deposits.into_iter().enumerate() {
         let deposit_data_list = VariableList::<_, U4294967296>::from(leaves[..=index].to_vec());
         state.eth1_data.deposit_root = Hash256::from_slice(&deposit_data_list.tree_hash_root());
-        process_deposits(&mut state, &[deposit], spec)?;
+        process_deposit(&mut state, &deposit, spec, true)?;
     }
 
     // Process activations
@@ -48,6 +48,9 @@ pub fn initialize_beacon_state_from_eth1<T: EthSpec>(
         }
     }
 
+    // Now that we have our validators, initialize the caches (including the committees)
+    state.build_all_caches(spec)?;
+
     // Populate active_index_roots and compact_committees_roots
     let indices_list = VariableList::<usize, T::ValidatorRegistryLimit>::from(
         state.get_active_validator_indices(T::genesis_epoch()),
@@ -59,3 +62,12 @@ pub fn initialize_beacon_state_from_eth1<T: EthSpec>(
 
     Ok(state)
 }
+
+/// Determine whether a candidate genesis state is suitable for starting the chain.
+///
+/// Spec v0.8.1
+pub fn is_valid_genesis_state<T: EthSpec>(state: &BeaconState<T>, spec: &ChainSpec) -> bool {
+    state.genesis_time >= spec.min_genesis_time
+        && state.get_active_validator_indices(T::genesis_epoch()).len() as u64
+            >= spec.min_genesis_active_validator_count
+}

eth2/state_processing/src/lib.rs

@@ -7,7 +7,7 @@ pub mod per_block_processing;
 pub mod per_epoch_processing;
 pub mod per_slot_processing;
 
-pub use genesis::initialize_beacon_state_from_eth1;
+pub use genesis::{initialize_beacon_state_from_eth1, is_valid_genesis_state};
 pub use per_block_processing::{
     errors::{BlockInvalid, BlockProcessingError},
     per_block_processing, per_block_processing_without_verifying_block_signature,

eth2/state_processing/src/per_block_processing.rs

@@ -358,7 +358,7 @@ pub fn process_deposits<T: EthSpec>(
         Invalid::DepositCountInvalid
     );
 
-    // Verify deposits in parallel.
+    // Verify merkle proofs in parallel.
     deposits
         .par_iter()
         .enumerate()
@@ -368,47 +368,67 @@ pub fn process_deposits<T: EthSpec>(
         })?;
 
     // Update the state in series.
-    for (i, deposit) in deposits.iter().enumerate() {
-        state.eth1_deposit_index += 1;
+    for deposit in deposits {
+        process_deposit(state, deposit, spec, false)?;
+    }
 
-        // Ensure the state's pubkey cache is fully up-to-date, it will be used to check to see if the
-        // depositing validator already exists in the registry.
-        state.update_pubkey_cache()?;
+    Ok(())
+}
 
-        // Get an `Option<u64>` where `u64` is the validator index if this deposit public key
-        // already exists in the beacon_state.
-        let validator_index =
-            get_existing_validator_index(state, deposit).map_err(|e| e.into_with_index(i))?;
+/// Process a single deposit, optionally verifying its merkle proof.
+///
+/// Spec v0.8.1
+pub fn process_deposit<T: EthSpec>(
+    state: &mut BeaconState<T>,
+    deposit: &Deposit,
+    spec: &ChainSpec,
+    verify_merkle_proof: bool,
+) -> Result<(), Error> {
+    let deposit_index = state.eth1_deposit_index as usize;
+    if verify_merkle_proof {
+        verify_deposit_merkle_proof(state, deposit, state.eth1_deposit_index, spec)
+            .map_err(|e| e.into_with_index(deposit_index))?;
+    }
 
-        let amount = deposit.data.amount;
+    state.eth1_deposit_index += 1;
 
-        if let Some(index) = validator_index {
-            // Update the existing validator balance.
-            safe_add_assign!(state.balances[index as usize], amount);
-        } else {
-            // The signature should be checked for new validators. Return early for a bad
-            // signature.
-            if verify_deposit_signature(state, deposit, spec).is_err() {
-                return Ok(());
-            }
+    // Ensure the state's pubkey cache is fully up-to-date, it will be used to check to see if the
+    // depositing validator already exists in the registry.
+    state.update_pubkey_cache()?;
 
-            // Create a new validator.
-            let validator = Validator {
-                pubkey: deposit.data.pubkey.clone(),
-                withdrawal_credentials: deposit.data.withdrawal_credentials,
-                activation_eligibility_epoch: spec.far_future_epoch,
-                activation_epoch: spec.far_future_epoch,
-                exit_epoch: spec.far_future_epoch,
-                withdrawable_epoch: spec.far_future_epoch,
-                effective_balance: std::cmp::min(
-                    amount - amount % spec.effective_balance_increment,
-                    spec.max_effective_balance,
-                ),
-                slashed: false,
-            };
-            state.validators.push(validator)?;
-            state.balances.push(deposit.data.amount)?;
+    // Get an `Option<u64>` where `u64` is the validator index if this deposit public key
+    // already exists in the beacon_state.
+    let validator_index = get_existing_validator_index(state, deposit)
+        .map_err(|e| e.into_with_index(deposit_index))?;
+
+    let amount = deposit.data.amount;
+
+    if let Some(index) = validator_index {
+        // Update the existing validator balance.
+        safe_add_assign!(state.balances[index as usize], amount);
+    } else {
+        // The signature should be checked for new validators. Return early for a bad
+        // signature.
+        if verify_deposit_signature(state, deposit, spec).is_err() {
+            return Ok(());
         }
+
+        // Create a new validator.
+        let validator = Validator {
+            pubkey: deposit.data.pubkey.clone(),
+            withdrawal_credentials: deposit.data.withdrawal_credentials,
+            activation_eligibility_epoch: spec.far_future_epoch,
+            activation_epoch: spec.far_future_epoch,
+            exit_epoch: spec.far_future_epoch,
+            withdrawable_epoch: spec.far_future_epoch,
+            effective_balance: std::cmp::min(
+                amount - amount % spec.effective_balance_increment,
+                spec.max_effective_balance,
+            ),
+            slashed: false,
+        };
+        state.validators.push(validator)?;
+        state.balances.push(deposit.data.amount)?;
     }
 
     Ok(())

eth2/types/src/beacon_state/pubkey_cache.rs

@@ -33,11 +33,8 @@ impl PubkeyCache {
         }
     }
 
-    /// Inserts a validator index into the map.
-    ///
-    /// The added index must equal the number of validators already added to the map. This ensures
-    /// that an index is never skipped.
+    /// Looks up a validator index's by their public key.
     pub fn get(&self, pubkey: &PublicKey) -> Option<ValidatorIndex> {
-        self.map.get(pubkey).cloned()
+        self.map.get(pubkey).copied()
     }
 }

tests/ef_tests/src/cases.rs

@@ -12,6 +12,8 @@ mod epoch_processing_final_updates;
 mod epoch_processing_justification_and_finalization;
 mod epoch_processing_registry_updates;
 mod epoch_processing_slashings;
+mod genesis_initialization;
+mod genesis_validity;
 mod operations_attestation;
 mod operations_attester_slashing;
 mod operations_block_header;
@@ -36,6 +38,8 @@ pub use epoch_processing_final_updates::*;
 pub use epoch_processing_justification_and_finalization::*;
 pub use epoch_processing_registry_updates::*;
 pub use epoch_processing_slashings::*;
+pub use genesis_initialization::*;
+pub use genesis_validity::*;
 pub use operations_attestation::*;
 pub use operations_attester_slashing::*;
 pub use operations_block_header::*;

tests/ef_tests/src/cases/genesis_initialization.rs

@@ -0,0 +1,45 @@
+use super::*;
+use crate::bls_setting::BlsSetting;
+use crate::case_result::compare_beacon_state_results_without_caches;
+use serde_derive::Deserialize;
+use state_processing::initialize_beacon_state_from_eth1;
+use types::{BeaconState, Deposit, EthSpec, Hash256};
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(bound = "E: EthSpec")]
+pub struct GenesisInitialization<E: EthSpec> {
+    pub description: String,
+    pub bls_setting: Option<BlsSetting>,
+    pub eth1_block_hash: Hash256,
+    pub eth1_timestamp: u64,
+    pub deposits: Vec<Deposit>,
+    pub state: Option<BeaconState<E>>,
+}
+
+impl<E: EthSpec> YamlDecode for GenesisInitialization<E> {
+    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
+        Ok(serde_yaml::from_str(yaml).unwrap())
+    }
+}
+
+impl<E: EthSpec> Case for GenesisInitialization<E> {
+    fn description(&self) -> String {
+        self.description.clone()
+    }
+
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        self.bls_setting.unwrap_or_default().check()?;
+        let spec = &E::default_spec();
+
+        let mut result = initialize_beacon_state_from_eth1(
+            self.eth1_block_hash,
+            self.eth1_timestamp,
+            self.deposits.clone(),
+            spec,
+        );
+
+        let mut expected = self.state.clone();
+
+        compare_beacon_state_results_without_caches(&mut result, &mut expected)
+    }
+}

tests/ef_tests/src/cases/genesis_validity.rs

@@ -0,0 +1,42 @@
+use super::*;
+use crate::bls_setting::BlsSetting;
+use serde_derive::Deserialize;
+use state_processing::is_valid_genesis_state;
+use types::{BeaconState, EthSpec};
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(bound = "E: EthSpec")]
+pub struct GenesisValidity<E: EthSpec> {
+    pub description: String,
+    pub bls_setting: Option<BlsSetting>,
+    pub genesis: BeaconState<E>,
+    pub is_valid: bool,
+}
+
+impl<E: EthSpec> YamlDecode for GenesisValidity<E> {
+    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
+        Ok(serde_yaml::from_str(yaml).unwrap())
+    }
+}
+
+impl<E: EthSpec> Case for GenesisValidity<E> {
+    fn description(&self) -> String {
+        self.description.clone()
+    }
+
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        self.bls_setting.unwrap_or_default().check()?;
+        let spec = &E::default_spec();
+
+        let is_valid = is_valid_genesis_state(&self.genesis, spec);
+
+        if is_valid == self.is_valid {
+            Ok(())
+        } else {
+            Err(Error::NotEqual(format!(
+                "Got {}, expected {}",
+                is_valid, self.is_valid
+            )))
+        }
+    }
+}

tests/ef_tests/src/doc.rs

@@ -134,6 +134,14 @@ impl Doc {
                 // FIXME: skipped due to compact committees issue
                 // run_test::<EpochProcessingFinalUpdates<MainnetEthSpec>>(self)
             }
+            ("genesis", "initialization", "minimal") => {
+                run_test::<GenesisInitialization<MinimalEthSpec>>(self)
+            }
+            ("genesis", "initialization", "mainnet") => {
+                run_test::<GenesisInitialization<MainnetEthSpec>>(self)
+            }
+            ("genesis", "validity", "minimal") => run_test::<GenesisValidity<MinimalEthSpec>>(self),
+            ("genesis", "validity", "mainnet") => run_test::<GenesisValidity<MainnetEthSpec>>(self),
             (runner, handler, config) => panic!(
                 "No implementation for runner: \"{}\", handler: \"{}\", config: \"{}\"",
                 runner, handler, config

tests/ef_tests/tests/tests.rs

@@ -205,3 +205,21 @@ fn epoch_processing_final_updates() {
             Doc::assert_tests_pass(file);
         });
 }
+
+#[test]
+fn genesis_initialization() {
+    yaml_files_in_test_dir(&Path::new("genesis").join("initialization"))
+        .into_par_iter()
+        .for_each(|file| {
+            Doc::assert_tests_pass(file);
+        });
+}
+
+#[test]
+fn genesis_validity() {
+    yaml_files_in_test_dir(&Path::new("genesis").join("validity"))
+        .into_par_iter()
+        .for_each(|file| {
+            Doc::assert_tests_pass(file);
+        });
+}