Apply eth2.0 config spec to ChainSpec (#605)

* Allow slot clock to work on genesis

* Loose over-strict requirements for slot clock tests

* fix slots_per_eth1_voting_period

* add utils for domain serializing and deserializing

* add serde_yaml to cargo

* add tmpdir to cargo

* cargo fmt src/utils

* add YamlConfig, allowing to serialize and deserialize yaml config files

* update to v0.9.1

* Comment out local tests

* update milliseconds_per_slot for minimal

* add safe_slots_to_update_justified

* Update to v0.9.1 and add deny_unknown_fields

* Specify that max_epochs_per_crosslink is for phase1

* Remove test stubs

* Use get to avoid panicking in serde_utils

* Use ? operator instead of explicit match return

eth2/types/Cargo.toml

@@ -31,6 +31,8 @@ tree_hash = "0.1.0"
 tree_hash_derive = "0.2"
 rand_xorshift = "0.2.0"
 cached_tree_hash = { path = "../utils/cached_tree_hash" }
+serde_yaml = "0.8.11"
+tempfile = "3.1.0"
 
 [dev-dependencies]
 env_logger = "0.7.1"

eth2/types/src/chain_spec.rs

@@ -1,7 +1,7 @@
 use crate::*;
 use int_to_bytes::int_to_bytes4;
 use serde_derive::{Deserialize, Serialize};
-use utils::{u8_from_hex_str, u8_to_hex_str};
+use utils::{u32_from_hex_str, u32_to_hex_str, u8_from_hex_str, u8_to_hex_str};
 
 /// Each of the BLS signature domains.
 ///
@@ -257,6 +257,7 @@ impl ChainSpec {
             network_id: 2, // lighthouse testnet network id
             boot_nodes,
             eth1_follow_distance: 16,
+            milliseconds_per_slot: 6_000,
             ..ChainSpec::mainnet()
         }
     }
@@ -317,3 +318,341 @@ mod tests {
         test_domain(Domain::VoluntaryExit, spec.domain_voluntary_exit, &spec);
     }
 }
+
+// Yaml Config is declared here in order to access domain fields of ChainSpec which are private fields.
+#[derive(Serialize, Deserialize, Debug, PartialEq)]
+#[serde(rename_all = "UPPERCASE")]
+#[serde(default)]
+#[serde(deny_unknown_fields)]
+/// Union of a ChainSpec struct and an EthSpec struct that holds constants used for the configs folder of the Ethereum 2 spec (https://github.com/ethereum/eth2.0-specs/tree/dev/configs)
+/// Spec v0.9.1
+pub struct YamlConfig {
+    // ChainSpec
+    far_future_epoch: u64,
+    base_rewards_per_epoch: u64,
+    deposit_contract_tree_depth: u64,
+    seconds_per_day: u64,
+    max_committees_per_slot: usize,
+    target_committee_size: usize,
+    min_per_epoch_churn_limit: u64,
+    churn_limit_quotient: u64,
+    shuffle_round_count: u8,
+    min_genesis_active_validator_count: u64,
+    min_genesis_time: u64,
+    min_deposit_amount: u64,
+    max_effective_balance: u64,
+    ejection_balance: u64,
+    effective_balance_increment: u64,
+    genesis_slot: u64,
+    #[serde(deserialize_with = "u8_from_hex_str", serialize_with = "u8_to_hex_str")]
+    bls_withdrawal_prefix: u8,
+    seconds_per_slot: u64,
+    min_attestation_inclusion_delay: u64,
+    min_seed_lookahead: u64,
+    min_validator_withdrawability_delay: u64,
+    persistent_committee_period: u64,
+    min_epochs_to_inactivity_penalty: u64,
+    base_reward_factor: u64,
+    whistleblower_reward_quotient: u64,
+    proposer_reward_quotient: u64,
+    inactivity_penalty_quotient: u64,
+    min_slashing_penalty_quotient: u64,
+    safe_slots_to_update_justified: u64,
+
+    #[serde(skip_serializing)]
+    genesis_fork: Fork,
+
+    #[serde(
+        deserialize_with = "u32_from_hex_str",
+        serialize_with = "u32_to_hex_str"
+    )]
+    domain_beacon_proposer: u32,
+    #[serde(
+        deserialize_with = "u32_from_hex_str",
+        serialize_with = "u32_to_hex_str"
+    )]
+    domain_beacon_attester: u32,
+    #[serde(
+        deserialize_with = "u32_from_hex_str",
+        serialize_with = "u32_to_hex_str"
+    )]
+    domain_randao: u32,
+    #[serde(
+        deserialize_with = "u32_from_hex_str",
+        serialize_with = "u32_to_hex_str"
+    )]
+    domain_deposit: u32,
+    #[serde(
+        deserialize_with = "u32_from_hex_str",
+        serialize_with = "u32_to_hex_str"
+    )]
+    domain_voluntary_exit: u32,
+    #[serde(
+        deserialize_with = "u32_from_hex_str",
+        serialize_with = "u32_to_hex_str"
+    )]
+    // EthSpec
+    justification_bits_length: u32,
+    max_validators_per_committee: u32,
+    genesis_epoch: Epoch,
+    slots_per_epoch: u64,
+    slots_per_eth1_voting_period: usize,
+    slots_per_historical_root: usize,
+    epochs_per_historical_vector: usize,
+    epochs_per_slashings_vector: usize,
+    historical_roots_limit: u64,
+    validator_registry_limit: u64,
+    max_proposer_slashings: u32,
+    max_attester_slashings: u32,
+    max_attestations: u32,
+    max_deposits: u32,
+    max_voluntary_exits: u32,
+
+    // Unused
+    #[serde(skip_serializing)]
+    early_derived_secret_penalty_max_future_epochs: u32,
+    #[serde(skip_serializing)]
+    max_seed_lookahead: u32,
+    #[serde(skip_serializing)]
+    deposit_contract_address: String,
+
+    // Phase 1
+    #[serde(skip_serializing)]
+    epochs_per_custody_period: u32,
+    #[serde(skip_serializing)]
+    custody_period_to_randao_padding: u32,
+    #[serde(skip_serializing)]
+    shard_slots_per_beacon_slot: u32,
+    #[serde(skip_serializing)]
+    epochs_per_shard_period: u32,
+    #[serde(skip_serializing)]
+    phase_1_fork_epoch: u32,
+    #[serde(skip_serializing)]
+    phase_1_fork_slot: u32,
+    #[serde(skip_serializing)]
+    domain_custody_bit_challenge: u32,
+    #[serde(skip_serializing)]
+    domain_shard_proposer: u32,
+    #[serde(skip_serializing)]
+    domain_shard_attester: u32,
+    #[serde(skip_serializing)]
+    max_epochs_per_crosslink: u64,
+}
+
+impl Default for YamlConfig {
+    fn default() -> Self {
+        let chain_spec = MainnetEthSpec::default_spec();
+        YamlConfig::from_spec::<MainnetEthSpec>(&chain_spec)
+    }
+}
+
+/// Spec v0.8.1
+impl YamlConfig {
+    pub fn from_spec<T: EthSpec>(spec: &ChainSpec) -> Self {
+        Self {
+            // ChainSpec
+            far_future_epoch: spec.far_future_epoch.into(),
+            base_rewards_per_epoch: spec.base_rewards_per_epoch,
+            deposit_contract_tree_depth: spec.deposit_contract_tree_depth,
+            seconds_per_day: spec.seconds_per_day,
+            max_committees_per_slot: spec.max_committees_per_slot,
+            target_committee_size: spec.target_committee_size,
+            min_per_epoch_churn_limit: spec.min_per_epoch_churn_limit,
+            churn_limit_quotient: spec.churn_limit_quotient,
+            shuffle_round_count: spec.shuffle_round_count,
+            min_genesis_active_validator_count: spec.min_genesis_active_validator_count,
+            min_genesis_time: spec.min_genesis_time,
+            min_deposit_amount: spec.min_deposit_amount,
+            max_effective_balance: spec.max_effective_balance,
+            ejection_balance: spec.ejection_balance,
+            effective_balance_increment: spec.effective_balance_increment,
+            genesis_slot: spec.genesis_slot.into(),
+            bls_withdrawal_prefix: spec.bls_withdrawal_prefix_byte,
+            seconds_per_slot: spec.milliseconds_per_slot / 1000,
+            min_attestation_inclusion_delay: spec.min_attestation_inclusion_delay,
+            min_seed_lookahead: spec.min_seed_lookahead.into(),
+            min_validator_withdrawability_delay: spec.min_validator_withdrawability_delay.into(),
+            persistent_committee_period: spec.persistent_committee_period,
+            min_epochs_to_inactivity_penalty: spec.min_epochs_to_inactivity_penalty,
+            base_reward_factor: spec.base_reward_factor,
+            whistleblower_reward_quotient: spec.whistleblower_reward_quotient,
+            proposer_reward_quotient: spec.proposer_reward_quotient,
+            inactivity_penalty_quotient: spec.inactivity_penalty_quotient,
+            min_slashing_penalty_quotient: spec.min_slashing_penalty_quotient,
+            genesis_fork: spec.genesis_fork.clone(),
+            safe_slots_to_update_justified: spec.safe_slots_to_update_justified,
+            domain_beacon_proposer: spec.domain_beacon_proposer,
+            domain_beacon_attester: spec.domain_beacon_attester,
+            domain_randao: spec.domain_randao,
+            domain_deposit: spec.domain_deposit,
+            domain_voluntary_exit: spec.domain_voluntary_exit,
+
+            // EthSpec
+            justification_bits_length: T::JustificationBitsLength::to_u32(),
+            max_validators_per_committee: T::MaxValidatorsPerCommittee::to_u32(),
+            genesis_epoch: T::genesis_epoch(),
+            slots_per_epoch: T::slots_per_epoch(),
+            slots_per_eth1_voting_period: T::slots_per_eth1_voting_period(),
+            slots_per_historical_root: T::slots_per_historical_root(),
+            epochs_per_historical_vector: T::epochs_per_historical_vector(),
+            epochs_per_slashings_vector: T::EpochsPerSlashingsVector::to_usize(),
+            historical_roots_limit: T::HistoricalRootsLimit::to_u64(),
+            validator_registry_limit: T::ValidatorRegistryLimit::to_u64(),
+            max_proposer_slashings: T::MaxProposerSlashings::to_u32(),
+            max_attester_slashings: T::MaxAttesterSlashings::to_u32(),
+            max_attestations: T::MaxAttestations::to_u32(),
+            max_deposits: T::MaxDeposits::to_u32(),
+            max_voluntary_exits: T::MaxVoluntaryExits::to_u32(),
+
+            // Unused
+            early_derived_secret_penalty_max_future_epochs: 0,
+            max_seed_lookahead: 0,
+            deposit_contract_address: String::new(),
+
+            // Phase 1
+            epochs_per_custody_period: 0,
+            custody_period_to_randao_padding: 0,
+            shard_slots_per_beacon_slot: 0,
+            epochs_per_shard_period: 0,
+            phase_1_fork_epoch: 0,
+            phase_1_fork_slot: 0,
+            domain_custody_bit_challenge: 0,
+            domain_shard_proposer: 0,
+            domain_shard_attester: 0,
+            max_epochs_per_crosslink: 0,
+        }
+    }
+
+    pub fn apply_to_chain_spec<T: EthSpec>(&self, chain_spec: &ChainSpec) -> Option<ChainSpec> {
+        // Checking for EthSpec constants
+        if self.justification_bits_length != T::JustificationBitsLength::to_u32()
+            || self.max_validators_per_committee != T::MaxValidatorsPerCommittee::to_u32()
+            || self.genesis_epoch != T::genesis_epoch()
+            || self.slots_per_epoch != T::slots_per_epoch()
+            || self.slots_per_eth1_voting_period != T::slots_per_eth1_voting_period()
+            || self.slots_per_historical_root != T::slots_per_historical_root()
+            || self.epochs_per_historical_vector != T::epochs_per_historical_vector()
+            || self.epochs_per_slashings_vector != T::EpochsPerSlashingsVector::to_usize()
+            || self.historical_roots_limit != T::HistoricalRootsLimit::to_u64()
+            || self.validator_registry_limit != T::ValidatorRegistryLimit::to_u64()
+            || self.max_proposer_slashings != T::MaxProposerSlashings::to_u32()
+            || self.max_attester_slashings != T::MaxAttesterSlashings::to_u32()
+            || self.max_attestations != T::MaxAttestations::to_u32()
+            || self.max_deposits != T::MaxDeposits::to_u32()
+            || self.max_voluntary_exits != T::MaxVoluntaryExits::to_u32()
+        {
+            return None;
+        }
+
+        // Create a ChainSpec from the yaml config
+        Some(ChainSpec {
+            far_future_epoch: Epoch::from(self.far_future_epoch),
+            base_rewards_per_epoch: self.base_rewards_per_epoch,
+            deposit_contract_tree_depth: self.deposit_contract_tree_depth,
+            seconds_per_day: self.seconds_per_day,
+            target_committee_size: self.target_committee_size,
+            min_per_epoch_churn_limit: self.min_per_epoch_churn_limit,
+            churn_limit_quotient: self.churn_limit_quotient,
+            shuffle_round_count: self.shuffle_round_count,
+            min_genesis_active_validator_count: self.min_genesis_active_validator_count,
+            min_genesis_time: self.min_genesis_time,
+            min_deposit_amount: self.min_deposit_amount,
+            max_effective_balance: self.max_effective_balance,
+            ejection_balance: self.ejection_balance,
+            effective_balance_increment: self.effective_balance_increment,
+            genesis_slot: Slot::from(self.genesis_slot),
+            bls_withdrawal_prefix_byte: self.bls_withdrawal_prefix,
+            milliseconds_per_slot: self.seconds_per_slot * 1000,
+            min_attestation_inclusion_delay: self.min_attestation_inclusion_delay,
+            min_seed_lookahead: Epoch::from(self.min_seed_lookahead),
+            min_validator_withdrawability_delay: Epoch::from(
+                self.min_validator_withdrawability_delay,
+            ),
+            persistent_committee_period: self.persistent_committee_period,
+            min_epochs_to_inactivity_penalty: self.min_epochs_to_inactivity_penalty,
+            base_reward_factor: self.base_reward_factor,
+            whistleblower_reward_quotient: self.whistleblower_reward_quotient,
+            proposer_reward_quotient: self.proposer_reward_quotient,
+            inactivity_penalty_quotient: self.inactivity_penalty_quotient,
+            min_slashing_penalty_quotient: self.min_slashing_penalty_quotient,
+            domain_beacon_proposer: self.domain_beacon_proposer,
+            domain_randao: self.domain_randao,
+            domain_deposit: self.domain_deposit,
+            domain_voluntary_exit: self.domain_voluntary_exit,
+            boot_nodes: chain_spec.boot_nodes.clone(),
+            genesis_fork: chain_spec.genesis_fork.clone(),
+            ..*chain_spec
+        })
+    }
+}
+
+#[cfg(test)]
+mod yaml_tests {
+    use super::*;
+    use std::fs::OpenOptions;
+    use tempfile::NamedTempFile;
+
+    #[test]
+    fn minimal_round_trip() {
+        // create temp file
+        let tmp_file = NamedTempFile::new().expect("failed to create temp file");
+        let writer = OpenOptions::new()
+            .read(false)
+            .write(true)
+            .open(tmp_file.as_ref())
+            .expect("error opening file");
+        let minimal_spec = ChainSpec::minimal();
+
+        let yamlconfig = YamlConfig::from_spec::<MinimalEthSpec>(&minimal_spec);
+        // write fresh minimal config to file
+        serde_yaml::to_writer(writer, &yamlconfig).expect("failed to write or serialize");
+
+        let reader = OpenOptions::new()
+            .read(true)
+            .write(false)
+            .open(tmp_file.as_ref())
+            .expect("error while opening the file");
+        // deserialize minimal config from file
+        let from: YamlConfig = serde_yaml::from_reader(reader).expect("error while deserializing");
+        assert_eq!(from, yamlconfig);
+    }
+
+    #[test]
+    fn mainnet_round_trip() {
+        let tmp_file = NamedTempFile::new().expect("failed to create temp file");
+        let writer = OpenOptions::new()
+            .read(false)
+            .write(true)
+            .open(tmp_file.as_ref())
+            .expect("error opening file");
+        let mainnet_spec = ChainSpec::mainnet();
+        let yamlconfig = YamlConfig::from_spec::<MainnetEthSpec>(&mainnet_spec);
+        serde_yaml::to_writer(writer, &yamlconfig).expect("failed to write or serialize");
+
+        let reader = OpenOptions::new()
+            .read(true)
+            .write(false)
+            .open(tmp_file.as_ref())
+            .expect("error while opening the file");
+        let from: YamlConfig = serde_yaml::from_reader(reader).expect("error while deserializing");
+        assert_eq!(from, yamlconfig);
+    }
+
+    #[test]
+    fn apply_to_spec() {
+        let mut spec = ChainSpec::minimal();
+        let yamlconfig = YamlConfig::from_spec::<MinimalEthSpec>(&spec);
+
+        // modifying the original spec
+        spec.deposit_contract_tree_depth += 1;
+        // Applying a yaml config with incorrect EthSpec should fail
+        let res = yamlconfig.apply_to_chain_spec::<MainnetEthSpec>(&spec);
+        assert_eq!(res, None);
+
+        // Applying a yaml config with correct EthSpec should NOT fail
+        let new_spec = yamlconfig
+            .apply_to_chain_spec::<MinimalEthSpec>(&spec)
+            .expect("should have applied spec");
+        assert_eq!(new_spec, ChainSpec::minimal());
+    }
+}

eth2/types/src/utils/serde_utils.rs

@@ -11,7 +11,11 @@ where
 {
     let s: String = Deserialize::deserialize(deserializer)?;
 
-    u8::from_str_radix(&s.as_str()[2..], 16).map_err(D::Error::custom)
+    let start = match s.as_str().get(2..) {
+        Some(start) => start,
+        None => return Err(D::Error::custom("string length too small")),
+    };
+    u8::from_str_radix(&start, 16).map_err(D::Error::custom)
 }
 
 #[allow(clippy::trivially_copy_pass_by_ref)] // Serde requires the `byte` to be a ref.
@@ -25,13 +29,45 @@ where
     serializer.serialize_str(&hex)
 }
 
+pub fn u32_from_hex_str<'de, D>(deserializer: D) -> Result<u32, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let s: String = Deserialize::deserialize(deserializer)?;
+    let start = s
+        .as_str()
+        .get(2..)
+        .ok_or_else(|| D::Error::custom("string length too small"))?;
+
+    u32::from_str_radix(&start, 16)
+        .map_err(D::Error::custom)
+        .map(u32::from_be)
+}
+
+#[allow(clippy::trivially_copy_pass_by_ref)] // Serde requires the `num` to be a ref.
+pub fn u32_to_hex_str<S>(num: &u32, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let mut hex: String = "0x".to_string();
+    let bytes = num.to_le_bytes();
+    hex.push_str(&hex::encode(&bytes));
+
+    serializer.serialize_str(&hex)
+}
+
 pub fn fork_from_hex_str<'de, D>(deserializer: D) -> Result<[u8; FORK_BYTES_LEN], D::Error>
 where
     D: Deserializer<'de>,
 {
     let s: String = Deserialize::deserialize(deserializer)?;
     let mut array = [0 as u8; FORK_BYTES_LEN];
-    let decoded: Vec<u8> = hex::decode(&s.as_str()[2..]).map_err(D::Error::custom)?;
+
+    let start = s
+        .as_str()
+        .get(2..)
+        .ok_or_else(|| D::Error::custom("string length too small"))?;
+    let decoded: Vec<u8> = hex::decode(&start).map_err(D::Error::custom)?;
 
     if decoded.len() != FORK_BYTES_LEN {
         return Err(D::Error::custom("Fork length too long"));
@@ -76,7 +112,12 @@ where
 {
     let s: String = Deserialize::deserialize(deserializer)?;
     let mut array = [0 as u8; GRAFFITI_BYTES_LEN];
-    let decoded: Vec<u8> = hex::decode(&s.as_str()[2..]).map_err(D::Error::custom)?;
+
+    let start = s
+        .as_str()
+        .get(2..)
+        .ok_or_else(|| D::Error::custom("string length too small"))?;
+    let decoded: Vec<u8> = hex::decode(&start).map_err(D::Error::custom)?;
 
     if decoded.len() > GRAFFITI_BYTES_LEN {
         return Err(D::Error::custom("Fork length too long"));