From 7bdddd682c75ffa32418dc63df3adb45a841ac30 Mon Sep 17 00:00:00 2001 From: pscott <30843220+pscott@users.noreply.github.com> Date: Thu, 28 Nov 2019 03:20:16 +0100 Subject: [PATCH] Apply eth2.0 config spec to ChainSpec (#605) * Allow slot clock to work on genesis * Loose over-strict requirements for slot clock tests * fix slots_per_eth1_voting_period * add utils for domain serializing and deserializing * add serde_yaml to cargo * add tmpdir to cargo * cargo fmt src/utils * add YamlConfig, allowing to serialize and deserialize yaml config files * update to v0.9.1 * Comment out local tests * update milliseconds_per_slot for minimal * add safe_slots_to_update_justified * Update to v0.9.1 and add deny_unknown_fields * Specify that max_epochs_per_crosslink is for phase1 * Remove test stubs * Use get to avoid panicking in serde_utils * Use ? operator instead of explicit match return --- eth2/types/Cargo.toml | 2 + eth2/types/src/chain_spec.rs | 341 +++++++++++++++++++++++++++- eth2/types/src/utils/serde_utils.rs | 47 +++- 3 files changed, 386 insertions(+), 4 deletions(-) diff --git a/eth2/types/Cargo.toml b/eth2/types/Cargo.toml index ca78da340..1edd5e54c 100644 --- a/eth2/types/Cargo.toml +++ b/eth2/types/Cargo.toml @@ -31,6 +31,8 @@ tree_hash = "0.1.0" tree_hash_derive = "0.2" rand_xorshift = "0.2.0" cached_tree_hash = { path = "../utils/cached_tree_hash" } +serde_yaml = "0.8.11" +tempfile = "3.1.0" [dev-dependencies] env_logger = "0.7.1" diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs index c6152269a..fff61186f 100644 --- a/eth2/types/src/chain_spec.rs +++ b/eth2/types/src/chain_spec.rs @@ -1,7 +1,7 @@ use crate::*; use int_to_bytes::int_to_bytes4; use serde_derive::{Deserialize, Serialize}; -use utils::{u8_from_hex_str, u8_to_hex_str}; +use utils::{u32_from_hex_str, u32_to_hex_str, u8_from_hex_str, u8_to_hex_str}; /// Each of the BLS signature domains. /// @@ -257,6 +257,7 @@ impl ChainSpec { network_id: 2, // lighthouse testnet network id boot_nodes, eth1_follow_distance: 16, + milliseconds_per_slot: 6_000, ..ChainSpec::mainnet() } } @@ -317,3 +318,341 @@ mod tests { test_domain(Domain::VoluntaryExit, spec.domain_voluntary_exit, &spec); } } + +// Yaml Config is declared here in order to access domain fields of ChainSpec which are private fields. +#[derive(Serialize, Deserialize, Debug, PartialEq)] +#[serde(rename_all = "UPPERCASE")] +#[serde(default)] +#[serde(deny_unknown_fields)] +/// Union of a ChainSpec struct and an EthSpec struct that holds constants used for the configs folder of the Ethereum 2 spec (https://github.com/ethereum/eth2.0-specs/tree/dev/configs) +/// Spec v0.9.1 +pub struct YamlConfig { + // ChainSpec + far_future_epoch: u64, + base_rewards_per_epoch: u64, + deposit_contract_tree_depth: u64, + seconds_per_day: u64, + max_committees_per_slot: usize, + target_committee_size: usize, + min_per_epoch_churn_limit: u64, + churn_limit_quotient: u64, + shuffle_round_count: u8, + min_genesis_active_validator_count: u64, + min_genesis_time: u64, + min_deposit_amount: u64, + max_effective_balance: u64, + ejection_balance: u64, + effective_balance_increment: u64, + genesis_slot: u64, + #[serde(deserialize_with = "u8_from_hex_str", serialize_with = "u8_to_hex_str")] + bls_withdrawal_prefix: u8, + seconds_per_slot: u64, + min_attestation_inclusion_delay: u64, + min_seed_lookahead: u64, + min_validator_withdrawability_delay: u64, + persistent_committee_period: u64, + min_epochs_to_inactivity_penalty: u64, + base_reward_factor: u64, + whistleblower_reward_quotient: u64, + proposer_reward_quotient: u64, + inactivity_penalty_quotient: u64, + min_slashing_penalty_quotient: u64, + safe_slots_to_update_justified: u64, + + #[serde(skip_serializing)] + genesis_fork: Fork, + + #[serde( + deserialize_with = "u32_from_hex_str", + serialize_with = "u32_to_hex_str" + )] + domain_beacon_proposer: u32, + #[serde( + deserialize_with = "u32_from_hex_str", + serialize_with = "u32_to_hex_str" + )] + domain_beacon_attester: u32, + #[serde( + deserialize_with = "u32_from_hex_str", + serialize_with = "u32_to_hex_str" + )] + domain_randao: u32, + #[serde( + deserialize_with = "u32_from_hex_str", + serialize_with = "u32_to_hex_str" + )] + domain_deposit: u32, + #[serde( + deserialize_with = "u32_from_hex_str", + serialize_with = "u32_to_hex_str" + )] + domain_voluntary_exit: u32, + #[serde( + deserialize_with = "u32_from_hex_str", + serialize_with = "u32_to_hex_str" + )] + // EthSpec + justification_bits_length: u32, + max_validators_per_committee: u32, + genesis_epoch: Epoch, + slots_per_epoch: u64, + slots_per_eth1_voting_period: usize, + slots_per_historical_root: usize, + epochs_per_historical_vector: usize, + epochs_per_slashings_vector: usize, + historical_roots_limit: u64, + validator_registry_limit: u64, + max_proposer_slashings: u32, + max_attester_slashings: u32, + max_attestations: u32, + max_deposits: u32, + max_voluntary_exits: u32, + + // Unused + #[serde(skip_serializing)] + early_derived_secret_penalty_max_future_epochs: u32, + #[serde(skip_serializing)] + max_seed_lookahead: u32, + #[serde(skip_serializing)] + deposit_contract_address: String, + + // Phase 1 + #[serde(skip_serializing)] + epochs_per_custody_period: u32, + #[serde(skip_serializing)] + custody_period_to_randao_padding: u32, + #[serde(skip_serializing)] + shard_slots_per_beacon_slot: u32, + #[serde(skip_serializing)] + epochs_per_shard_period: u32, + #[serde(skip_serializing)] + phase_1_fork_epoch: u32, + #[serde(skip_serializing)] + phase_1_fork_slot: u32, + #[serde(skip_serializing)] + domain_custody_bit_challenge: u32, + #[serde(skip_serializing)] + domain_shard_proposer: u32, + #[serde(skip_serializing)] + domain_shard_attester: u32, + #[serde(skip_serializing)] + max_epochs_per_crosslink: u64, +} + +impl Default for YamlConfig { + fn default() -> Self { + let chain_spec = MainnetEthSpec::default_spec(); + YamlConfig::from_spec::(&chain_spec) + } +} + +/// Spec v0.8.1 +impl YamlConfig { + pub fn from_spec(spec: &ChainSpec) -> Self { + Self { + // ChainSpec + far_future_epoch: spec.far_future_epoch.into(), + base_rewards_per_epoch: spec.base_rewards_per_epoch, + deposit_contract_tree_depth: spec.deposit_contract_tree_depth, + seconds_per_day: spec.seconds_per_day, + max_committees_per_slot: spec.max_committees_per_slot, + target_committee_size: spec.target_committee_size, + min_per_epoch_churn_limit: spec.min_per_epoch_churn_limit, + churn_limit_quotient: spec.churn_limit_quotient, + shuffle_round_count: spec.shuffle_round_count, + min_genesis_active_validator_count: spec.min_genesis_active_validator_count, + min_genesis_time: spec.min_genesis_time, + min_deposit_amount: spec.min_deposit_amount, + max_effective_balance: spec.max_effective_balance, + ejection_balance: spec.ejection_balance, + effective_balance_increment: spec.effective_balance_increment, + genesis_slot: spec.genesis_slot.into(), + bls_withdrawal_prefix: spec.bls_withdrawal_prefix_byte, + seconds_per_slot: spec.milliseconds_per_slot / 1000, + min_attestation_inclusion_delay: spec.min_attestation_inclusion_delay, + min_seed_lookahead: spec.min_seed_lookahead.into(), + min_validator_withdrawability_delay: spec.min_validator_withdrawability_delay.into(), + persistent_committee_period: spec.persistent_committee_period, + min_epochs_to_inactivity_penalty: spec.min_epochs_to_inactivity_penalty, + base_reward_factor: spec.base_reward_factor, + whistleblower_reward_quotient: spec.whistleblower_reward_quotient, + proposer_reward_quotient: spec.proposer_reward_quotient, + inactivity_penalty_quotient: spec.inactivity_penalty_quotient, + min_slashing_penalty_quotient: spec.min_slashing_penalty_quotient, + genesis_fork: spec.genesis_fork.clone(), + safe_slots_to_update_justified: spec.safe_slots_to_update_justified, + domain_beacon_proposer: spec.domain_beacon_proposer, + domain_beacon_attester: spec.domain_beacon_attester, + domain_randao: spec.domain_randao, + domain_deposit: spec.domain_deposit, + domain_voluntary_exit: spec.domain_voluntary_exit, + + // EthSpec + justification_bits_length: T::JustificationBitsLength::to_u32(), + max_validators_per_committee: T::MaxValidatorsPerCommittee::to_u32(), + genesis_epoch: T::genesis_epoch(), + slots_per_epoch: T::slots_per_epoch(), + slots_per_eth1_voting_period: T::slots_per_eth1_voting_period(), + slots_per_historical_root: T::slots_per_historical_root(), + epochs_per_historical_vector: T::epochs_per_historical_vector(), + epochs_per_slashings_vector: T::EpochsPerSlashingsVector::to_usize(), + historical_roots_limit: T::HistoricalRootsLimit::to_u64(), + validator_registry_limit: T::ValidatorRegistryLimit::to_u64(), + max_proposer_slashings: T::MaxProposerSlashings::to_u32(), + max_attester_slashings: T::MaxAttesterSlashings::to_u32(), + max_attestations: T::MaxAttestations::to_u32(), + max_deposits: T::MaxDeposits::to_u32(), + max_voluntary_exits: T::MaxVoluntaryExits::to_u32(), + + // Unused + early_derived_secret_penalty_max_future_epochs: 0, + max_seed_lookahead: 0, + deposit_contract_address: String::new(), + + // Phase 1 + epochs_per_custody_period: 0, + custody_period_to_randao_padding: 0, + shard_slots_per_beacon_slot: 0, + epochs_per_shard_period: 0, + phase_1_fork_epoch: 0, + phase_1_fork_slot: 0, + domain_custody_bit_challenge: 0, + domain_shard_proposer: 0, + domain_shard_attester: 0, + max_epochs_per_crosslink: 0, + } + } + + pub fn apply_to_chain_spec(&self, chain_spec: &ChainSpec) -> Option { + // Checking for EthSpec constants + if self.justification_bits_length != T::JustificationBitsLength::to_u32() + || self.max_validators_per_committee != T::MaxValidatorsPerCommittee::to_u32() + || self.genesis_epoch != T::genesis_epoch() + || self.slots_per_epoch != T::slots_per_epoch() + || self.slots_per_eth1_voting_period != T::slots_per_eth1_voting_period() + || self.slots_per_historical_root != T::slots_per_historical_root() + || self.epochs_per_historical_vector != T::epochs_per_historical_vector() + || self.epochs_per_slashings_vector != T::EpochsPerSlashingsVector::to_usize() + || self.historical_roots_limit != T::HistoricalRootsLimit::to_u64() + || self.validator_registry_limit != T::ValidatorRegistryLimit::to_u64() + || self.max_proposer_slashings != T::MaxProposerSlashings::to_u32() + || self.max_attester_slashings != T::MaxAttesterSlashings::to_u32() + || self.max_attestations != T::MaxAttestations::to_u32() + || self.max_deposits != T::MaxDeposits::to_u32() + || self.max_voluntary_exits != T::MaxVoluntaryExits::to_u32() + { + return None; + } + + // Create a ChainSpec from the yaml config + Some(ChainSpec { + far_future_epoch: Epoch::from(self.far_future_epoch), + base_rewards_per_epoch: self.base_rewards_per_epoch, + deposit_contract_tree_depth: self.deposit_contract_tree_depth, + seconds_per_day: self.seconds_per_day, + target_committee_size: self.target_committee_size, + min_per_epoch_churn_limit: self.min_per_epoch_churn_limit, + churn_limit_quotient: self.churn_limit_quotient, + shuffle_round_count: self.shuffle_round_count, + min_genesis_active_validator_count: self.min_genesis_active_validator_count, + min_genesis_time: self.min_genesis_time, + min_deposit_amount: self.min_deposit_amount, + max_effective_balance: self.max_effective_balance, + ejection_balance: self.ejection_balance, + effective_balance_increment: self.effective_balance_increment, + genesis_slot: Slot::from(self.genesis_slot), + bls_withdrawal_prefix_byte: self.bls_withdrawal_prefix, + milliseconds_per_slot: self.seconds_per_slot * 1000, + min_attestation_inclusion_delay: self.min_attestation_inclusion_delay, + min_seed_lookahead: Epoch::from(self.min_seed_lookahead), + min_validator_withdrawability_delay: Epoch::from( + self.min_validator_withdrawability_delay, + ), + persistent_committee_period: self.persistent_committee_period, + min_epochs_to_inactivity_penalty: self.min_epochs_to_inactivity_penalty, + base_reward_factor: self.base_reward_factor, + whistleblower_reward_quotient: self.whistleblower_reward_quotient, + proposer_reward_quotient: self.proposer_reward_quotient, + inactivity_penalty_quotient: self.inactivity_penalty_quotient, + min_slashing_penalty_quotient: self.min_slashing_penalty_quotient, + domain_beacon_proposer: self.domain_beacon_proposer, + domain_randao: self.domain_randao, + domain_deposit: self.domain_deposit, + domain_voluntary_exit: self.domain_voluntary_exit, + boot_nodes: chain_spec.boot_nodes.clone(), + genesis_fork: chain_spec.genesis_fork.clone(), + ..*chain_spec + }) + } +} + +#[cfg(test)] +mod yaml_tests { + use super::*; + use std::fs::OpenOptions; + use tempfile::NamedTempFile; + + #[test] + fn minimal_round_trip() { + // create temp file + let tmp_file = NamedTempFile::new().expect("failed to create temp file"); + let writer = OpenOptions::new() + .read(false) + .write(true) + .open(tmp_file.as_ref()) + .expect("error opening file"); + let minimal_spec = ChainSpec::minimal(); + + let yamlconfig = YamlConfig::from_spec::(&minimal_spec); + // write fresh minimal config to file + serde_yaml::to_writer(writer, &yamlconfig).expect("failed to write or serialize"); + + let reader = OpenOptions::new() + .read(true) + .write(false) + .open(tmp_file.as_ref()) + .expect("error while opening the file"); + // deserialize minimal config from file + let from: YamlConfig = serde_yaml::from_reader(reader).expect("error while deserializing"); + assert_eq!(from, yamlconfig); + } + + #[test] + fn mainnet_round_trip() { + let tmp_file = NamedTempFile::new().expect("failed to create temp file"); + let writer = OpenOptions::new() + .read(false) + .write(true) + .open(tmp_file.as_ref()) + .expect("error opening file"); + let mainnet_spec = ChainSpec::mainnet(); + let yamlconfig = YamlConfig::from_spec::(&mainnet_spec); + serde_yaml::to_writer(writer, &yamlconfig).expect("failed to write or serialize"); + + let reader = OpenOptions::new() + .read(true) + .write(false) + .open(tmp_file.as_ref()) + .expect("error while opening the file"); + let from: YamlConfig = serde_yaml::from_reader(reader).expect("error while deserializing"); + assert_eq!(from, yamlconfig); + } + + #[test] + fn apply_to_spec() { + let mut spec = ChainSpec::minimal(); + let yamlconfig = YamlConfig::from_spec::(&spec); + + // modifying the original spec + spec.deposit_contract_tree_depth += 1; + // Applying a yaml config with incorrect EthSpec should fail + let res = yamlconfig.apply_to_chain_spec::(&spec); + assert_eq!(res, None); + + // Applying a yaml config with correct EthSpec should NOT fail + let new_spec = yamlconfig + .apply_to_chain_spec::(&spec) + .expect("should have applied spec"); + assert_eq!(new_spec, ChainSpec::minimal()); + } +} diff --git a/eth2/types/src/utils/serde_utils.rs b/eth2/types/src/utils/serde_utils.rs index a9b75079c..8d8e7dff0 100644 --- a/eth2/types/src/utils/serde_utils.rs +++ b/eth2/types/src/utils/serde_utils.rs @@ -11,7 +11,11 @@ where { let s: String = Deserialize::deserialize(deserializer)?; - u8::from_str_radix(&s.as_str()[2..], 16).map_err(D::Error::custom) + let start = match s.as_str().get(2..) { + Some(start) => start, + None => return Err(D::Error::custom("string length too small")), + }; + u8::from_str_radix(&start, 16).map_err(D::Error::custom) } #[allow(clippy::trivially_copy_pass_by_ref)] // Serde requires the `byte` to be a ref. @@ -25,13 +29,45 @@ where serializer.serialize_str(&hex) } +pub fn u32_from_hex_str<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let s: String = Deserialize::deserialize(deserializer)?; + let start = s + .as_str() + .get(2..) + .ok_or_else(|| D::Error::custom("string length too small"))?; + + u32::from_str_radix(&start, 16) + .map_err(D::Error::custom) + .map(u32::from_be) +} + +#[allow(clippy::trivially_copy_pass_by_ref)] // Serde requires the `num` to be a ref. +pub fn u32_to_hex_str(num: &u32, serializer: S) -> Result +where + S: Serializer, +{ + let mut hex: String = "0x".to_string(); + let bytes = num.to_le_bytes(); + hex.push_str(&hex::encode(&bytes)); + + serializer.serialize_str(&hex) +} + pub fn fork_from_hex_str<'de, D>(deserializer: D) -> Result<[u8; FORK_BYTES_LEN], D::Error> where D: Deserializer<'de>, { let s: String = Deserialize::deserialize(deserializer)?; let mut array = [0 as u8; FORK_BYTES_LEN]; - let decoded: Vec = hex::decode(&s.as_str()[2..]).map_err(D::Error::custom)?; + + let start = s + .as_str() + .get(2..) + .ok_or_else(|| D::Error::custom("string length too small"))?; + let decoded: Vec = hex::decode(&start).map_err(D::Error::custom)?; if decoded.len() != FORK_BYTES_LEN { return Err(D::Error::custom("Fork length too long")); @@ -76,7 +112,12 @@ where { let s: String = Deserialize::deserialize(deserializer)?; let mut array = [0 as u8; GRAFFITI_BYTES_LEN]; - let decoded: Vec = hex::decode(&s.as_str()[2..]).map_err(D::Error::custom)?; + + let start = s + .as_str() + .get(2..) + .ok_or_else(|| D::Error::custom("string length too small"))?; + let decoded: Vec = hex::decode(&start).map_err(D::Error::custom)?; if decoded.len() > GRAFFITI_BYTES_LEN { return Err(D::Error::custom("Fork length too long"));