From ddac7540bc5f687b5e9aeee6b918c72f711f7b07 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 8 Mar 2019 16:10:21 +1100 Subject: [PATCH] Allow test_harness to load validators from file. --- .../test_harness/src/beacon_chain_harness.rs | 45 ++++++++++++---- .../beacon_chain/test_harness/src/prepare.rs | 52 +++++++++++++------ eth2/types/Cargo.toml | 1 + eth2/types/src/beacon_state/builder.rs | 40 +++++++++++--- eth2/types/src/slot_epoch.rs | 6 +-- eth2/types/src/validator.rs | 4 +- 6 files changed, 109 insertions(+), 39 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index c41f6fa1e..1ebe4dc74 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -11,6 +11,7 @@ use rayon::prelude::*; use slot_clock::TestingSlotClock; use ssz::TreeHash; use std::collections::HashSet; +use std::fs::File; use std::iter::FromIterator; use std::path::Path; use std::sync::Arc; @@ -54,22 +55,44 @@ impl BeaconChainHarness { block_hash: Hash256::zero(), }; - let (keypairs, initial_validator_deposits) = if let Some(path) = validators_dir { - let keypairs_path = path.join("keypairs.yaml"); - let deposits_path = path.join("deposits.yaml"); - load_deposits_from_file( - validator_count, - &keypairs_path.as_path(), - &deposits_path.as_path(), - ) + let mut state_builder = BeaconStateBuilder::new(genesis_time, latest_eth1_data, &spec); + + // If a `validators_dir` is specified, load the keypairs and validators from YAML files. + // + // Otherwise, build all the keypairs and initial validator deposits manually. + // + // It is _much_ faster to load from YAML, however it does skip all the initial processing + // and verification of `Deposits`, so it is a slightly less comprehensive test. + let keypairs = if let Some(path) = validators_dir { + debug!("Loading validator keypairs from file..."); + let keypairs_file = File::open(path.join("keypairs.yaml")).unwrap(); + let mut keypairs: Vec = serde_yaml::from_reader(&keypairs_file).unwrap(); + keypairs.truncate(validator_count); + + debug!("Loading validators from file..."); + let validators_file = File::open(path.join("validators.yaml")).unwrap(); + let mut validators: Vec = serde_yaml::from_reader(&validators_file).unwrap(); + validators.truncate(validator_count); + + let balances = vec![32_000_000_000; validator_count]; + + state_builder.import_existing_validators( + validators, + balances, + validator_count as u64, + &spec, + ); + + keypairs } else { + debug!("Generating validator keypairs..."); let keypairs = generate_deterministic_keypairs(validator_count); + debug!("Generating initial validator deposits..."); let deposits = generate_deposits_from_keypairs(&keypairs, genesis_time, &spec); - (keypairs, deposits) + state_builder.process_initial_deposits(&deposits, &spec); + keypairs }; - let mut state_builder = BeaconStateBuilder::new(genesis_time, latest_eth1_data, &spec); - state_builder.process_initial_deposits(&initial_validator_deposits, &spec); let genesis_state = state_builder.build(&spec).unwrap(); let state_root = Hash256::from_slice(&genesis_state.hash_tree_root()); let genesis_block = BeaconBlock::genesis(state_root, &spec); diff --git a/beacon_node/beacon_chain/test_harness/src/prepare.rs b/beacon_node/beacon_chain/test_harness/src/prepare.rs index 0a4f4c34f..36a99317f 100644 --- a/beacon_node/beacon_chain/test_harness/src/prepare.rs +++ b/beacon_node/beacon_chain/test_harness/src/prepare.rs @@ -1,6 +1,5 @@ -use crate::beacon_chain_harness::{ - generate_deposits_from_keypairs, generate_deterministic_keypairs, -}; +use crate::beacon_chain_harness::generate_deterministic_keypairs; +use bls::get_withdrawal_credentials; use clap::{value_t, ArgMatches}; use log::debug; use serde_yaml; @@ -9,33 +8,52 @@ use std::{fs, fs::File}; use types::*; const KEYPAIRS_FILE: &str = "keypairs.yaml"; -const DEPOSITS_FILE: &str = "deposits.yaml"; +const VALIDATORS_FILE: &str = "validators.yaml"; pub fn prepare(matches: &ArgMatches, spec: &ChainSpec) { let validator_count = value_t!(matches.value_of("validator_count"), usize) .expect("Validator count is required argument"); - let genesis_time = - value_t!(matches.value_of("genesis_time"), u64).expect("Genesis time is required argument"); let output_dir = matches .value_of("output_dir") .expect("Output dir has a default value."); - debug!("Created keypairs and deposits, writing to file..."); + debug!("Created keypairs and validators, writing to file..."); fs::create_dir_all(Path::new(output_dir)).unwrap(); - // Ensure that keypairs is dropped before writing deposits, this provides a big memory saving + // Ensure that keypairs is dropped before writing validators, this provides a big memory saving // for large validator_counts. - let deposits = { + let validators: Vec = { debug!("Creating {} keypairs...", validator_count); let keypairs = generate_deterministic_keypairs(validator_count); debug!("Writing {} keypairs to file...", validator_count); write_keypairs(output_dir, &keypairs); - debug!("Creating {} deposits to file...", validator_count); - generate_deposits_from_keypairs(&keypairs, genesis_time, &spec) + debug!("Creating {} validators...", validator_count); + keypairs + .iter() + .map(|keypair| generate_validator(&keypair, spec)) + .collect() }; - debug!("Writing {} deposits to file...", validator_count); - write_deposits(output_dir, &deposits); + + debug!("Writing {} validators to file...", validator_count); + write_validators(output_dir, &validators); +} + +fn generate_validator(keypair: &Keypair, spec: &ChainSpec) -> Validator { + let withdrawal_credentials = Hash256::from_slice(&get_withdrawal_credentials( + &keypair.pk, + spec.bls_withdrawal_prefix_byte, + )); + + Validator { + pubkey: keypair.pk.clone(), + withdrawal_credentials, + activation_epoch: spec.far_future_epoch, + exit_epoch: spec.far_future_epoch, + withdrawable_epoch: spec.far_future_epoch, + initiated_exit: false, + slashed: false, + } } fn write_keypairs(output_dir: &str, keypairs: &[Keypair]) { @@ -44,8 +62,8 @@ fn write_keypairs(output_dir: &str, keypairs: &[Keypair]) { serde_yaml::to_writer(keypairs_file, &keypairs).unwrap(); } -fn write_deposits(output_dir: &str, deposits: &[Deposit]) { - let deposits_path = Path::new(output_dir).join(DEPOSITS_FILE); - let deposits_file = File::create(deposits_path).unwrap(); - serde_yaml::to_writer(deposits_file, &deposits).unwrap(); +fn write_validators(output_dir: &str, validators: &[Validator]) { + let validators_path = Path::new(output_dir).join(VALIDATORS_FILE); + let validators_file = File::create(validators_path).unwrap(); + serde_yaml::to_writer(validators_file, &validators).unwrap(); } diff --git a/eth2/types/Cargo.toml b/eth2/types/Cargo.toml index ea1343dba..e2930040d 100644 --- a/eth2/types/Cargo.toml +++ b/eth2/types/Cargo.toml @@ -17,6 +17,7 @@ rand = "0.5.5" serde = "1.0" serde_derive = "1.0" serde_json = "1.0" +serde_yaml = "0.8" slog = "^2.2.3" ssz = { path = "../utils/ssz" } ssz_derive = { path = "../utils/ssz_derive" } diff --git a/eth2/types/src/beacon_state/builder.rs b/eth2/types/src/beacon_state/builder.rs index 4bb5e2cc6..f6d7b3900 100644 --- a/eth2/types/src/beacon_state/builder.rs +++ b/eth2/types/src/beacon_state/builder.rs @@ -33,7 +33,7 @@ impl BeaconStateBuilder { } } - /// Produce the first state of the Beacon Chain. + /// Process deposit objects. /// /// Spec v0.4.0 pub fn process_initial_deposits( @@ -48,19 +48,47 @@ impl BeaconStateBuilder { self.state.process_deposits(deposit_data, spec); + self.activate_genesis_validators(spec); + + self.state.deposit_index = initial_validator_deposits.len() as u64; + } + + fn activate_genesis_validators(&mut self, spec: &ChainSpec) { for validator_index in 0..self.state.validator_registry.len() { if self.state.get_effective_balance(validator_index, spec) >= spec.max_deposit_amount { self.state.activate_validator(validator_index, true, spec); } } - - self.state.deposit_index = initial_validator_deposits.len() as u64; } - /// Builds a `BeaconState` using the `BeaconState::genesis(..)` function. + /// Instantiate the validator registry from a YAML file. /// - /// Each validator is assigned a unique, randomly-generated keypair and all - /// proof-of-possessions are verified during genesis. + /// This skips a lot of signing and verification, useful for fast test setups. + /// + /// Spec v0.4.0 + pub fn import_existing_validators( + &mut self, + validators: Vec, + initial_balances: Vec, + deposit_index: u64, + spec: &ChainSpec, + ) { + self.state.validator_registry = validators; + + assert_eq!( + self.state.validator_registry.len(), + initial_balances.len(), + "Not enough balances for validators" + ); + + self.state.validator_balances = initial_balances; + + self.activate_genesis_validators(spec); + + self.state.deposit_index = deposit_index; + } + + /// Updates the final state variables and returns a fully built genesis state. /// /// Spec v0.4.0 pub fn build(mut self, spec: &ChainSpec) -> Result { diff --git a/eth2/types/src/slot_epoch.rs b/eth2/types/src/slot_epoch.rs index 7753027a6..2af7f5196 100644 --- a/eth2/types/src/slot_epoch.rs +++ b/eth2/types/src/slot_epoch.rs @@ -12,7 +12,7 @@ use crate::slot_height::SlotHeight; /// may lead to programming errors which are not detected by the compiler. use crate::test_utils::TestRandom; use rand::RngCore; -use serde_derive::Serialize; +use serde_derive::{Deserialize, Serialize}; use slog; use ssz::{hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash}; use std::cmp::{Ord, Ordering}; @@ -21,10 +21,10 @@ use std::hash::{Hash, Hasher}; use std::iter::Iterator; use std::ops::{Add, AddAssign, Div, DivAssign, Mul, MulAssign, Rem, Sub, SubAssign}; -#[derive(Eq, Debug, Clone, Copy, Default, Serialize)] +#[derive(Eq, Debug, Clone, Copy, Default, Serialize, Deserialize)] pub struct Slot(u64); -#[derive(Eq, Debug, Clone, Copy, Default, Serialize)] +#[derive(Eq, Debug, Clone, Copy, Default, Serialize, Deserialize)] pub struct Epoch(u64); impl_common!(Slot); diff --git a/eth2/types/src/validator.rs b/eth2/types/src/validator.rs index 43701ca05..59f6c5826 100644 --- a/eth2/types/src/validator.rs +++ b/eth2/types/src/validator.rs @@ -1,13 +1,13 @@ use crate::{test_utils::TestRandom, Epoch, Hash256, PublicKey}; use rand::RngCore; -use serde_derive::Serialize; +use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; /// Information about a `BeaconChain` validator. /// /// Spec v0.4.0 -#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, TestRandom, TreeHash)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TestRandom, TreeHash)] pub struct Validator { pub pubkey: PublicKey, pub withdrawal_credentials: Hash256,