Merge pull request #299 from sigp/fix-test-harness

Fix test harness

beacon_node/beacon_chain/test_harness/Cargo.toml

@@ -12,12 +12,7 @@ path = "src/bin.rs"
 name = "test_harness"
 path = "src/lib.rs"
 
-[[bench]]
-name = "state_transition"
-harness = false
-
 [dev-dependencies]
-criterion = "0.2"
 state_processing = { path = "../../../eth2/state_processing" }
 
 [dependencies]

beacon_node/beacon_chain/test_harness/benches/state_transition.rs

@@ -1,69 +0,0 @@
-use criterion::Criterion;
-use criterion::{black_box, criterion_group, criterion_main, Benchmark};
-// use env_logger::{Builder, Env};
-use state_processing::SlotProcessable;
-use test_harness::BeaconChainHarness;
-use types::{ChainSpec, Hash256};
-
-fn mid_epoch_state_transition(c: &mut Criterion) {
-    // Builder::from_env(Env::default().default_filter_or("debug")).init();
-
-    let validator_count = 1000;
-    let mut rig = BeaconChainHarness::new(ChainSpec::foundation(), validator_count);
-
-    let epoch_depth = (rig.spec.slots_per_epoch * 2) + (rig.spec.slots_per_epoch / 2);
-
-    for _ in 0..epoch_depth {
-        rig.advance_chain_with_block();
-    }
-
-    let state = rig.beacon_chain.state.read().clone();
-
-    assert!((state.slot + 1) % rig.spec.slots_per_epoch != 0);
-
-    c.bench_function("mid-epoch state transition 10k validators", move |b| {
-        let state = state.clone();
-        b.iter(|| {
-            let mut state = state.clone();
-            black_box(state.per_slot_processing(Hash256::zero(), &rig.spec))
-        })
-    });
-}
-
-fn epoch_boundary_state_transition(c: &mut Criterion) {
-    // Builder::from_env(Env::default().default_filter_or("debug")).init();
-
-    let validator_count = 10000;
-    let mut rig = BeaconChainHarness::new(ChainSpec::foundation(), validator_count);
-
-    let epoch_depth = rig.spec.slots_per_epoch * 2;
-
-    for _ in 0..(epoch_depth - 1) {
-        rig.advance_chain_with_block();
-    }
-
-    let state = rig.beacon_chain.state.read().clone();
-
-    assert_eq!((state.slot + 1) % rig.spec.slots_per_epoch, 0);
-
-    c.bench(
-        "routines",
-        Benchmark::new("routine_1", move |b| {
-            let state = state.clone();
-            b.iter(|| {
-                let mut state = state.clone();
-                black_box(black_box(
-                    state.per_slot_processing(Hash256::zero(), &rig.spec),
-                ))
-            })
-        })
-        .sample_size(5), // sample size is low because function is sloooow.
-    );
-}
-
-criterion_group!(
-    benches,
-    mid_epoch_state_transition,
-    epoch_boundary_state_transition
-);
-criterion_main!(benches);

beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs

@@ -1,7 +1,6 @@
 use super::ValidatorHarness;
 use beacon_chain::{BeaconChain, BlockProcessingOutcome};
 pub use beacon_chain::{BeaconChainError, CheckPoint};
-use bls::get_withdrawal_credentials;
 use db::{
     stores::{BeaconBlockStore, BeaconStateStore},
     MemoryDB,
@@ -12,15 +11,9 @@ use rayon::prelude::*;
 use slot_clock::TestingSlotClock;
 use ssz::TreeHash;
 use std::collections::HashSet;
-use std::fs::File;
 use std::iter::FromIterator;
-use std::path::Path;
 use std::sync::Arc;
-use types::{beacon_state::BeaconStateBuilder, test_utils::generate_deterministic_keypairs, *};
+use types::{test_utils::TestingBeaconStateBuilder, *};
-
-mod generate_deposits;
-
-pub use generate_deposits::generate_deposits_from_keypairs;
 
 /// The beacon chain harness simulates a single beacon node with `validator_count` validators connected
 /// to it. Each validator is provided a borrow to the beacon chain, where it may read
@@ -42,95 +35,17 @@ impl BeaconChainHarness {
     ///
     /// - A keypair, `BlockProducer` and `Attester` for each validator.
     /// - A new BeaconChain struct where the given validators are in the genesis.
-    pub fn new(
+    pub fn new(spec: ChainSpec, validator_count: usize) -> Self {
-        spec: ChainSpec,
-        validator_count: usize,
-        validators_dir: Option<&Path>,
-        skip_deposit_verification: bool,
-    ) -> Self {
         let db = Arc::new(MemoryDB::open());
         let block_store = Arc::new(BeaconBlockStore::new(db.clone()));
         let state_store = Arc::new(BeaconStateStore::new(db.clone()));
-        let genesis_time = 1_549_935_547; // 12th Feb 2018 (arbitrary value in the past).
         let slot_clock = TestingSlotClock::new(spec.genesis_slot.as_u64());
         let fork_choice = BitwiseLMDGhost::new(block_store.clone(), state_store.clone());
-        let latest_eth1_data = Eth1Data {
-            deposit_root: Hash256::zero(),
-            block_hash: Hash256::zero(),
-        };
 
-        let mut state_builder = BeaconStateBuilder::new(genesis_time, latest_eth1_data, &spec);
+        let state_builder =
+            TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, &spec);
+        let (genesis_state, keypairs) = state_builder.build();
 
-        // If a `validators_dir` is specified, load the keypairs a YAML file.
-        //
-        // Otherwise, generate them deterministically where the first validator has a secret key of
-        // `1`, etc.
-        let keypairs = if let Some(path) = validators_dir {
-            debug!("Loading validator keypairs from file...");
-            let keypairs_file = File::open(path.join("keypairs.yaml")).unwrap();
-            let mut keypairs: Vec<Keypair> = serde_yaml::from_reader(&keypairs_file).unwrap();
-            keypairs.truncate(validator_count);
-            keypairs
-        } else {
-            debug!("Generating validator keypairs...");
-            generate_deterministic_keypairs(validator_count)
-        };
-
-        // Skipping deposit verification means directly generating `Validator` records, instead
-        // of generating `Deposit` objects, verifying them and converting them into `Validator`
-        // records.
-        //
-        // It is much faster to skip deposit verification, however it does not test the initial
-        // validator induction part of beacon chain genesis.
-        if skip_deposit_verification {
-            let validators = keypairs
-                .iter()
-                .map(|keypair| {
-                    let withdrawal_credentials = Hash256::from_slice(&get_withdrawal_credentials(
-                        &keypair.pk,
-                        spec.bls_withdrawal_prefix_byte,
-                    ));
-
-                    Validator {
-                        pubkey: keypair.pk.clone(),
-                        withdrawal_credentials,
-                        activation_epoch: spec.far_future_epoch,
-                        exit_epoch: spec.far_future_epoch,
-                        withdrawable_epoch: spec.far_future_epoch,
-                        initiated_exit: false,
-                        slashed: false,
-                    }
-                })
-                .collect();
-
-            let balances = vec![32_000_000_000; validator_count];
-
-            state_builder.import_existing_validators(
-                validators,
-                balances,
-                validator_count as u64,
-                &spec,
-            );
-        } else {
-            debug!("Generating initial validator deposits...");
-            let deposits = generate_deposits_from_keypairs(
-                &keypairs,
-                genesis_time,
-                spec.get_domain(
-                    spec.genesis_epoch,
-                    Domain::Deposit,
-                    &Fork {
-                        previous_version: spec.genesis_fork_version,
-                        current_version: spec.genesis_fork_version,
-                        epoch: spec.genesis_epoch,
-                    },
-                ),
-                &spec,
-            );
-            state_builder.process_initial_deposits(&deposits, &spec);
-        };
-
-        let genesis_state = state_builder.build(&spec).unwrap();
         let state_root = Hash256::from_slice(&genesis_state.hash_tree_root());
         let genesis_block = BeaconBlock::genesis(state_root, &spec);
 

beacon_node/beacon_chain/test_harness/src/beacon_chain_harness/generate_deposits.rs

@@ -1,46 +0,0 @@
-use bls::get_withdrawal_credentials;
-use log::debug;
-use rayon::prelude::*;
-use types::*;
-
-/// Generates a `Deposit` for each keypairs
-pub fn generate_deposits_from_keypairs(
-    keypairs: &[Keypair],
-    genesis_time: u64,
-    domain: u64,
-    spec: &ChainSpec,
-) -> Vec<Deposit> {
-    debug!(
-        "Generating {} validator deposits from random keypairs...",
-        keypairs.len()
-    );
-
-    let initial_validator_deposits = keypairs
-        .par_iter()
-        .map(|keypair| {
-            let withdrawal_credentials = Hash256::from_slice(
-                &get_withdrawal_credentials(&keypair.pk, spec.bls_withdrawal_prefix_byte)[..],
-            );
-            Deposit {
-                branch: vec![], // branch verification is not specified.
-                index: 0,       // index verification is not specified.
-                deposit_data: DepositData {
-                    amount: 32_000_000_000, // 32 ETH (in Gwei)
-                    timestamp: genesis_time - 1,
-                    deposit_input: DepositInput {
-                        pubkey: keypair.pk.clone(),
-                        // Validator can withdraw using their main keypair.
-                        withdrawal_credentials: withdrawal_credentials.clone(),
-                        proof_of_possession: DepositInput::create_proof_of_possession(
-                            &keypair,
-                            &withdrawal_credentials,
-                            domain,
-                        ),
-                    },
-                },
-            }
-        })
-        .collect();
-
-    initial_validator_deposits
-}

beacon_node/beacon_chain/test_harness/src/lib.rs

@@ -15,7 +15,7 @@
 //! let validator_count = 8;
 //! let spec = ChainSpec::few_validators();
 //!
-//! let mut harness = BeaconChainHarness::new(spec, validator_count, None, true);
+//! let mut harness = BeaconChainHarness::new(spec, validator_count);
 //!
 //! harness.advance_chain_with_block();
 //!

beacon_node/beacon_chain/test_harness/src/run_test.rs

@@ -1,6 +1,5 @@
 use crate::test_case::TestCase;
 use clap::ArgMatches;
-use std::path::Path;
 use std::{fs::File, io::prelude::*};
 use yaml_rust::YamlLoader;
 
@@ -17,10 +16,6 @@ pub fn run_test(matches: &ArgMatches) {
         };
 
         for doc in &docs {
-            let validators_dir = matches
-                .value_of("validators_dir")
-                .and_then(|dir_str| Some(Path::new(dir_str)));
-
             // For each `test_cases` YAML in the document, build a `TestCase`, execute it and
             // assert that the execution result matches the test_case description.
             //
@@ -35,7 +30,7 @@ pub fn run_test(matches: &ArgMatches) {
             // panics with a message.
             for test_case in doc["test_cases"].as_vec().unwrap() {
                 let test_case = TestCase::from_yaml(test_case);
-                test_case.assert_result_valid(test_case.execute(validators_dir))
+                test_case.assert_result_valid(test_case.execute())
             }
         }
     }

beacon_node/beacon_chain/test_harness/src/test_case.rs

@@ -6,7 +6,6 @@ use beacon_chain::CheckPoint;
 use bls::get_withdrawal_credentials;
 use log::{info, warn};
 use ssz::SignedRoot;
-use std::path::Path;
 use types::*;
 
 use types::test_utils::{TestingAttesterSlashingBuilder, TestingProposerSlashingBuilder};
@@ -69,7 +68,7 @@ impl TestCase {
 
     /// Executes the test case, returning an `ExecutionResult`.
     #[allow(clippy::cyclomatic_complexity)]
-    pub fn execute(&self, validators_dir: Option<&Path>) -> ExecutionResult {
+    pub fn execute(&self) -> ExecutionResult {
         let spec = self.spec();
         let validator_count = self.config.deposits_for_chain_start;
         let slots = self.config.num_slots;
@@ -79,7 +78,7 @@ impl TestCase {
             validator_count
         );
 
-        let mut harness = BeaconChainHarness::new(spec, validator_count, validators_dir, true);
+        let mut harness = BeaconChainHarness::new(spec, validator_count);
 
         info!("Starting simulation across {} slots...", slots);
 

beacon_node/beacon_chain/test_harness/tests/chain.rs

@@ -10,7 +10,7 @@ fn it_can_build_on_genesis_block() {
     let spec = ChainSpec::few_validators();
     let validator_count = 8;
 
-    let mut harness = BeaconChainHarness::new(spec, validator_count as usize, None, true);
+    let mut harness = BeaconChainHarness::new(spec, validator_count as usize);
 
     harness.advance_chain_with_block();
 }
@@ -25,7 +25,7 @@ fn it_can_produce_past_first_epoch_boundary() {
 
     debug!("Starting harness build...");
 
-    let mut harness = BeaconChainHarness::new(spec, validator_count, None, true);
+    let mut harness = BeaconChainHarness::new(spec, validator_count);
 
     debug!("Harness built, tests starting..");