Merge branch 'master' into lighthouse-255

beacon_node/beacon_chain/test_harness/Cargo.toml

@@ -12,12 +12,7 @@ path = "src/bin.rs"
 name = "test_harness"
 path = "src/lib.rs"
 
-[[bench]]
-name = "state_transition"
-harness = false
-
 [dev-dependencies]
-criterion = "0.2"
 state_processing = { path = "../../../eth2/state_processing" }
 
 [dependencies]

beacon_node/beacon_chain/test_harness/benches/state_transition.rs

@@ -1,69 +0,0 @@
-use criterion::Criterion;
-use criterion::{black_box, criterion_group, criterion_main, Benchmark};
-// use env_logger::{Builder, Env};
-use state_processing::SlotProcessable;
-use test_harness::BeaconChainHarness;
-use types::{ChainSpec, Hash256};
-
-fn mid_epoch_state_transition(c: &mut Criterion) {
-    // Builder::from_env(Env::default().default_filter_or("debug")).init();
-
-    let validator_count = 1000;
-    let mut rig = BeaconChainHarness::new(ChainSpec::foundation(), validator_count);
-
-    let epoch_depth = (rig.spec.slots_per_epoch * 2) + (rig.spec.slots_per_epoch / 2);
-
-    for _ in 0..epoch_depth {
-        rig.advance_chain_with_block();
-    }
-
-    let state = rig.beacon_chain.state.read().clone();
-
-    assert!((state.slot + 1) % rig.spec.slots_per_epoch != 0);
-
-    c.bench_function("mid-epoch state transition 10k validators", move |b| {
-        let state = state.clone();
-        b.iter(|| {
-            let mut state = state.clone();
-            black_box(state.per_slot_processing(Hash256::zero(), &rig.spec))
-        })
-    });
-}
-
-fn epoch_boundary_state_transition(c: &mut Criterion) {
-    // Builder::from_env(Env::default().default_filter_or("debug")).init();
-
-    let validator_count = 10000;
-    let mut rig = BeaconChainHarness::new(ChainSpec::foundation(), validator_count);
-
-    let epoch_depth = rig.spec.slots_per_epoch * 2;
-
-    for _ in 0..(epoch_depth - 1) {
-        rig.advance_chain_with_block();
-    }
-
-    let state = rig.beacon_chain.state.read().clone();
-
-    assert_eq!((state.slot + 1) % rig.spec.slots_per_epoch, 0);
-
-    c.bench(
-        "routines",
-        Benchmark::new("routine_1", move |b| {
-            let state = state.clone();
-            b.iter(|| {
-                let mut state = state.clone();
-                black_box(black_box(
-                    state.per_slot_processing(Hash256::zero(), &rig.spec),
-                ))
-            })
-        })
-        .sample_size(5), // sample size is low because function is sloooow.
-    );
-}
-
-criterion_group!(
-    benches,
-    mid_epoch_state_transition,
-    epoch_boundary_state_transition
-);
-criterion_main!(benches);

beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs

@@ -1,7 +1,6 @@
 use super::ValidatorHarness;
 use beacon_chain::{BeaconChain, BlockProcessingOutcome};
 pub use beacon_chain::{BeaconChainError, CheckPoint};
-use bls::get_withdrawal_credentials;
 use db::{
     stores::{BeaconBlockStore, BeaconStateStore},
     MemoryDB,
@@ -12,15 +11,9 @@ use rayon::prelude::*;
 use slot_clock::TestingSlotClock;
 use ssz::TreeHash;
 use std::collections::HashSet;
-use std::fs::File;
 use std::iter::FromIterator;
-use std::path::Path;
 use std::sync::Arc;
-use types::{beacon_state::BeaconStateBuilder, test_utils::generate_deterministic_keypairs, *};
+use types::{test_utils::TestingBeaconStateBuilder, *};
-
-mod generate_deposits;
-
-pub use generate_deposits::generate_deposits_from_keypairs;
 
 /// The beacon chain harness simulates a single beacon node with `validator_count` validators connected
 /// to it. Each validator is provided a borrow to the beacon chain, where it may read
@@ -42,95 +35,17 @@ impl BeaconChainHarness {
     ///
     /// - A keypair, `BlockProducer` and `Attester` for each validator.
     /// - A new BeaconChain struct where the given validators are in the genesis.
-    pub fn new(
+    pub fn new(spec: ChainSpec, validator_count: usize) -> Self {
-        spec: ChainSpec,
-        validator_count: usize,
-        validators_dir: Option<&Path>,
-        skip_deposit_verification: bool,
-    ) -> Self {
         let db = Arc::new(MemoryDB::open());
         let block_store = Arc::new(BeaconBlockStore::new(db.clone()));
         let state_store = Arc::new(BeaconStateStore::new(db.clone()));
-        let genesis_time = 1_549_935_547; // 12th Feb 2018 (arbitrary value in the past).
         let slot_clock = TestingSlotClock::new(spec.genesis_slot.as_u64());
         let fork_choice = BitwiseLMDGhost::new(block_store.clone(), state_store.clone());
-        let latest_eth1_data = Eth1Data {
-            deposit_root: Hash256::zero(),
-            block_hash: Hash256::zero(),
-        };
 
-        let mut state_builder = BeaconStateBuilder::new(genesis_time, latest_eth1_data, &spec);
+        let state_builder =
+            TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, &spec);
+        let (genesis_state, keypairs) = state_builder.build();
 
-        // If a `validators_dir` is specified, load the keypairs a YAML file.
-        //
-        // Otherwise, generate them deterministically where the first validator has a secret key of
-        // `1`, etc.
-        let keypairs = if let Some(path) = validators_dir {
-            debug!("Loading validator keypairs from file...");
-            let keypairs_file = File::open(path.join("keypairs.yaml")).unwrap();
-            let mut keypairs: Vec<Keypair> = serde_yaml::from_reader(&keypairs_file).unwrap();
-            keypairs.truncate(validator_count);
-            keypairs
-        } else {
-            debug!("Generating validator keypairs...");
-            generate_deterministic_keypairs(validator_count)
-        };
-
-        // Skipping deposit verification means directly generating `Validator` records, instead
-        // of generating `Deposit` objects, verifying them and converting them into `Validator`
-        // records.
-        //
-        // It is much faster to skip deposit verification, however it does not test the initial
-        // validator induction part of beacon chain genesis.
-        if skip_deposit_verification {
-            let validators = keypairs
-                .iter()
-                .map(|keypair| {
-                    let withdrawal_credentials = Hash256::from_slice(&get_withdrawal_credentials(
-                        &keypair.pk,
-                        spec.bls_withdrawal_prefix_byte,
-                    ));
-
-                    Validator {
-                        pubkey: keypair.pk.clone(),
-                        withdrawal_credentials,
-                        activation_epoch: spec.far_future_epoch,
-                        exit_epoch: spec.far_future_epoch,
-                        withdrawable_epoch: spec.far_future_epoch,
-                        initiated_exit: false,
-                        slashed: false,
-                    }
-                })
-                .collect();
-
-            let balances = vec![32_000_000_000; validator_count];
-
-            state_builder.import_existing_validators(
-                validators,
-                balances,
-                validator_count as u64,
-                &spec,
-            );
-        } else {
-            debug!("Generating initial validator deposits...");
-            let deposits = generate_deposits_from_keypairs(
-                &keypairs,
-                genesis_time,
-                spec.get_domain(
-                    spec.genesis_epoch,
-                    Domain::Deposit,
-                    &Fork {
-                        previous_version: spec.genesis_fork_version,
-                        current_version: spec.genesis_fork_version,
-                        epoch: spec.genesis_epoch,
-                    },
-                ),
-                &spec,
-            );
-            state_builder.process_initial_deposits(&deposits, &spec);
-        };
-
-        let genesis_state = state_builder.build(&spec).unwrap();
         let state_root = Hash256::from_slice(&genesis_state.hash_tree_root());
         let genesis_block = BeaconBlock::genesis(state_root, &spec);
 

beacon_node/beacon_chain/test_harness/src/beacon_chain_harness/generate_deposits.rs

@@ -1,46 +0,0 @@
-use bls::get_withdrawal_credentials;
-use log::debug;
-use rayon::prelude::*;
-use types::*;
-
-/// Generates a `Deposit` for each keypairs
-pub fn generate_deposits_from_keypairs(
-    keypairs: &[Keypair],
-    genesis_time: u64,
-    domain: u64,
-    spec: &ChainSpec,
-) -> Vec<Deposit> {
-    debug!(
-        "Generating {} validator deposits from random keypairs...",
-        keypairs.len()
-    );
-
-    let initial_validator_deposits = keypairs
-        .par_iter()
-        .map(|keypair| {
-            let withdrawal_credentials = Hash256::from_slice(
-                &get_withdrawal_credentials(&keypair.pk, spec.bls_withdrawal_prefix_byte)[..],
-            );
-            Deposit {
-                branch: vec![], // branch verification is not specified.
-                index: 0,       // index verification is not specified.
-                deposit_data: DepositData {
-                    amount: 32_000_000_000, // 32 ETH (in Gwei)
-                    timestamp: genesis_time - 1,
-                    deposit_input: DepositInput {
-                        pubkey: keypair.pk.clone(),
-                        // Validator can withdraw using their main keypair.
-                        withdrawal_credentials: withdrawal_credentials.clone(),
-                        proof_of_possession: DepositInput::create_proof_of_possession(
-                            &keypair,
-                            &withdrawal_credentials,
-                            domain,
-                        ),
-                    },
-                },
-            }
-        })
-        .collect();
-
-    initial_validator_deposits
-}

beacon_node/beacon_chain/test_harness/src/lib.rs

@@ -15,7 +15,7 @@
 //! let validator_count = 8;
 //! let spec = ChainSpec::few_validators();
 //!
-//! let mut harness = BeaconChainHarness::new(spec, validator_count, None, true);
+//! let mut harness = BeaconChainHarness::new(spec, validator_count);
 //!
 //! harness.advance_chain_with_block();
 //!

beacon_node/beacon_chain/test_harness/src/run_test.rs

@@ -1,6 +1,5 @@
 use crate::test_case::TestCase;
 use clap::ArgMatches;
-use std::path::Path;
 use std::{fs::File, io::prelude::*};
 use yaml_rust::YamlLoader;
 
@@ -17,10 +16,6 @@ pub fn run_test(matches: &ArgMatches) {
         };
 
         for doc in &docs {
-            let validators_dir = matches
-                .value_of("validators_dir")
-                .and_then(|dir_str| Some(Path::new(dir_str)));
-
             // For each `test_cases` YAML in the document, build a `TestCase`, execute it and
             // assert that the execution result matches the test_case description.
             //
@@ -35,7 +30,7 @@ pub fn run_test(matches: &ArgMatches) {
             // panics with a message.
             for test_case in doc["test_cases"].as_vec().unwrap() {
                 let test_case = TestCase::from_yaml(test_case);
-                test_case.assert_result_valid(test_case.execute(validators_dir))
+                test_case.assert_result_valid(test_case.execute())
             }
         }
     }

beacon_node/beacon_chain/test_harness/src/test_case.rs

@@ -6,7 +6,6 @@ use beacon_chain::CheckPoint;
 use bls::get_withdrawal_credentials;
 use log::{info, warn};
 use ssz::SignedRoot;
-use std::path::Path;
 use types::*;
 
 use types::test_utils::{TestingAttesterSlashingBuilder, TestingProposerSlashingBuilder};
@@ -69,7 +68,7 @@ impl TestCase {
 
     /// Executes the test case, returning an `ExecutionResult`.
     #[allow(clippy::cyclomatic_complexity)]
-    pub fn execute(&self, validators_dir: Option<&Path>) -> ExecutionResult {
+    pub fn execute(&self) -> ExecutionResult {
         let spec = self.spec();
         let validator_count = self.config.deposits_for_chain_start;
         let slots = self.config.num_slots;
@@ -79,7 +78,7 @@ impl TestCase {
             validator_count
         );
 
-        let mut harness = BeaconChainHarness::new(spec, validator_count, validators_dir, true);
+        let mut harness = BeaconChainHarness::new(spec, validator_count);
 
         info!("Starting simulation across {} slots...", slots);
 

beacon_node/beacon_chain/test_harness/tests/chain.rs

@@ -10,7 +10,7 @@ fn it_can_build_on_genesis_block() {
     let spec = ChainSpec::few_validators();
     let validator_count = 8;
 
-    let mut harness = BeaconChainHarness::new(spec, validator_count as usize, None, true);
+    let mut harness = BeaconChainHarness::new(spec, validator_count as usize);
 
     harness.advance_chain_with_block();
 }
@@ -25,7 +25,7 @@ fn it_can_produce_past_first_epoch_boundary() {
 
     debug!("Starting harness build...");
 
-    let mut harness = BeaconChainHarness::new(spec, validator_count, None, true);
+    let mut harness = BeaconChainHarness::new(spec, validator_count);
 
     debug!("Harness built, tests starting..");
 

eth2/state_processing/benches/bench_block_processing.rs

@@ -426,6 +426,23 @@ fn bench_block_processing(
         .sample_size(10),
     );
 
+    let mut state = initial_state.clone();
+    state.drop_pubkey_cache();
+    c.bench(
+        &format!("{}/block_processing", desc),
+        Benchmark::new("build_pubkey_cache", move |b| {
+            b.iter_batched(
+                || state.clone(),
+                |mut state| {
+                    state.update_pubkey_cache().unwrap();
+                    state
+                },
+                criterion::BatchSize::SmallInput,
+            )
+        })
+        .sample_size(10),
+    );
+
     let block = initial_block.clone();
     c.bench(
         &format!("{}/block_processing", desc),

eth2/state_processing/benches/benches.rs

@@ -1,14 +1,11 @@
-use criterion::Benchmark;
 use criterion::Criterion;
 use criterion::{criterion_group, criterion_main};
 use env_logger::{Builder, Env};
-use types::test_utils::TestingBeaconStateBuilder;
-use types::*;
 
 mod bench_block_processing;
 mod bench_epoch_processing;
 
-pub const VALIDATOR_COUNT: usize = 300_032;
+pub const VALIDATOR_COUNT: usize = 16_384;
 
 // `LOG_LEVEL == "debug"` gives logs, but they're very noisy and slow down benching.
 pub const LOG_LEVEL: &str = "";

eth2/state_processing/src/per_block_processing.rs

@@ -373,19 +373,20 @@ pub fn process_deposits(
                 .map_err(|e| e.into_with_index(i))
         })?;
 
-    let public_key_to_index_hashmap = build_public_key_hashmap(&state);
-
     // Check `state.deposit_index` and update the state in series.
     for (i, deposit) in deposits.iter().enumerate() {
         verify_deposit_index(state, deposit).map_err(|e| e.into_with_index(i))?;
 
+        // Ensure the state's pubkey cache is fully up-to-date, it will be used to check to see if the
+        // depositing validator already exists in the registry.
+        state.update_pubkey_cache()?;
+
         // Get an `Option<u64>` where `u64` is the validator index if this deposit public key
         // already exists in the beacon_state.
         //
         // This function also verifies the withdrawal credentials.
         let validator_index =
-            get_existing_validator_index(state, deposit, &public_key_to_index_hashmap)
+            get_existing_validator_index(state, deposit).map_err(|e| e.into_with_index(i))?;
-                .map_err(|e| e.into_with_index(i))?;
 
         let deposit_data = &deposit.deposit_data;
         let deposit_input = &deposit.deposit_data.deposit_input;

eth2/state_processing/src/per_block_processing/errors.rs

@@ -294,6 +294,8 @@ impl_into_with_index_without_beacon_error!(
 pub enum DepositValidationError {
     /// Validation completed successfully and the object is invalid.
     Invalid(DepositInvalid),
+    /// Encountered a `BeaconStateError` whilst attempting to determine validity.
+    BeaconStateError(BeaconStateError),
 }
 
 /// Describes why an object is invalid.
@@ -313,7 +315,8 @@ pub enum DepositInvalid {
     BadMerkleProof,
 }
 
-impl_into_with_index_without_beacon_error!(DepositValidationError, DepositInvalid);
+impl_from_beacon_state_error!(DepositValidationError);
+impl_into_with_index_with_beacon_error!(DepositValidationError, DepositInvalid);
 
 /*
  * `Exit` Validation

eth2/state_processing/src/per_block_processing/verify_deposit.rs

@@ -72,11 +72,12 @@ pub fn build_public_key_hashmap(state: &BeaconState) -> PublicKeyValidatorIndexH
 pub fn get_existing_validator_index(
     state: &BeaconState,
     deposit: &Deposit,
-    pubkey_map: &HashMap<PublicKey, u64>,
 ) -> Result<Option<u64>, Error> {
     let deposit_input = &deposit.deposit_data.deposit_input;
 
-    let validator_index = pubkey_map.get(&deposit_input.pubkey).and_then(|i| Some(*i));
+    let validator_index = state
+        .get_validator_index(&deposit_input.pubkey)?
+        .and_then(|i| Some(i));
 
     match validator_index {
         None => Ok(None),
@@ -86,7 +87,7 @@ pub fn get_existing_validator_index(
                     == state.validator_registry[index as usize].withdrawal_credentials,
                 Invalid::BadWithdrawalCredentials
             );
-            Ok(Some(index))
+            Ok(Some(index as u64))
         }
     }
 }

eth2/types/src/beacon_state.rs

@@ -5,6 +5,7 @@ use helpers::*;
 use honey_badger_split::SplitExt;
 use int_to_bytes::int_to_bytes32;
 use log::{debug, error, trace};
+use pubkey_cache::PubkeyCache;
 use rand::RngCore;
 use serde_derive::Serialize;
 use ssz::{hash, Decodable, DecodeError, Encodable, SignedRoot, SszStream, TreeHash};
@@ -16,6 +17,7 @@ pub use builder::BeaconStateBuilder;
 mod builder;
 mod epoch_cache;
 pub mod helpers;
+mod pubkey_cache;
 mod tests;
 
 pub type Committee = Vec<usize>;
@@ -52,6 +54,11 @@ pub enum Error {
     InsufficientAttestations,
     InsufficientCommittees,
     EpochCacheUninitialized(RelativeEpoch),
+    PubkeyCacheInconsistent,
+    PubkeyCacheIncomplete {
+        cache_len: usize,
+        registry_len: usize,
+    },
 }
 
 macro_rules! safe_add_assign {
@@ -108,6 +115,7 @@ pub struct BeaconState {
     // Caching (not in the spec)
     pub cache_index_offset: usize,
     pub caches: Vec<EpochCache>,
+    pub pubkey_cache: PubkeyCache,
 }
 
 impl BeaconState {
@@ -186,6 +194,7 @@ impl BeaconState {
              */
             cache_index_offset: 0,
             caches: vec![EpochCache::empty(); CACHED_EPOCHS],
+            pubkey_cache: PubkeyCache::empty(),
         }
     }
 
@@ -293,6 +302,46 @@ impl BeaconState {
         }
     }
 
+    /// Updates the pubkey cache, if required.
+    ///
+    /// Adds all `pubkeys` from the `validator_registry` which are not already in the cache. Will
+    /// never re-add a pubkey.
+    pub fn update_pubkey_cache(&mut self) -> Result<(), Error> {
+        for (i, validator) in self
+            .validator_registry
+            .iter()
+            .enumerate()
+            .skip(self.pubkey_cache.len())
+        {
+            let success = self.pubkey_cache.insert(validator.pubkey.clone(), i);
+            if !success {
+                return Err(Error::PubkeyCacheInconsistent);
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Completely drops the `pubkey_cache`, replacing it with a new, empty cache.
+    pub fn drop_pubkey_cache(&mut self) {
+        self.pubkey_cache = PubkeyCache::empty()
+    }
+
+    /// If a validator pubkey exists in the validator registry, returns `Some(i)`, otherwise
+    /// returns `None`.
+    ///
+    /// Requires a fully up-to-date `pubkey_cache`, returns an error if this is not the case.
+    pub fn get_validator_index(&self, pubkey: &PublicKey) -> Result<Option<usize>, Error> {
+        if self.pubkey_cache.len() == self.validator_registry.len() {
+            Ok(self.pubkey_cache.get(pubkey))
+        } else {
+            Err(Error::PubkeyCacheIncomplete {
+                cache_len: self.pubkey_cache.len(),
+                registry_len: self.validator_registry.len(),
+            })
+        }
+    }
+
     /// The epoch corresponding to `self.slot`.
     ///
     /// Spec v0.4.0
@@ -1188,6 +1237,7 @@ impl Decodable for BeaconState {
                 deposit_index,
                 cache_index_offset: 0,
                 caches: vec![EpochCache::empty(); CACHED_EPOCHS],
+                pubkey_cache: PubkeyCache::empty(),
             },
             i,
         ))
@@ -1258,6 +1308,7 @@ impl<T: RngCore> TestRandom<T> for BeaconState {
             deposit_index: <_>::random_for_test(rng),
             cache_index_offset: 0,
             caches: vec![EpochCache::empty(); CACHED_EPOCHS],
+            pubkey_cache: PubkeyCache::empty(),
         }
     }
 }

eth2/types/src/beacon_state/pubkey_cache.rs

@@ -0,0 +1,45 @@
+use crate::*;
+use serde_derive::Serialize;
+use std::collections::HashMap;
+
+type ValidatorIndex = usize;
+
+#[derive(Debug, PartialEq, Clone, Default, Serialize)]
+pub struct PubkeyCache {
+    map: HashMap<PublicKey, ValidatorIndex>,
+}
+
+impl PubkeyCache {
+    /// Instantiates a new, empty cache.
+    pub fn empty() -> Self {
+        Self {
+            map: HashMap::new(),
+        }
+    }
+
+    /// Returns the number of validator indices already in the map.
+    pub fn len(&self) -> ValidatorIndex {
+        self.map.len()
+    }
+
+    /// Inserts a validator index into the map.
+    ///
+    /// The added index must equal the number of validators already added to the map. This ensures
+    /// that an index is never skipped.
+    pub fn insert(&mut self, pubkey: PublicKey, index: ValidatorIndex) -> bool {
+        if index == self.map.len() {
+            self.map.insert(pubkey, index);
+            true
+        } else {
+            false
+        }
+    }
+
+    /// Inserts a validator index into the map.
+    ///
+    /// The added index must equal the number of validators already added to the map. This ensures
+    /// that an index is never skipped.
+    pub fn get(&self, pubkey: &PublicKey) -> Option<ValidatorIndex> {
+        self.map.get(pubkey).cloned()
+    }
+}

eth2/types/src/lib.rs

@@ -1,3 +1,5 @@
+//! Ethereum 2.0 types
+
 #[macro_use]
 pub mod test_utils;
 

eth2/types/src/test_utils/testing_beacon_state_builder.rs

@@ -160,6 +160,8 @@ impl TestingBeaconStateBuilder {
         state.build_epoch_cache(RelativeEpoch::Current, &spec)?;
         state.build_epoch_cache(RelativeEpoch::Next, &spec)?;
 
+        state.update_pubkey_cache()?;
+
         Ok(())
     }
 

eth2/utils/bls/Cargo.toml

@@ -5,7 +5,7 @@ authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 
 [dependencies]
-bls-aggregates = { git = "https://github.com/sigp/signature-schemes", tag = "0.6.0" }
+bls-aggregates = { git = "https://github.com/sigp/signature-schemes", tag = "0.6.1" }
 hashing = { path = "../hashing" }
 hex = "0.3"
 serde = "1.0"