cerc-io/lighthouse
16
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into lighthouse-255

Browse Source
This commit is contained in:
thojest 2019-03-18 10:32:35 +01:00
commit c269cb40c4
18 changed files with 143 additions and 235 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
beacon_node/beacon_chain/test_harness/Cargo.toml
View File

@ -12,12 +12,7 @@ path = "src/bin.rs"
name = "test_harness"
path = "src/lib.rs"
[[bench]]
name = "state_transition"
harness = false
[dev-dependencies]
criterion = "0.2"
state_processing = { path = "../../../eth2/state_processing" }
[dependencies]

69
beacon_node/beacon_chain/test_harness/benches/state_transition.rs
View File

@ -1,69 +0,0 @@
use criterion::Criterion;
use criterion::{black_box, criterion_group, criterion_main, Benchmark};
// use env_logger::{Builder, Env};
use state_processing::SlotProcessable;
use test_harness::BeaconChainHarness;
use types::{ChainSpec, Hash256};
fn mid_epoch_state_transition(c: &mut Criterion) {
// Builder::from_env(Env::default().default_filter_or("debug")).init();
let validator_count = 1000;
let mut rig = BeaconChainHarness::new(ChainSpec::foundation(), validator_count);
let epoch_depth = (rig.spec.slots_per_epoch * 2) + (rig.spec.slots_per_epoch / 2);
for _ in 0..epoch_depth {
rig.advance_chain_with_block();
}
let state = rig.beacon_chain.state.read().clone();
assert!((state.slot + 1) % rig.spec.slots_per_epoch != 0);
c.bench_function("mid-epoch state transition 10k validators", move |b| {
let state = state.clone();
b.iter(|| {
let mut state = state.clone();
black_box(state.per_slot_processing(Hash256::zero(), &rig.spec))
})
});
}
fn epoch_boundary_state_transition(c: &mut Criterion) {
// Builder::from_env(Env::default().default_filter_or("debug")).init();
let validator_count = 10000;
let mut rig = BeaconChainHarness::new(ChainSpec::foundation(), validator_count);
let epoch_depth = rig.spec.slots_per_epoch * 2;
for _ in 0..(epoch_depth - 1) {
rig.advance_chain_with_block();
}
let state = rig.beacon_chain.state.read().clone();
assert_eq!((state.slot + 1) % rig.spec.slots_per_epoch, 0);
c.bench(
"routines",
Benchmark::new("routine_1", move |b| {
let state = state.clone();
b.iter(|| {
let mut state = state.clone();
black_box(black_box(
state.per_slot_processing(Hash256::zero(), &rig.spec),
))
})
})
.sample_size(5), // sample size is low because function is sloooow.
);
}
criterion_group!(
benches,
mid_epoch_state_transition,
epoch_boundary_state_transition
);
criterion_main!(benches);

95
beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs
View File

@ -1,7 +1,6 @@
use super::ValidatorHarness;
use beacon_chain::{BeaconChain, BlockProcessingOutcome};
pub use beacon_chain::{BeaconChainError, CheckPoint};
use bls::get_withdrawal_credentials;
use db::{
stores::{BeaconBlockStore, BeaconStateStore},
MemoryDB,
@ -12,15 +11,9 @@ use rayon::prelude::*;
use slot_clock::TestingSlotClock;
use ssz::TreeHash;
use std::collections::HashSet;
use std::fs::File;
use std::iter::FromIterator;
use std::path::Path;
use std::sync::Arc;
use types::{beacon_state::BeaconStateBuilder, test_utils::generate_deterministic_keypairs, *};
mod generate_deposits;
pub use generate_deposits::generate_deposits_from_keypairs;
use types::{test_utils::TestingBeaconStateBuilder, *};
/// The beacon chain harness simulates a single beacon node with `validator_count` validators connected
/// to it. Each validator is provided a borrow to the beacon chain, where it may read
@ -42,95 +35,17 @@ impl BeaconChainHarness {
///
/// - A keypair, `BlockProducer` and `Attester` for each validator.
/// - A new BeaconChain struct where the given validators are in the genesis.
pub fn new(
spec: ChainSpec,
validator_count: usize,
validators_dir: Option<&Path>,
skip_deposit_verification: bool,
) -> Self {
pub fn new(spec: ChainSpec, validator_count: usize) -> Self {
let db = Arc::new(MemoryDB::open());
let block_store = Arc::new(BeaconBlockStore::new(db.clone()));
let state_store = Arc::new(BeaconStateStore::new(db.clone()));
let genesis_time = 1_549_935_547; // 12th Feb 2018 (arbitrary value in the past).
let slot_clock = TestingSlotClock::new(spec.genesis_slot.as_u64());
let fork_choice = BitwiseLMDGhost::new(block_store.clone(), state_store.clone());
let latest_eth1_data = Eth1Data {
deposit_root: Hash256::zero(),
block_hash: Hash256::zero(),
};
let mut state_builder = BeaconStateBuilder::new(genesis_time, latest_eth1_data, &spec);
let state_builder =
TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, &spec);
let (genesis_state, keypairs) = state_builder.build();
// If a `validators_dir` is specified, load the keypairs a YAML file.
//
// Otherwise, generate them deterministically where the first validator has a secret key of
// `1`, etc.
let keypairs = if let Some(path) = validators_dir {
debug!("Loading validator keypairs from file...");
let keypairs_file = File::open(path.join("keypairs.yaml")).unwrap();
let mut keypairs: Vec<Keypair> = serde_yaml::from_reader(&keypairs_file).unwrap();
keypairs.truncate(validator_count);
keypairs
} else {
debug!("Generating validator keypairs...");
generate_deterministic_keypairs(validator_count)
};
// Skipping deposit verification means directly generating `Validator` records, instead
// of generating `Deposit` objects, verifying them and converting them into `Validator`
// records.
//
// It is much faster to skip deposit verification, however it does not test the initial
// validator induction part of beacon chain genesis.
if skip_deposit_verification {
let validators = keypairs
.iter()
.map(|keypair| {
let withdrawal_credentials = Hash256::from_slice(&get_withdrawal_credentials(
&keypair.pk,
spec.bls_withdrawal_prefix_byte,
));
Validator {
pubkey: keypair.pk.clone(),
withdrawal_credentials,
activation_epoch: spec.far_future_epoch,
exit_epoch: spec.far_future_epoch,
withdrawable_epoch: spec.far_future_epoch,
initiated_exit: false,
slashed: false,
}
})
.collect();
let balances = vec![32_000_000_000; validator_count];
state_builder.import_existing_validators(
validators,
balances,
validator_count as u64,
&spec,
);
} else {
debug!("Generating initial validator deposits...");
let deposits = generate_deposits_from_keypairs(
&keypairs,
genesis_time,
spec.get_domain(
spec.genesis_epoch,
Domain::Deposit,
&Fork {
previous_version: spec.genesis_fork_version,
current_version: spec.genesis_fork_version,
epoch: spec.genesis_epoch,
},
),
&spec,
);
state_builder.process_initial_deposits(&deposits, &spec);
};
let genesis_state = state_builder.build(&spec).unwrap();
let state_root = Hash256::from_slice(&genesis_state.hash_tree_root());
let genesis_block = BeaconBlock::genesis(state_root, &spec);

46
beacon_node/beacon_chain/test_harness/src/beacon_chain_harness/generate_deposits.rs
View File

@ -1,46 +0,0 @@
use bls::get_withdrawal_credentials;
use log::debug;
use rayon::prelude::*;
use types::*;
/// Generates a `Deposit` for each keypairs
pub fn generate_deposits_from_keypairs(
keypairs: &[Keypair],
genesis_time: u64,
domain: u64,
spec: &ChainSpec,
) -> Vec<Deposit> {
debug!(
"Generating {} validator deposits from random keypairs...",
keypairs.len()
);
let initial_validator_deposits = keypairs
.par_iter()
.map(|keypair| {
let withdrawal_credentials = Hash256::from_slice(
&get_withdrawal_credentials(&keypair.pk, spec.bls_withdrawal_prefix_byte)[..],
);
Deposit {
branch: vec![], // branch verification is not specified.
index: 0, // index verification is not specified.
deposit_data: DepositData {
amount: 32_000_000_000, // 32 ETH (in Gwei)
timestamp: genesis_time - 1,
deposit_input: DepositInput {
pubkey: keypair.pk.clone(),
// Validator can withdraw using their main keypair.
withdrawal_credentials: withdrawal_credentials.clone(),
proof_of_possession: DepositInput::create_proof_of_possession(
&keypair,
&withdrawal_credentials,
domain,
),
},
},
}
})
.collect();
initial_validator_deposits
}

2
beacon_node/beacon_chain/test_harness/src/lib.rs
View File

@ -15,7 +15,7 @@
//! let validator_count = 8;
//! let spec = ChainSpec::few_validators();
//!
//! let mut harness = BeaconChainHarness::new(spec, validator_count, None, true);
//! let mut harness = BeaconChainHarness::new(spec, validator_count);
//!
//! harness.advance_chain_with_block();
//!

7
beacon_node/beacon_chain/test_harness/src/run_test.rs
View File

@ -1,6 +1,5 @@
use crate::test_case::TestCase;
use clap::ArgMatches;
use std::path::Path;
use std::{fs::File, io::prelude::*};
use yaml_rust::YamlLoader;
@ -17,10 +16,6 @@ pub fn run_test(matches: &ArgMatches) {
};
for doc in &docs {
let validators_dir = matches
.value_of("validators_dir")
.and_then(|dir_str| Some(Path::new(dir_str)));
// For each `test_cases` YAML in the document, build a `TestCase`, execute it and
// assert that the execution result matches the test_case description.
//
@ -35,7 +30,7 @@ pub fn run_test(matches: &ArgMatches) {
// panics with a message.
for test_case in doc["test_cases"].as_vec().unwrap() {
let test_case = TestCase::from_yaml(test_case);
test_case.assert_result_valid(test_case.execute(validators_dir))
test_case.assert_result_valid(test_case.execute())
}
}
}

5
beacon_node/beacon_chain/test_harness/src/test_case.rs
View File

@ -6,7 +6,6 @@ use beacon_chain::CheckPoint;
use bls::get_withdrawal_credentials;
use log::{info, warn};
use ssz::SignedRoot;
use std::path::Path;
use types::*;
use types::test_utils::{TestingAttesterSlashingBuilder, TestingProposerSlashingBuilder};
@ -69,7 +68,7 @@ impl TestCase {
/// Executes the test case, returning an `ExecutionResult`.
#[allow(clippy::cyclomatic_complexity)]
pub fn execute(&self, validators_dir: Option<&Path>) -> ExecutionResult {
pub fn execute(&self) -> ExecutionResult {
let spec = self.spec();
let validator_count = self.config.deposits_for_chain_start;
let slots = self.config.num_slots;
@ -79,7 +78,7 @@ impl TestCase {
validator_count
);
let mut harness = BeaconChainHarness::new(spec, validator_count, validators_dir, true);
let mut harness = BeaconChainHarness::new(spec, validator_count);
info!("Starting simulation across {} slots...", slots);

4
beacon_node/beacon_chain/test_harness/tests/chain.rs
View File

@ -10,7 +10,7 @@ fn it_can_build_on_genesis_block() {
let spec = ChainSpec::few_validators();
let validator_count = 8;
let mut harness = BeaconChainHarness::new(spec, validator_count as usize, None, true);
let mut harness = BeaconChainHarness::new(spec, validator_count as usize);
harness.advance_chain_with_block();
}
@ -25,7 +25,7 @@ fn it_can_produce_past_first_epoch_boundary() {
debug!("Starting harness build...");
let mut harness = BeaconChainHarness::new(spec, validator_count, None, true);
let mut harness = BeaconChainHarness::new(spec, validator_count);
debug!("Harness built, tests starting..");

17
eth2/state_processing/benches/bench_block_processing.rs
View File

@ -426,6 +426,23 @@ fn bench_block_processing(
.sample_size(10),
);
let mut state = initial_state.clone();
state.drop_pubkey_cache();
c.bench(
&format!("{}/block_processing", desc),
Benchmark::new("build_pubkey_cache", move |b| {
b.iter_batched(
|| state.clone(),
|mut state| {
state.update_pubkey_cache().unwrap();
state
},
criterion::BatchSize::SmallInput,
)
})
.sample_size(10),
);
let block = initial_block.clone();
c.bench(
&format!("{}/block_processing", desc),

5
eth2/state_processing/benches/benches.rs
View File

@ -1,14 +1,11 @@
use criterion::Benchmark;
use criterion::Criterion;
use criterion::{criterion_group, criterion_main};
use env_logger::{Builder, Env};
use types::test_utils::TestingBeaconStateBuilder;
use types::*;
mod bench_block_processing;
mod bench_epoch_processing;
pub const VALIDATOR_COUNT: usize = 300_032;
pub const VALIDATOR_COUNT: usize = 16_384;
// `LOG_LEVEL == "debug"` gives logs, but they're very noisy and slow down benching.
pub const LOG_LEVEL: &str = "";

9
eth2/state_processing/src/per_block_processing.rs
View File

@ -373,19 +373,20 @@ pub fn process_deposits(
.map_err(|e| e.into_with_index(i))
})?;
let public_key_to_index_hashmap = build_public_key_hashmap(&state);
// Check `state.deposit_index` and update the state in series.
for (i, deposit) in deposits.iter().enumerate() {
verify_deposit_index(state, deposit).map_err(|e| e.into_with_index(i))?;
// Ensure the state's pubkey cache is fully up-to-date, it will be used to check to see if the
// depositing validator already exists in the registry.
state.update_pubkey_cache()?;
// Get an `Option<u64>` where `u64` is the validator index if this deposit public key
// already exists in the beacon_state.
//
// This function also verifies the withdrawal credentials.
let validator_index =
get_existing_validator_index(state, deposit, &public_key_to_index_hashmap)
.map_err(|e| e.into_with_index(i))?;
get_existing_validator_index(state, deposit).map_err(|e| e.into_with_index(i))?;
let deposit_data = &deposit.deposit_data;
let deposit_input = &deposit.deposit_data.deposit_input;

5
eth2/state_processing/src/per_block_processing/errors.rs
View File

@ -294,6 +294,8 @@ impl_into_with_index_without_beacon_error!(
pub enum DepositValidationError {
/// Validation completed successfully and the object is invalid.
Invalid(DepositInvalid),
/// Encountered a `BeaconStateError` whilst attempting to determine validity.
BeaconStateError(BeaconStateError),
}
/// Describes why an object is invalid.
@ -313,7 +315,8 @@ pub enum DepositInvalid {
BadMerkleProof,
}
impl_into_with_index_without_beacon_error!(DepositValidationError, DepositInvalid);
impl_from_beacon_state_error!(DepositValidationError);
impl_into_with_index_with_beacon_error!(DepositValidationError, DepositInvalid);
/*
* `Exit` Validation

7
eth2/state_processing/src/per_block_processing/verify_deposit.rs
View File

@ -72,11 +72,12 @@ pub fn build_public_key_hashmap(state: &BeaconState) -> PublicKeyValidatorIndexH
pub fn get_existing_validator_index(
state: &BeaconState,
deposit: &Deposit,
pubkey_map: &HashMap<PublicKey, u64>,
) -> Result<Option<u64>, Error> {
let deposit_input = &deposit.deposit_data.deposit_input;
let validator_index = pubkey_map.get(&deposit_input.pubkey).and_then(|i| Some(*i));
let validator_index = state
.get_validator_index(&deposit_input.pubkey)?
.and_then(|i| Some(i));
match validator_index {
None => Ok(None),
@ -86,7 +87,7 @@ pub fn get_existing_validator_index(
== state.validator_registry[index as usize].withdrawal_credentials,
Invalid::BadWithdrawalCredentials
);
Ok(Some(index))
Ok(Some(index as u64))
}
}
}

51
eth2/types/src/beacon_state.rs
View File

@ -5,6 +5,7 @@ use helpers::*;
use honey_badger_split::SplitExt;
use int_to_bytes::int_to_bytes32;
use log::{debug, error, trace};
use pubkey_cache::PubkeyCache;
use rand::RngCore;
use serde_derive::Serialize;
use ssz::{hash, Decodable, DecodeError, Encodable, SignedRoot, SszStream, TreeHash};
@ -16,6 +17,7 @@ pub use builder::BeaconStateBuilder;
mod builder;
mod epoch_cache;
pub mod helpers;
mod pubkey_cache;
mod tests;
pub type Committee = Vec<usize>;
@ -52,6 +54,11 @@ pub enum Error {
InsufficientAttestations,
InsufficientCommittees,
EpochCacheUninitialized(RelativeEpoch),
PubkeyCacheInconsistent,
PubkeyCacheIncomplete {
cache_len: usize,
registry_len: usize,
},
}
macro_rules! safe_add_assign {
@ -108,6 +115,7 @@ pub struct BeaconState {
// Caching (not in the spec)
pub cache_index_offset: usize,
pub caches: Vec<EpochCache>,
pub pubkey_cache: PubkeyCache,
}
impl BeaconState {
@ -186,6 +194,7 @@ impl BeaconState {
*/
cache_index_offset: 0,
caches: vec![EpochCache::empty(); CACHED_EPOCHS],
pubkey_cache: PubkeyCache::empty(),
}
}
@ -293,6 +302,46 @@ impl BeaconState {
}
}
/// Updates the pubkey cache, if required.
///
/// Adds all `pubkeys` from the `validator_registry` which are not already in the cache. Will
/// never re-add a pubkey.
pub fn update_pubkey_cache(&mut self) -> Result<(), Error> {
for (i, validator) in self
.validator_registry
.iter()
.enumerate()
.skip(self.pubkey_cache.len())
{
let success = self.pubkey_cache.insert(validator.pubkey.clone(), i);
if !success {
return Err(Error::PubkeyCacheInconsistent);
}
}
Ok(())
}
/// Completely drops the `pubkey_cache`, replacing it with a new, empty cache.
pub fn drop_pubkey_cache(&mut self) {
self.pubkey_cache = PubkeyCache::empty()
}
/// If a validator pubkey exists in the validator registry, returns `Some(i)`, otherwise
/// returns `None`.
///
/// Requires a fully up-to-date `pubkey_cache`, returns an error if this is not the case.
pub fn get_validator_index(&self, pubkey: &PublicKey) -> Result<Option<usize>, Error> {
if self.pubkey_cache.len() == self.validator_registry.len() {
Ok(self.pubkey_cache.get(pubkey))
} else {
Err(Error::PubkeyCacheIncomplete {
cache_len: self.pubkey_cache.len(),
registry_len: self.validator_registry.len(),
})
}
}
/// The epoch corresponding to `self.slot`.
///
/// Spec v0.4.0
@ -1188,6 +1237,7 @@ impl Decodable for BeaconState {
deposit_index,
cache_index_offset: 0,
caches: vec![EpochCache::empty(); CACHED_EPOCHS],
pubkey_cache: PubkeyCache::empty(),
},
i,
))
@ -1258,6 +1308,7 @@ impl<T: RngCore> TestRandom<T> for BeaconState {
deposit_index: <_>::random_for_test(rng),
cache_index_offset: 0,
caches: vec![EpochCache::empty(); CACHED_EPOCHS],
pubkey_cache: PubkeyCache::empty(),
}
}
}

45
eth2/types/src/beacon_state/pubkey_cache.rs Normal file
View File

@ -0,0 +1,45 @@
use crate::*;
use serde_derive::Serialize;
use std::collections::HashMap;
type ValidatorIndex = usize;
#[derive(Debug, PartialEq, Clone, Default, Serialize)]
pub struct PubkeyCache {
map: HashMap<PublicKey, ValidatorIndex>,
}
impl PubkeyCache {
/// Instantiates a new, empty cache.
pub fn empty() -> Self {
Self {
map: HashMap::new(),
}
}
/// Returns the number of validator indices already in the map.
pub fn len(&self) -> ValidatorIndex {
self.map.len()
}
/// Inserts a validator index into the map.
///
/// The added index must equal the number of validators already added to the map. This ensures
/// that an index is never skipped.
pub fn insert(&mut self, pubkey: PublicKey, index: ValidatorIndex) -> bool {
if index == self.map.len() {
self.map.insert(pubkey, index);
true
} else {
false
}
}
/// Inserts a validator index into the map.
///
/// The added index must equal the number of validators already added to the map. This ensures
/// that an index is never skipped.
pub fn get(&self, pubkey: &PublicKey) -> Option<ValidatorIndex> {
self.map.get(pubkey).cloned()
}
}

2
eth2/types/src/lib.rs
View File

@ -1,3 +1,5 @@
//! Ethereum 2.0 types
#[macro_use]
pub mod test_utils;

2
eth2/types/src/test_utils/testing_beacon_state_builder.rs
View File

@ -160,6 +160,8 @@ impl TestingBeaconStateBuilder {
state.build_epoch_cache(RelativeEpoch::Current, &spec)?;
state.build_epoch_cache(RelativeEpoch::Next, &spec)?;
state.update_pubkey_cache()?;
Ok(())
}

2
eth2/utils/bls/Cargo.toml
View File

@ -5,7 +5,7 @@ authors = ["Paul Hauner <paul@paulhauner.com>"]
edition = "2018"
[dependencies]
bls-aggregates = { git = "https://github.com/sigp/signature-schemes", tag = "0.6.0" }
bls-aggregates = { git = "https://github.com/sigp/signature-schemes", tag = "0.6.1" }
hashing = { path = "../hashing" }
hex = "0.3"
serde = "1.0"