cerc-io/lighthouse
16
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Add builder for benches

Browse Source 
Also adds a "sane" case for block processing
This commit is contained in:
Paul Hauner 2019-03-14 16:44:56 +11:00
parent ed49023b87
commit 2535f47f13
No known key found for this signature in database
GPG Key ID: D362883A9218FCC6
4 changed files with 266 additions and 248 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

243
eth2/state_processing/benches/bench_block_processing.rs
View File

@ -1,6 +1,5 @@
use criterion::Criterion;
use criterion::{black_box, Benchmark};
use log::debug;
use ssz::TreeHash;
use state_processing::{
per_block_processing,
@ -10,252 +9,12 @@ use state_processing::{
verify_block_signature,
},
};
use types::test_utils::{TestingBeaconBlockBuilder, TestingBeaconStateBuilder};
use types::*;
/// Run the benchmarking suite on a foundation spec with 16,384 validators.
pub fn bench_block_processing_n_validators(c: &mut Criterion, validator_count: usize) {
let spec = ChainSpec::foundation();
let bench_builder = BlockBenchingBuilder::new(validator_count, &spec);
let (mut state, keypairs) = build_state(validator_count, &spec);
let block = build_block(&mut state, &keypairs, &spec);
assert_eq!(
block.body.proposer_slashings.len(),
spec.max_proposer_slashings as usize,
"The block should have the maximum possible proposer slashings"
);
assert_eq!(
block.body.attester_slashings.len(),
spec.max_attester_slashings as usize,
"The block should have the maximum possible attester slashings"
);
for attester_slashing in &block.body.attester_slashings {
let len_1 = attester_slashing
.slashable_attestation_1
.validator_indices
.len();
let len_2 = attester_slashing
.slashable_attestation_1
.validator_indices
.len();
assert!(
(len_1 == len_2) && (len_2 == spec.max_indices_per_slashable_vote as usize),
"Each attester slashing should have the maximum possible validator indices"
);
}
assert_eq!(
block.body.attestations.len(),
spec.max_attestations as usize,
"The block should have the maximum possible attestations."
);
assert_eq!(
block.body.deposits.len(),
spec.max_deposits as usize,
"The block should have the maximum possible deposits."
);
assert_eq!(
block.body.voluntary_exits.len(),
spec.max_voluntary_exits as usize,
"The block should have the maximum possible voluntary exits."
);
assert_eq!(
block.body.transfers.len(),
spec.max_transfers as usize,
"The block should have the maximum possible transfers."
);
bench_block_processing(
c,
&block,
&state,
&spec,
&format!("{}_validators", validator_count),
);
}
fn build_state(validator_count: usize, spec: &ChainSpec) -> (BeaconState, Vec<Keypair>) {
let mut builder =
TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, &spec);
// Set the state to be just before an epoch transition.
let target_slot = (spec.genesis_epoch + 4).end_slot(spec.slots_per_epoch);
builder.teleport_to_slot(target_slot, &spec);
// Builds all caches; benches will not contain shuffling/committee building times.
builder.build_caches(&spec).unwrap();
builder.build()
}
pub struct BlockBenchingBuilder {
pub state_builder: TestingBeaconStateBuilder,
pub block_builder: TestingBeaconBlockBuilder,
pub num_validators: usize,
pub num_proposer_slashings: usize,
pub num_attester_slashings: usize,
pub num_indices_per_slashable_vote: usize,
pub num_attestations: usize,
pub num_deposits: usize,
pub num_exits: usize,
pub num_transfers: usize,
}
impl BlockBenchingBuilder {
pub fn new(num_validators: usize, spec: &ChainSpec) -> Self {
let mut state_builder =
TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(num_validators, &spec);
let mut block_builder = TestingBeaconBlockBuilder::new(spec);
Self {
state_builder,
block_builder,
num_validators: 0,
num_proposer_slashings: 0,
num_attester_slashings: 0,
num_indices_per_slashable_vote: spec.max_indices_per_slashable_vote as usize,
num_attestations: 0,
num_deposits: 0,
num_exits: 0,
num_transfers: 0
}
}
pub fn maximize_block_operations(&mut self, spec: &ChainSpec) {
self.num_proposer_slashings = spec.max_proposer_slashings as usize;
self.num_attester_slashings = spec.max_attester_slashings as usize;
self.num_indices_per_slashable_vote = spec.max_indices_per_slashable_vote as usize;
self.num_attestations = spec.max_attestations as usize;
self.num_deposits = spec.max_deposits as usize;
self.num_exits = spec.max_voluntary_exits as usize;
self.num_transfers = spec.max_transfers as usize;
}
pub fn set_slot(&mut self, slot: Slot, spec: &ChainSpec) {
self.state_builder.teleport_to_slot(slot, &spec);
}
pub fn build_caches(&mut self, spec: &ChainSpec) {
// Builds all caches; benches will not contain shuffling/committee building times.
self.state_builder.build_caches(&spec).unwrap();
}
pub fn build(self, spec: &ChainSpec) -> (BeaconBlock, BeaconState) {
let (mut state, keypairs) = self.state_builder.build();
let builder = &mut self.block_builder;
builder.set_slot(state.slot);
let proposer_index = state.get_beacon_proposer_index(state.slot, spec).unwrap();
let keypair = &keypairs[proposer_index];
builder.set_randao_reveal(&keypair.sk, &state.fork, spec);
// Used as a stream of validator indices for use in slashings, exits, etc.
let mut validators_iter = (0..keypairs.len() as u64).into_iter();
// Insert `ProposerSlashing` objects.
debug!(
"Inserting {} proposer slashings...",
self.num_proposer_slashings
);
for _ in 0..self.num_proposer_slashings {
let validator_index = validators_iter.next().expect("Insufficient validators.");
builder.insert_proposer_slashing(
validator_index,
&keypairs[validator_index as usize].sk,
&state.fork,
spec,
);
}
// Insert `AttesterSlashing` objects
debug!(
"Inserting {} attester slashings...",
self.num_attester_slashings
);
for _ in 0..self.num_attester_slashings {
let mut attesters: Vec<u64> = vec![];
let mut secret_keys: Vec<&SecretKey> = vec![];
for _ in 0..self.num_indices_per_slashable_vote {
let validator_index = validators_iter.next().expect("Insufficient validators.");
attesters.push(validator_index);
secret_keys.push(&keypairs[validator_index as usize].sk);
}
builder.insert_attester_slashing(&attesters, &secret_keys, &state.fork, spec);
}
// Insert `Attestation` objects.
debug!("Inserting {} attestations...", self.num_attestations);
let all_secret_keys: Vec<&SecretKey> = keypairs.iter().map(|keypair| &keypair.sk).collect();
builder
.insert_attestations(&state, &all_secret_keys, self.num_attestations as usize, spec)
.unwrap();
// Insert `Deposit` objects.
debug!("Inserting {} deposits...", self.num_deposits);
for i in 0..self.num_deposits {
builder.insert_deposit(32_000_000_000, state.deposit_index + (i as u64), &state, spec);
}
// Insert the maximum possible number of `Exit` objects.
debug!("Inserting {} exits...", self.num_exits);
for _ in 0..self.num_exits {
let validator_index = validators_iter.next().expect("Insufficient validators.");
builder.insert_exit(
&state,
validator_index,
&keypairs[validator_index as usize].sk,
spec,
);
}
// Insert the maximum possible number of `Transfer` objects.
debug!("Inserting {} transfers...", self.num_transfers);
for _ in 0..self.num_transfers {
let validator_index = validators_iter.next().expect("Insufficient validators.");
// Manually set the validator to be withdrawn.
state.validator_registry[validator_index as usize].withdrawable_epoch =
state.previous_epoch(spec);
builder.insert_transfer(
&state,
validator_index,
validator_index,
1,
keypairs[validator_index as usize].clone(),
spec,
);
}
let mut block = builder.build(&keypair.sk, &state.fork, spec);
// Set the eth1 data to be different from the state.
block.eth1_data.block_hash = Hash256::from_slice(&vec![42; 32]);
(block, state)
}
}
/// Run the detailed benchmarking suite on the given `BeaconState`.
///
/// `desc` will be added to the title of each bench.
fn bench_block_processing(
pub fn bench_block_processing(
c: &mut Criterion,
initial_block: &BeaconBlock,
initial_state: &BeaconState,

94
eth2/state_processing/benches/benches.rs
View File

@ -1,23 +1,107 @@
use block_benching_builder::BlockBenchingBuilder;
use criterion::Criterion;
use criterion::{criterion_group, criterion_main};
use env_logger::{Builder, Env};
use log::info;
use types::*;
mod bench_block_processing;
mod bench_epoch_processing;
mod block_benching_builder;
pub const VALIDATOR_COUNT: usize = 16_384;
// `LOG_LEVEL == "debug"` gives logs, but they're very noisy and slow down benching.
pub const LOG_LEVEL: &str = "";
pub const LOG_LEVEL: &str = "info";
pub fn state_processing(c: &mut Criterion) {
/// Build a worst-case block and benchmark processing it.
pub fn block_processing_worst_case(c: &mut Criterion) {
if LOG_LEVEL != "" {
Builder::from_env(Env::default().default_filter_or(LOG_LEVEL)).init();
}
info!(
"Building worst case block bench with {} validators",
VALIDATOR_COUNT
);
bench_epoch_processing::bench_epoch_processing_n_validators(c, VALIDATOR_COUNT);
bench_block_processing::bench_block_processing_n_validators(c, VALIDATOR_COUNT);
// Use the specifications from the Eth2.0 spec.
let spec = ChainSpec::foundation();
// Create a builder for configuring the block and state for benching.
let mut bench_builder = BlockBenchingBuilder::new(VALIDATOR_COUNT, &spec);
// Set the number of included operations to be maximum (e.g., `MAX_ATTESTATIONS`, etc.)
bench_builder.maximize_block_operations(&spec);
// Set the state and block to be in the last slot of the 4th epoch.
let last_slot_of_epoch = (spec.genesis_epoch + 4).end_slot(spec.slots_per_epoch);
bench_builder.set_slot(last_slot_of_epoch, &spec);
// Build all the state caches so the build times aren't included in the benches.
bench_builder.build_caches(&spec);
// Generate the block and state.
let (block, state) = bench_builder.build(&spec);
// Run the benches.
bench_block_processing::bench_block_processing(
c,
&block,
&state,
&spec,
&format!("{}_validators/worst_case", VALIDATOR_COUNT),
);
}
criterion_group!(benches, state_processing);
/// Build a reasonable-case block and benchmark processing it.
pub fn block_processing_reasonable_case(c: &mut Criterion) {
info!(
"Building reasonable case block bench with {} validators",
VALIDATOR_COUNT
);
// Use the specifications from the Eth2.0 spec.
let spec = ChainSpec::foundation();
// Create a builder for configuring the block and state for benching.
let mut bench_builder = BlockBenchingBuilder::new(VALIDATOR_COUNT, &spec);
// Set the number of included operations to what we might expect normally.
bench_builder.num_proposer_slashings = 0;
bench_builder.num_attester_slashings = 0;
bench_builder.num_attestations = (spec.shard_count / spec.slots_per_epoch) as usize;
bench_builder.num_deposits = 2;
bench_builder.num_exits = 2;
bench_builder.num_transfers = 2;
// Set the state and block to be in the last slot of the 4th epoch.
let last_slot_of_epoch = (spec.genesis_epoch + 4).end_slot(spec.slots_per_epoch);
bench_builder.set_slot(last_slot_of_epoch, &spec);
// Build all the state caches so the build times aren't included in the benches.
bench_builder.build_caches(&spec);
// Generate the block and state.
let (block, state) = bench_builder.build(&spec);
// Run the benches.
bench_block_processing::bench_block_processing(
c,
&block,
&state,
&spec,
&format!("{}_validators/reasonable_case", VALIDATOR_COUNT),
);
}
pub fn state_processing(c: &mut Criterion) {
bench_epoch_processing::bench_epoch_processing_n_validators(c, VALIDATOR_COUNT);
}
criterion_group!(
benches,
block_processing_reasonable_case,
block_processing_worst_case,
state_processing
);
criterion_main!(benches);

175
eth2/state_processing/benches/block_benching_builder.rs Normal file
View File

@ -0,0 +1,175 @@
use log::info;
use types::test_utils::{TestingBeaconBlockBuilder, TestingBeaconStateBuilder};
use types::*;
pub struct BlockBenchingBuilder {
pub state_builder: TestingBeaconStateBuilder,
pub block_builder: TestingBeaconBlockBuilder,
pub num_validators: usize,
pub num_proposer_slashings: usize,
pub num_attester_slashings: usize,
pub num_indices_per_slashable_vote: usize,
pub num_attestations: usize,
pub num_deposits: usize,
pub num_exits: usize,
pub num_transfers: usize,
}
impl BlockBenchingBuilder {
pub fn new(num_validators: usize, spec: &ChainSpec) -> Self {
let state_builder =
TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(num_validators, &spec);
let block_builder = TestingBeaconBlockBuilder::new(spec);
Self {
state_builder,
block_builder,
num_validators: 0,
num_proposer_slashings: 0,
num_attester_slashings: 0,
num_indices_per_slashable_vote: spec.max_indices_per_slashable_vote as usize,
num_attestations: 0,
num_deposits: 0,
num_exits: 0,
num_transfers: 0,
}
}
pub fn maximize_block_operations(&mut self, spec: &ChainSpec) {
self.num_proposer_slashings = spec.max_proposer_slashings as usize;
self.num_attester_slashings = spec.max_attester_slashings as usize;
self.num_indices_per_slashable_vote = spec.max_indices_per_slashable_vote as usize;
self.num_attestations = spec.max_attestations as usize;
self.num_deposits = spec.max_deposits as usize;
self.num_exits = spec.max_voluntary_exits as usize;
self.num_transfers = spec.max_transfers as usize;
}
pub fn set_slot(&mut self, slot: Slot, spec: &ChainSpec) {
self.state_builder.teleport_to_slot(slot, &spec);
}
pub fn build_caches(&mut self, spec: &ChainSpec) {
// Builds all caches; benches will not contain shuffling/committee building times.
self.state_builder.build_caches(&spec).unwrap();
}
pub fn build(mut self, spec: &ChainSpec) -> (BeaconBlock, BeaconState) {
let (mut state, keypairs) = self.state_builder.build();
let builder = &mut self.block_builder;
builder.set_slot(state.slot);
let proposer_index = state.get_beacon_proposer_index(state.slot, spec).unwrap();
let keypair = &keypairs[proposer_index];
builder.set_randao_reveal(&keypair.sk, &state.fork, spec);
// Used as a stream of validator indices for use in slashings, exits, etc.
let mut validators_iter = (0..keypairs.len() as u64).into_iter();
// Insert `ProposerSlashing` objects.
for _ in 0..self.num_proposer_slashings {
let validator_index = validators_iter.next().expect("Insufficient validators.");
builder.insert_proposer_slashing(
validator_index,
&keypairs[validator_index as usize].sk,
&state.fork,
spec,
);
}
info!(
"Inserted {} proposer slashings.",
builder.block.body.proposer_slashings.len()
);
// Insert `AttesterSlashing` objects
for _ in 0..self.num_attester_slashings {
let mut attesters: Vec<u64> = vec![];
let mut secret_keys: Vec<&SecretKey> = vec![];
for _ in 0..self.num_indices_per_slashable_vote {
let validator_index = validators_iter.next().expect("Insufficient validators.");
attesters.push(validator_index);
secret_keys.push(&keypairs[validator_index as usize].sk);
}
builder.insert_attester_slashing(&attesters, &secret_keys, &state.fork, spec);
}
info!(
"Inserted {} attester slashings.",
builder.block.body.attester_slashings.len()
);
// Insert `Attestation` objects.
let all_secret_keys: Vec<&SecretKey> = keypairs.iter().map(|keypair| &keypair.sk).collect();
builder
.insert_attestations(
&state,
&all_secret_keys,
self.num_attestations as usize,
spec,
)
.unwrap();
info!(
"Inserted {} attestations.",
builder.block.body.attestations.len()
);
// Insert `Deposit` objects.
for i in 0..self.num_deposits {
builder.insert_deposit(
32_000_000_000,
state.deposit_index + (i as u64),
&state,
spec,
);
}
info!("Inserted {} deposits.", builder.block.body.deposits.len());
// Insert the maximum possible number of `Exit` objects.
for _ in 0..self.num_exits {
let validator_index = validators_iter.next().expect("Insufficient validators.");
builder.insert_exit(
&state,
validator_index,
&keypairs[validator_index as usize].sk,
spec,
);
}
info!(
"Inserted {} exits.",
builder.block.body.voluntary_exits.len()
);
// Insert the maximum possible number of `Transfer` objects.
for _ in 0..self.num_transfers {
let validator_index = validators_iter.next().expect("Insufficient validators.");
// Manually set the validator to be withdrawn.
state.validator_registry[validator_index as usize].withdrawable_epoch =
state.previous_epoch(spec);
builder.insert_transfer(
&state,
validator_index,
validator_index,
1,
keypairs[validator_index as usize].clone(),
spec,
);
}
info!("Inserted {} transfers.", builder.block.body.transfers.len());
let mut block = self.block_builder.build(&keypair.sk, &state.fork, spec);
// Set the eth1 data to be different from the state.
block.eth1_data.block_hash = Hash256::from_slice(&vec![42; 32]);
(block, state)
}
}

2
eth2/types/src/test_utils/testing_beacon_block_builder.rs
View File

@ -12,7 +12,7 @@ use ssz::{SignedRoot, TreeHash};
///
/// This struct should **never be used for production purposes.**
pub struct TestingBeaconBlockBuilder {
block: BeaconBlock,
pub block: BeaconBlock,
}
impl TestingBeaconBlockBuilder {