diff --git a/eth2/state_processing/benches/bench_block_processing.rs b/eth2/state_processing/benches/bench_block_processing.rs index a315717b2..2ee08c96a 100644 --- a/eth2/state_processing/benches/bench_block_processing.rs +++ b/eth2/state_processing/benches/bench_block_processing.rs @@ -1,6 +1,5 @@ use criterion::Criterion; use criterion::{black_box, Benchmark}; -use log::debug; use ssz::TreeHash; use state_processing::{ per_block_processing, @@ -10,252 +9,12 @@ use state_processing::{ verify_block_signature, }, }; -use types::test_utils::{TestingBeaconBlockBuilder, TestingBeaconStateBuilder}; use types::*; -/// Run the benchmarking suite on a foundation spec with 16,384 validators. -pub fn bench_block_processing_n_validators(c: &mut Criterion, validator_count: usize) { - let spec = ChainSpec::foundation(); - - let bench_builder = BlockBenchingBuilder::new(validator_count, &spec); - - let (mut state, keypairs) = build_state(validator_count, &spec); - let block = build_block(&mut state, &keypairs, &spec); - - assert_eq!( - block.body.proposer_slashings.len(), - spec.max_proposer_slashings as usize, - "The block should have the maximum possible proposer slashings" - ); - - assert_eq!( - block.body.attester_slashings.len(), - spec.max_attester_slashings as usize, - "The block should have the maximum possible attester slashings" - ); - - for attester_slashing in &block.body.attester_slashings { - let len_1 = attester_slashing - .slashable_attestation_1 - .validator_indices - .len(); - let len_2 = attester_slashing - .slashable_attestation_1 - .validator_indices - .len(); - assert!( - (len_1 == len_2) && (len_2 == spec.max_indices_per_slashable_vote as usize), - "Each attester slashing should have the maximum possible validator indices" - ); - } - - assert_eq!( - block.body.attestations.len(), - spec.max_attestations as usize, - "The block should have the maximum possible attestations." - ); - - assert_eq!( - block.body.deposits.len(), - spec.max_deposits as usize, - "The block should have the maximum possible deposits." - ); - - assert_eq!( - block.body.voluntary_exits.len(), - spec.max_voluntary_exits as usize, - "The block should have the maximum possible voluntary exits." - ); - - assert_eq!( - block.body.transfers.len(), - spec.max_transfers as usize, - "The block should have the maximum possible transfers." - ); - - bench_block_processing( - c, - &block, - &state, - &spec, - &format!("{}_validators", validator_count), - ); -} - -fn build_state(validator_count: usize, spec: &ChainSpec) -> (BeaconState, Vec) { - let mut builder = - TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, &spec); - - // Set the state to be just before an epoch transition. - let target_slot = (spec.genesis_epoch + 4).end_slot(spec.slots_per_epoch); - builder.teleport_to_slot(target_slot, &spec); - - // Builds all caches; benches will not contain shuffling/committee building times. - builder.build_caches(&spec).unwrap(); - - builder.build() -} - -pub struct BlockBenchingBuilder { - pub state_builder: TestingBeaconStateBuilder, - pub block_builder: TestingBeaconBlockBuilder, - - pub num_validators: usize, - pub num_proposer_slashings: usize, - pub num_attester_slashings: usize, - pub num_indices_per_slashable_vote: usize, - pub num_attestations: usize, - pub num_deposits: usize, - pub num_exits: usize, - pub num_transfers: usize, -} - -impl BlockBenchingBuilder { - pub fn new(num_validators: usize, spec: &ChainSpec) -> Self { - let mut state_builder = - TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(num_validators, &spec); - let mut block_builder = TestingBeaconBlockBuilder::new(spec); - - Self { - state_builder, - block_builder, - num_validators: 0, - num_proposer_slashings: 0, - num_attester_slashings: 0, - num_indices_per_slashable_vote: spec.max_indices_per_slashable_vote as usize, - num_attestations: 0, - num_deposits: 0, - num_exits: 0, - num_transfers: 0 - } - } - - pub fn maximize_block_operations(&mut self, spec: &ChainSpec) { - self.num_proposer_slashings = spec.max_proposer_slashings as usize; - self.num_attester_slashings = spec.max_attester_slashings as usize; - self.num_indices_per_slashable_vote = spec.max_indices_per_slashable_vote as usize; - self.num_attestations = spec.max_attestations as usize; - self.num_deposits = spec.max_deposits as usize; - self.num_exits = spec.max_voluntary_exits as usize; - self.num_transfers = spec.max_transfers as usize; - } - - pub fn set_slot(&mut self, slot: Slot, spec: &ChainSpec) { - self.state_builder.teleport_to_slot(slot, &spec); - } - - pub fn build_caches(&mut self, spec: &ChainSpec) { - // Builds all caches; benches will not contain shuffling/committee building times. - self.state_builder.build_caches(&spec).unwrap(); - } - - pub fn build(self, spec: &ChainSpec) -> (BeaconBlock, BeaconState) { - let (mut state, keypairs) = self.state_builder.build(); - let builder = &mut self.block_builder; - - builder.set_slot(state.slot); - - let proposer_index = state.get_beacon_proposer_index(state.slot, spec).unwrap(); - let keypair = &keypairs[proposer_index]; - - builder.set_randao_reveal(&keypair.sk, &state.fork, spec); - - // Used as a stream of validator indices for use in slashings, exits, etc. - let mut validators_iter = (0..keypairs.len() as u64).into_iter(); - - // Insert `ProposerSlashing` objects. - debug!( - "Inserting {} proposer slashings...", - self.num_proposer_slashings - ); - for _ in 0..self.num_proposer_slashings { - let validator_index = validators_iter.next().expect("Insufficient validators."); - - builder.insert_proposer_slashing( - validator_index, - &keypairs[validator_index as usize].sk, - &state.fork, - spec, - ); - } - - // Insert `AttesterSlashing` objects - debug!( - "Inserting {} attester slashings...", - self.num_attester_slashings - ); - for _ in 0..self.num_attester_slashings { - let mut attesters: Vec = vec![]; - let mut secret_keys: Vec<&SecretKey> = vec![]; - - for _ in 0..self.num_indices_per_slashable_vote { - let validator_index = validators_iter.next().expect("Insufficient validators."); - - attesters.push(validator_index); - secret_keys.push(&keypairs[validator_index as usize].sk); - } - - builder.insert_attester_slashing(&attesters, &secret_keys, &state.fork, spec); - } - - // Insert `Attestation` objects. - debug!("Inserting {} attestations...", self.num_attestations); - let all_secret_keys: Vec<&SecretKey> = keypairs.iter().map(|keypair| &keypair.sk).collect(); - builder - .insert_attestations(&state, &all_secret_keys, self.num_attestations as usize, spec) - .unwrap(); - - // Insert `Deposit` objects. - debug!("Inserting {} deposits...", self.num_deposits); - for i in 0..self.num_deposits { - builder.insert_deposit(32_000_000_000, state.deposit_index + (i as u64), &state, spec); - } - - // Insert the maximum possible number of `Exit` objects. - debug!("Inserting {} exits...", self.num_exits); - for _ in 0..self.num_exits { - let validator_index = validators_iter.next().expect("Insufficient validators."); - - builder.insert_exit( - &state, - validator_index, - &keypairs[validator_index as usize].sk, - spec, - ); - } - - // Insert the maximum possible number of `Transfer` objects. - debug!("Inserting {} transfers...", self.num_transfers); - for _ in 0..self.num_transfers { - let validator_index = validators_iter.next().expect("Insufficient validators."); - - // Manually set the validator to be withdrawn. - state.validator_registry[validator_index as usize].withdrawable_epoch = - state.previous_epoch(spec); - - builder.insert_transfer( - &state, - validator_index, - validator_index, - 1, - keypairs[validator_index as usize].clone(), - spec, - ); - } - - let mut block = builder.build(&keypair.sk, &state.fork, spec); - - // Set the eth1 data to be different from the state. - block.eth1_data.block_hash = Hash256::from_slice(&vec![42; 32]); - - (block, state) - } -} - /// Run the detailed benchmarking suite on the given `BeaconState`. /// /// `desc` will be added to the title of each bench. -fn bench_block_processing( +pub fn bench_block_processing( c: &mut Criterion, initial_block: &BeaconBlock, initial_state: &BeaconState, diff --git a/eth2/state_processing/benches/benches.rs b/eth2/state_processing/benches/benches.rs index af384b00a..685858c78 100644 --- a/eth2/state_processing/benches/benches.rs +++ b/eth2/state_processing/benches/benches.rs @@ -1,23 +1,107 @@ +use block_benching_builder::BlockBenchingBuilder; use criterion::Criterion; use criterion::{criterion_group, criterion_main}; use env_logger::{Builder, Env}; +use log::info; +use types::*; mod bench_block_processing; mod bench_epoch_processing; +mod block_benching_builder; pub const VALIDATOR_COUNT: usize = 16_384; // `LOG_LEVEL == "debug"` gives logs, but they're very noisy and slow down benching. -pub const LOG_LEVEL: &str = ""; +pub const LOG_LEVEL: &str = "info"; -pub fn state_processing(c: &mut Criterion) { +/// Build a worst-case block and benchmark processing it. +pub fn block_processing_worst_case(c: &mut Criterion) { if LOG_LEVEL != "" { Builder::from_env(Env::default().default_filter_or(LOG_LEVEL)).init(); } + info!( + "Building worst case block bench with {} validators", + VALIDATOR_COUNT + ); - bench_epoch_processing::bench_epoch_processing_n_validators(c, VALIDATOR_COUNT); - bench_block_processing::bench_block_processing_n_validators(c, VALIDATOR_COUNT); + // Use the specifications from the Eth2.0 spec. + let spec = ChainSpec::foundation(); + + // Create a builder for configuring the block and state for benching. + let mut bench_builder = BlockBenchingBuilder::new(VALIDATOR_COUNT, &spec); + + // Set the number of included operations to be maximum (e.g., `MAX_ATTESTATIONS`, etc.) + bench_builder.maximize_block_operations(&spec); + + // Set the state and block to be in the last slot of the 4th epoch. + let last_slot_of_epoch = (spec.genesis_epoch + 4).end_slot(spec.slots_per_epoch); + bench_builder.set_slot(last_slot_of_epoch, &spec); + + // Build all the state caches so the build times aren't included in the benches. + bench_builder.build_caches(&spec); + + // Generate the block and state. + let (block, state) = bench_builder.build(&spec); + + // Run the benches. + bench_block_processing::bench_block_processing( + c, + &block, + &state, + &spec, + &format!("{}_validators/worst_case", VALIDATOR_COUNT), + ); } -criterion_group!(benches, state_processing); +/// Build a reasonable-case block and benchmark processing it. +pub fn block_processing_reasonable_case(c: &mut Criterion) { + info!( + "Building reasonable case block bench with {} validators", + VALIDATOR_COUNT + ); + + // Use the specifications from the Eth2.0 spec. + let spec = ChainSpec::foundation(); + + // Create a builder for configuring the block and state for benching. + let mut bench_builder = BlockBenchingBuilder::new(VALIDATOR_COUNT, &spec); + + // Set the number of included operations to what we might expect normally. + bench_builder.num_proposer_slashings = 0; + bench_builder.num_attester_slashings = 0; + bench_builder.num_attestations = (spec.shard_count / spec.slots_per_epoch) as usize; + bench_builder.num_deposits = 2; + bench_builder.num_exits = 2; + bench_builder.num_transfers = 2; + + // Set the state and block to be in the last slot of the 4th epoch. + let last_slot_of_epoch = (spec.genesis_epoch + 4).end_slot(spec.slots_per_epoch); + bench_builder.set_slot(last_slot_of_epoch, &spec); + + // Build all the state caches so the build times aren't included in the benches. + bench_builder.build_caches(&spec); + + // Generate the block and state. + let (block, state) = bench_builder.build(&spec); + + // Run the benches. + bench_block_processing::bench_block_processing( + c, + &block, + &state, + &spec, + &format!("{}_validators/reasonable_case", VALIDATOR_COUNT), + ); +} + +pub fn state_processing(c: &mut Criterion) { + bench_epoch_processing::bench_epoch_processing_n_validators(c, VALIDATOR_COUNT); +} + +criterion_group!( + benches, + block_processing_reasonable_case, + block_processing_worst_case, + state_processing +); criterion_main!(benches); diff --git a/eth2/state_processing/benches/block_benching_builder.rs b/eth2/state_processing/benches/block_benching_builder.rs new file mode 100644 index 000000000..b993851d7 --- /dev/null +++ b/eth2/state_processing/benches/block_benching_builder.rs @@ -0,0 +1,175 @@ +use log::info; +use types::test_utils::{TestingBeaconBlockBuilder, TestingBeaconStateBuilder}; +use types::*; + +pub struct BlockBenchingBuilder { + pub state_builder: TestingBeaconStateBuilder, + pub block_builder: TestingBeaconBlockBuilder, + + pub num_validators: usize, + pub num_proposer_slashings: usize, + pub num_attester_slashings: usize, + pub num_indices_per_slashable_vote: usize, + pub num_attestations: usize, + pub num_deposits: usize, + pub num_exits: usize, + pub num_transfers: usize, +} + +impl BlockBenchingBuilder { + pub fn new(num_validators: usize, spec: &ChainSpec) -> Self { + let state_builder = + TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(num_validators, &spec); + let block_builder = TestingBeaconBlockBuilder::new(spec); + + Self { + state_builder, + block_builder, + num_validators: 0, + num_proposer_slashings: 0, + num_attester_slashings: 0, + num_indices_per_slashable_vote: spec.max_indices_per_slashable_vote as usize, + num_attestations: 0, + num_deposits: 0, + num_exits: 0, + num_transfers: 0, + } + } + + pub fn maximize_block_operations(&mut self, spec: &ChainSpec) { + self.num_proposer_slashings = spec.max_proposer_slashings as usize; + self.num_attester_slashings = spec.max_attester_slashings as usize; + self.num_indices_per_slashable_vote = spec.max_indices_per_slashable_vote as usize; + self.num_attestations = spec.max_attestations as usize; + self.num_deposits = spec.max_deposits as usize; + self.num_exits = spec.max_voluntary_exits as usize; + self.num_transfers = spec.max_transfers as usize; + } + + pub fn set_slot(&mut self, slot: Slot, spec: &ChainSpec) { + self.state_builder.teleport_to_slot(slot, &spec); + } + + pub fn build_caches(&mut self, spec: &ChainSpec) { + // Builds all caches; benches will not contain shuffling/committee building times. + self.state_builder.build_caches(&spec).unwrap(); + } + + pub fn build(mut self, spec: &ChainSpec) -> (BeaconBlock, BeaconState) { + let (mut state, keypairs) = self.state_builder.build(); + let builder = &mut self.block_builder; + + builder.set_slot(state.slot); + + let proposer_index = state.get_beacon_proposer_index(state.slot, spec).unwrap(); + let keypair = &keypairs[proposer_index]; + + builder.set_randao_reveal(&keypair.sk, &state.fork, spec); + + // Used as a stream of validator indices for use in slashings, exits, etc. + let mut validators_iter = (0..keypairs.len() as u64).into_iter(); + + // Insert `ProposerSlashing` objects. + for _ in 0..self.num_proposer_slashings { + let validator_index = validators_iter.next().expect("Insufficient validators."); + + builder.insert_proposer_slashing( + validator_index, + &keypairs[validator_index as usize].sk, + &state.fork, + spec, + ); + } + info!( + "Inserted {} proposer slashings.", + builder.block.body.proposer_slashings.len() + ); + + // Insert `AttesterSlashing` objects + for _ in 0..self.num_attester_slashings { + let mut attesters: Vec = vec![]; + let mut secret_keys: Vec<&SecretKey> = vec![]; + + for _ in 0..self.num_indices_per_slashable_vote { + let validator_index = validators_iter.next().expect("Insufficient validators."); + + attesters.push(validator_index); + secret_keys.push(&keypairs[validator_index as usize].sk); + } + + builder.insert_attester_slashing(&attesters, &secret_keys, &state.fork, spec); + } + info!( + "Inserted {} attester slashings.", + builder.block.body.attester_slashings.len() + ); + + // Insert `Attestation` objects. + let all_secret_keys: Vec<&SecretKey> = keypairs.iter().map(|keypair| &keypair.sk).collect(); + builder + .insert_attestations( + &state, + &all_secret_keys, + self.num_attestations as usize, + spec, + ) + .unwrap(); + info!( + "Inserted {} attestations.", + builder.block.body.attestations.len() + ); + + // Insert `Deposit` objects. + for i in 0..self.num_deposits { + builder.insert_deposit( + 32_000_000_000, + state.deposit_index + (i as u64), + &state, + spec, + ); + } + info!("Inserted {} deposits.", builder.block.body.deposits.len()); + + // Insert the maximum possible number of `Exit` objects. + for _ in 0..self.num_exits { + let validator_index = validators_iter.next().expect("Insufficient validators."); + + builder.insert_exit( + &state, + validator_index, + &keypairs[validator_index as usize].sk, + spec, + ); + } + info!( + "Inserted {} exits.", + builder.block.body.voluntary_exits.len() + ); + + // Insert the maximum possible number of `Transfer` objects. + for _ in 0..self.num_transfers { + let validator_index = validators_iter.next().expect("Insufficient validators."); + + // Manually set the validator to be withdrawn. + state.validator_registry[validator_index as usize].withdrawable_epoch = + state.previous_epoch(spec); + + builder.insert_transfer( + &state, + validator_index, + validator_index, + 1, + keypairs[validator_index as usize].clone(), + spec, + ); + } + info!("Inserted {} transfers.", builder.block.body.transfers.len()); + + let mut block = self.block_builder.build(&keypair.sk, &state.fork, spec); + + // Set the eth1 data to be different from the state. + block.eth1_data.block_hash = Hash256::from_slice(&vec![42; 32]); + + (block, state) + } +} diff --git a/eth2/types/src/test_utils/testing_beacon_block_builder.rs b/eth2/types/src/test_utils/testing_beacon_block_builder.rs index ecb42e27b..58633b5ce 100644 --- a/eth2/types/src/test_utils/testing_beacon_block_builder.rs +++ b/eth2/types/src/test_utils/testing_beacon_block_builder.rs @@ -12,7 +12,7 @@ use ssz::{SignedRoot, TreeHash}; /// /// This struct should **never be used for production purposes.** pub struct TestingBeaconBlockBuilder { - block: BeaconBlock, + pub block: BeaconBlock, } impl TestingBeaconBlockBuilder {