cerc-io/lighthouse
16
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Ensure drop times aren't included in benchmarks

Browse Source 
Also moves to the new `iter_batched` method on criterion (instead of
`iter_with_setup`.
This commit is contained in:
Paul Hauner 2019-03-11 10:56:31 +11:00
parent a44d80006a
commit 6ae99a1462
No known key found for this signature in database
GPG Key ID: D362883A9218FCC6
2 changed files with 142 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
eth2/state_processing/benches/block_processing_benches.rs
View File

@ -16,7 +16,7 @@ use types::*;
pub fn block_processing_16k_validators(c: &mut Criterion) { pub fn block_processing_16k_validators(c: &mut Criterion) {
let spec = ChainSpec::foundation(); let spec = ChainSpec::foundation();
let validator_count = 16_384; let validator_count = 300_032;
let (mut state, keypairs) = build_state(validator_count, &spec); let (mut state, keypairs) = build_state(validator_count, &spec);
let block = build_block(&mut state, &keypairs, &spec); let block = build_block(&mut state, &keypairs, &spec);
@ -199,9 +199,13 @@ fn bench_block_processing(
c.bench( c.bench(
&format!("block_processing_{}", desc), &format!("block_processing_{}", desc),
Benchmark::new("verify_block_signature", move |b| { Benchmark::new("verify_block_signature", move |b| {
b.iter_with_setup( b.iter_batched(
|| state.clone(), || state.clone(),
|mut state| black_box(verify_block_signature(&mut state, &block, &spec).unwrap()), |mut state| {
verify_block_signature(&mut state, &block, &spec).unwrap();
state
},
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(10), .sample_size(10),
@ -213,9 +217,13 @@ fn bench_block_processing(
c.bench( c.bench(
&format!("block_processing_{}", desc), &format!("block_processing_{}", desc),
Benchmark::new("process_randao", move |b| { Benchmark::new("process_randao", move |b| {
b.iter_with_setup( b.iter_batched(
|| state.clone(), || state.clone(),
|mut state| black_box(process_randao(&mut state, &block, &spec).unwrap()), |mut state| {
process_randao(&mut state, &block, &spec).unwrap();
state
},
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(10), .sample_size(10),
@ -226,9 +234,13 @@ fn bench_block_processing(
c.bench( c.bench(
&format!("block_processing_{}", desc), &format!("block_processing_{}", desc),
Benchmark::new("process_eth1_data", move |b| { Benchmark::new("process_eth1_data", move |b| {
b.iter_with_setup( b.iter_batched(
|| state.clone(), || state.clone(),
|mut state| black_box(process_eth1_data(&mut state, &block.eth1_data).unwrap()), |mut state| {
process_eth1_data(&mut state, &block.eth1_data).unwrap();
state
},
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(10), .sample_size(10),
@ -240,18 +252,14 @@ fn bench_block_processing(
c.bench( c.bench(
&format!("block_processing_{}", desc), &format!("block_processing_{}", desc),
Benchmark::new("process_proposer_slashings", move |b| { Benchmark::new("process_proposer_slashings", move |b| {
b.iter_with_setup( b.iter_batched(
|| state.clone(), || state.clone(),
|mut state| { |mut state| {
black_box( process_proposer_slashings(&mut state, &block.body.proposer_slashings, &spec)
process_proposer_slashings( .unwrap();
&mut state, state
&block.body.proposer_slashings,
&spec,
)
.unwrap(),
)
}, },
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(10), .sample_size(10),
@ -263,18 +271,14 @@ fn bench_block_processing(
c.bench( c.bench(
&format!("block_processing_{}", desc), &format!("block_processing_{}", desc),
Benchmark::new("process_attester_slashings", move |b| { Benchmark::new("process_attester_slashings", move |b| {
b.iter_with_setup( b.iter_batched(
|| state.clone(), || state.clone(),
|mut state| { |mut state| {
black_box( process_attester_slashings(&mut state, &block.body.attester_slashings, &spec)
process_attester_slashings( .unwrap();
&mut state, state
&block.body.attester_slashings,
&spec,
)
.unwrap(),
)
}, },
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(10), .sample_size(10),
@ -286,13 +290,13 @@ fn bench_block_processing(
c.bench( c.bench(
&format!("block_processing_{}", desc), &format!("block_processing_{}", desc),
Benchmark::new("process_attestations", move |b| { Benchmark::new("process_attestations", move |b| {
b.iter_with_setup( b.iter_batched(
|| state.clone(), || state.clone(),
|mut state| { |mut state| {
black_box( process_attestations(&mut state, &block.body.attestations, &spec).unwrap();
process_attestations(&mut state, &block.body.attestations, &spec).unwrap(), state
)
}, },
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(10), .sample_size(10),
@ -304,11 +308,13 @@ fn bench_block_processing(
c.bench( c.bench(
&format!("block_processing_{}", desc), &format!("block_processing_{}", desc),
Benchmark::new("process_deposits", move |b| { Benchmark::new("process_deposits", move |b| {
b.iter_with_setup( b.iter_batched(
|| state.clone(), || state.clone(),
|mut state| { |mut state| {
black_box(process_deposits(&mut state, &block.body.deposits, &spec).unwrap()) process_deposits(&mut state, &block.body.deposits, &spec).unwrap();
state
}, },
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(10), .sample_size(10),
@ -320,13 +326,13 @@ fn bench_block_processing(
c.bench( c.bench(
&format!("block_processing_{}", desc), &format!("block_processing_{}", desc),
Benchmark::new("process_exits", move |b| { Benchmark::new("process_exits", move |b| {
b.iter_with_setup( b.iter_batched(
|| state.clone(), || state.clone(),
|mut state| { |mut state| {
black_box( process_exits(&mut state, &block.body.voluntary_exits, &spec).unwrap();
process_exits(&mut state, &block.body.voluntary_exits, &spec).unwrap(), state
)
}, },
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(10), .sample_size(10),
@ -338,11 +344,13 @@ fn bench_block_processing(
c.bench( c.bench(
&format!("block_processing_{}", desc), &format!("block_processing_{}", desc),
Benchmark::new("process_transfers", move |b| { Benchmark::new("process_transfers", move |b| {
b.iter_with_setup( b.iter_batched(
|| state.clone(), || state.clone(),
|mut state| { |mut state| {
black_box(process_transfers(&mut state, &block.body.transfers, &spec).unwrap()) process_transfers(&mut state, &block.body.transfers, &spec).unwrap();
state
}, },
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(10), .sample_size(10),
@ -354,9 +362,13 @@ fn bench_block_processing(
c.bench( c.bench(
&format!("block_processing_{}", desc), &format!("block_processing_{}", desc),
Benchmark::new("per_block_processing", move |b| { Benchmark::new("per_block_processing", move |b| {
b.iter_with_setup( b.iter_batched(
|| state.clone(), || state.clone(),
|mut state| black_box(per_block_processing(&mut state, &block, &spec).unwrap()), |mut state| {
per_block_processing(&mut state, &block, &spec).unwrap();
state
},
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(10), .sample_size(10),
@ -368,15 +380,15 @@ fn bench_block_processing(
c.bench( c.bench(
&format!("block_processing_{}", desc), &format!("block_processing_{}", desc),
Benchmark::new("build_previous_state_epoch_cache", move |b| { Benchmark::new("build_previous_state_epoch_cache", move |b| {
b.iter_with_setup( b.iter_batched(
|| state.clone(), || state.clone(),
|mut state| { |mut state| {
black_box( state
state .build_epoch_cache(RelativeEpoch::Previous, &spec)
.build_epoch_cache(RelativeEpoch::Previous, &spec) .unwrap();
.unwrap(), state
)
}, },
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(10), .sample_size(10),
@ -388,15 +400,15 @@ fn bench_block_processing(
c.bench( c.bench(
&format!("block_processing_{}", desc), &format!("block_processing_{}", desc),
Benchmark::new("build_current_state_epoch_cache", move |b| { Benchmark::new("build_current_state_epoch_cache", move |b| {
b.iter_with_setup( b.iter_batched(
|| state.clone(), || state.clone(),
|mut state| { |mut state| {
black_box( state
state .build_epoch_cache(RelativeEpoch::Current, &spec)
.build_epoch_cache(RelativeEpoch::Current, &spec) .unwrap();
.unwrap(), state
)
}, },
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(10), .sample_size(10),

123
eth2/state_processing/benches/epoch_processing_benches.rs
View File

@ -13,14 +13,14 @@ use state_processing::{
}; };
use types::{validator_registry::get_active_validator_indices, *}; use types::{validator_registry::get_active_validator_indices, *};
pub const BENCHING_SAMPLE_SIZE: usize = 100; pub const BENCHING_SAMPLE_SIZE: usize = 10;
pub const SMALL_BENCHING_SAMPLE_SIZE: usize = 10; pub const SMALL_BENCHING_SAMPLE_SIZE: usize = 10;
/// Run the benchmarking suite on a foundation spec with 16,384 validators. /// Run the benchmarking suite on a foundation spec with 16,384 validators.
pub fn epoch_processing_16k_validators(c: &mut Criterion) { pub fn epoch_processing_16k_validators(c: &mut Criterion) {
let spec = ChainSpec::foundation(); let spec = ChainSpec::foundation();
let validator_count = 16_384; let validator_count = 300_032;
let mut builder = BeaconStateBencher::new(validator_count, &spec); let mut builder = BeaconStateBencher::new(validator_count, &spec);
@ -67,7 +67,7 @@ pub fn epoch_processing_16k_validators(c: &mut Criterion) {
"Epochs since finality should be 4" "Epochs since finality should be 4"
); );
bench_epoch_processing(c, &state, &spec, "16k_validators"); bench_epoch_processing(c, &state, &spec, &format!("{}_validators", validator_count));
} }
/// Run the detailed benchmarking suite on the given `BeaconState`. /// Run the detailed benchmarking suite on the given `BeaconState`.
@ -79,9 +79,13 @@ fn bench_epoch_processing(c: &mut Criterion, state: &BeaconState, spec: &ChainSp
c.bench( c.bench(
&format!("epoch_process_with_caches_{}", desc), &format!("epoch_process_with_caches_{}", desc),
Benchmark::new("calculate_active_validator_indices", move |b| { Benchmark::new("calculate_active_validator_indices", move |b| {
b.iter_with_setup( b.iter_batched(
|| state_clone.clone(), || state_clone.clone(),
|mut state| black_box(calculate_active_validator_indices(&mut state, &spec_clone)), |mut state| {
calculate_active_validator_indices(&mut state, &spec_clone);
state
},
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(BENCHING_SAMPLE_SIZE), .sample_size(BENCHING_SAMPLE_SIZE),
@ -93,11 +97,13 @@ fn bench_epoch_processing(c: &mut Criterion, state: &BeaconState, spec: &ChainSp
c.bench( c.bench(
&format!("epoch_process_with_caches_{}", desc), &format!("epoch_process_with_caches_{}", desc),
Benchmark::new("calculate_current_total_balance", move |b| { Benchmark::new("calculate_current_total_balance", move |b| {
b.iter_with_setup( b.iter_batched(
|| state_clone.clone(), || state_clone.clone(),
|state| { |state| {
black_box(state.get_total_balance(&active_validator_indices[..], &spec_clone)) state.get_total_balance(&active_validator_indices[..], &spec_clone);
state
}, },
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(BENCHING_SAMPLE_SIZE), .sample_size(BENCHING_SAMPLE_SIZE),
@ -108,17 +114,19 @@ fn bench_epoch_processing(c: &mut Criterion, state: &BeaconState, spec: &ChainSp
c.bench( c.bench(
&format!("epoch_process_with_caches_{}", desc), &format!("epoch_process_with_caches_{}", desc),
Benchmark::new("calculate_previous_total_balance", move |b| { Benchmark::new("calculate_previous_total_balance", move |b| {
b.iter_with_setup( b.iter_batched(
|| state_clone.clone(), || state_clone.clone(),
|state| { |state| {
black_box(state.get_total_balance( state.get_total_balance(
&get_active_validator_indices( &get_active_validator_indices(
&state.validator_registry, &state.validator_registry,
state.previous_epoch(&spec_clone), state.previous_epoch(&spec_clone),
)[..], )[..],
&spec_clone, &spec_clone,
)) );
state
}, },
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(BENCHING_SAMPLE_SIZE), .sample_size(BENCHING_SAMPLE_SIZE),
@ -129,9 +137,13 @@ fn bench_epoch_processing(c: &mut Criterion, state: &BeaconState, spec: &ChainSp
c.bench( c.bench(
&format!("epoch_process_with_caches_{}", desc), &format!("epoch_process_with_caches_{}", desc),
Benchmark::new("process_eth1_data", move |b| { Benchmark::new("process_eth1_data", move |b| {
b.iter_with_setup( b.iter_batched(
|| state_clone.clone(), || state_clone.clone(),
|mut state| black_box(process_eth1_data(&mut state, &spec_clone)), |mut state| {
process_eth1_data(&mut state, &spec_clone);
state
},
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(BENCHING_SAMPLE_SIZE), .sample_size(BENCHING_SAMPLE_SIZE),
@ -142,9 +154,13 @@ fn bench_epoch_processing(c: &mut Criterion, state: &BeaconState, spec: &ChainSp
c.bench( c.bench(
&format!("epoch_process_with_caches_{}", desc), &format!("epoch_process_with_caches_{}", desc),
Benchmark::new("calculate_attester_sets", move |b| { Benchmark::new("calculate_attester_sets", move |b| {
b.iter_with_setup( b.iter_batched(
|| state_clone.clone(), || state_clone.clone(),
|mut state| black_box(calculate_attester_sets(&mut state, &spec_clone).unwrap()), |mut state| {
calculate_attester_sets(&mut state, &spec_clone).unwrap();
state
},
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(BENCHING_SAMPLE_SIZE), .sample_size(BENCHING_SAMPLE_SIZE),
@ -163,18 +179,20 @@ fn bench_epoch_processing(c: &mut Criterion, state: &BeaconState, spec: &ChainSp
c.bench( c.bench(
&format!("epoch_process_with_caches_{}", desc), &format!("epoch_process_with_caches_{}", desc),
Benchmark::new("process_justification", move |b| { Benchmark::new("process_justification", move |b| {
b.iter_with_setup( b.iter_batched(
|| state_clone.clone(), || state_clone.clone(),
|mut state| { |mut state| {
black_box(process_justification( process_justification(
&mut state, &mut state,
current_total_balance, current_total_balance,
previous_total_balance, previous_total_balance,
attesters.previous_epoch_boundary.balance, attesters.previous_epoch_boundary.balance,
attesters.current_epoch_boundary.balance, attesters.current_epoch_boundary.balance,
&spec_clone, &spec_clone,
)) );
state
}, },
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(10), .sample_size(10),
@ -185,9 +203,10 @@ fn bench_epoch_processing(c: &mut Criterion, state: &BeaconState, spec: &ChainSp
c.bench( c.bench(
&format!("epoch_process_with_caches_{}", desc), &format!("epoch_process_with_caches_{}", desc),
Benchmark::new("process_crosslinks", move |b| { Benchmark::new("process_crosslinks", move |b| {
b.iter_with_setup( b.iter_batched(
|| state_clone.clone(), || state_clone.clone(),
|mut state| black_box(process_crosslinks(&mut state, &spec_clone).unwrap()), |mut state| black_box(process_crosslinks(&mut state, &spec_clone).unwrap()),
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(BENCHING_SAMPLE_SIZE), .sample_size(BENCHING_SAMPLE_SIZE),
@ -206,21 +225,21 @@ fn bench_epoch_processing(c: &mut Criterion, state: &BeaconState, spec: &ChainSp
c.bench( c.bench(
&format!("epoch_process_with_caches_{}", desc), &format!("epoch_process_with_caches_{}", desc),
Benchmark::new("process_rewards_and_penalties", move |b| { Benchmark::new("process_rewards_and_penalties", move |b| {
b.iter_with_setup( b.iter_batched(
|| state_clone.clone(), || state_clone.clone(),
|mut state| { |mut state| {
black_box( process_rewards_and_penalities(
process_rewards_and_penalities( &mut state,
&mut state, &active_validator_indices,
&active_validator_indices, &attesters,
&attesters, previous_total_balance,
previous_total_balance, &winning_root_for_shards,
&winning_root_for_shards, &spec_clone,
&spec_clone,
)
.unwrap(),
) )
.unwrap();
state
}, },
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(SMALL_BENCHING_SAMPLE_SIZE), .sample_size(SMALL_BENCHING_SAMPLE_SIZE),
@ -231,9 +250,13 @@ fn bench_epoch_processing(c: &mut Criterion, state: &BeaconState, spec: &ChainSp
c.bench( c.bench(
&format!("epoch_process_with_caches_{}", desc), &format!("epoch_process_with_caches_{}", desc),
Benchmark::new("process_ejections", move |b| { Benchmark::new("process_ejections", move |b| {
b.iter_with_setup( b.iter_batched(
|| state_clone.clone(), || state_clone.clone(),
|mut state| black_box(state.process_ejections(&spec_clone)), |mut state| {
state.process_ejections(&spec_clone);
state
},
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(BENCHING_SAMPLE_SIZE), .sample_size(BENCHING_SAMPLE_SIZE),
@ -268,9 +291,13 @@ fn bench_epoch_processing(c: &mut Criterion, state: &BeaconState, spec: &ChainSp
c.bench( c.bench(
&format!("epoch_process_with_caches_{}", desc), &format!("epoch_process_with_caches_{}", desc),
Benchmark::new("process_validator_registry", move |b| { Benchmark::new("process_validator_registry", move |b| {
b.iter_with_setup( b.iter_batched(
|| state_clone.clone(), || state_clone.clone(),
|mut state| black_box(process_validator_registry(&mut state, &spec_clone)), |mut state| {
process_validator_registry(&mut state, &spec_clone).unwrap();
state
},
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(BENCHING_SAMPLE_SIZE), .sample_size(BENCHING_SAMPLE_SIZE),
@ -281,11 +308,13 @@ fn bench_epoch_processing(c: &mut Criterion, state: &BeaconState, spec: &ChainSp
c.bench( c.bench(
&format!("epoch_process_with_caches_{}", desc), &format!("epoch_process_with_caches_{}", desc),
Benchmark::new("update_active_tree_index_roots", move |b| { Benchmark::new("update_active_tree_index_roots", move |b| {
b.iter_with_setup( b.iter_batched(
|| state_clone.clone(), || state_clone.clone(),
|mut state| { |mut state| {
black_box(update_active_tree_index_roots(&mut state, &spec_clone).unwrap()) update_active_tree_index_roots(&mut state, &spec_clone).unwrap();
state
}, },
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(BENCHING_SAMPLE_SIZE), .sample_size(BENCHING_SAMPLE_SIZE),
@ -296,9 +325,13 @@ fn bench_epoch_processing(c: &mut Criterion, state: &BeaconState, spec: &ChainSp
c.bench( c.bench(
&format!("epoch_process_with_caches_{}", desc), &format!("epoch_process_with_caches_{}", desc),
Benchmark::new("update_latest_slashed_balances", move |b| { Benchmark::new("update_latest_slashed_balances", move |b| {
b.iter_with_setup( b.iter_batched(
|| state_clone.clone(), || state_clone.clone(),
|mut state| black_box(update_latest_slashed_balances(&mut state, &spec_clone)), |mut state| {
update_latest_slashed_balances(&mut state, &spec_clone);
state
},
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(BENCHING_SAMPLE_SIZE), .sample_size(BENCHING_SAMPLE_SIZE),
@ -309,9 +342,13 @@ fn bench_epoch_processing(c: &mut Criterion, state: &BeaconState, spec: &ChainSp
c.bench( c.bench(
&format!("epoch_process_with_caches_{}", desc), &format!("epoch_process_with_caches_{}", desc),
Benchmark::new("clean_attestations", move |b| { Benchmark::new("clean_attestations", move |b| {
b.iter_with_setup( b.iter_batched(
|| state_clone.clone(), || state_clone.clone(),
|mut state| black_box(clean_attestations(&mut state, &spec_clone)), |mut state| {
clean_attestations(&mut state, &spec_clone);
state
},
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(BENCHING_SAMPLE_SIZE), .sample_size(BENCHING_SAMPLE_SIZE),
@ -322,9 +359,10 @@ fn bench_epoch_processing(c: &mut Criterion, state: &BeaconState, spec: &ChainSp
c.bench( c.bench(
&format!("epoch_process_with_caches_{}", desc), &format!("epoch_process_with_caches_{}", desc),
Benchmark::new("per_epoch_processing", move |b| { Benchmark::new("per_epoch_processing", move |b| {
b.iter_with_setup( b.iter_batched(
|| state_clone.clone(), || state_clone.clone(),
|mut state| black_box(per_epoch_processing(&mut state, &spec_clone).unwrap()), |mut state| black_box(per_epoch_processing(&mut state, &spec_clone).unwrap()),
criterion::BatchSize::SmallInput,
) )
}) })
.sample_size(SMALL_BENCHING_SAMPLE_SIZE), .sample_size(SMALL_BENCHING_SAMPLE_SIZE),
@ -334,10 +372,7 @@ fn bench_epoch_processing(c: &mut Criterion, state: &BeaconState, spec: &ChainSp
c.bench( c.bench(
&format!("epoch_process_with_caches_{}", desc), &format!("epoch_process_with_caches_{}", desc),
Benchmark::new("tree_hash_state", move |b| { Benchmark::new("tree_hash_state", move |b| {
b.iter_with_setup( b.iter(|| black_box(state_clone.hash_tree_root()))
|| state_clone.clone(),
|state| black_box(state.hash_tree_root()),
)
}) })
.sample_size(SMALL_BENCHING_SAMPLE_SIZE), .sample_size(SMALL_BENCHING_SAMPLE_SIZE),
); );