cerc-io/lighthouse
16
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Update benchmarks, add to CI (#988)

Browse Source
This commit is contained in:
Michael Sproul 2020-04-06 20:16:08 +10:00 committed by GitHub
parent 4cba745df6
commit 869b0621d6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 48 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.github/workflows/test-suite.yml vendored
View File

@ -58,3 +58,10 @@ jobs:
run: sudo npm install -g ganache-cli run: sudo npm install -g ganache-cli
- name: Run the beacon chain sim - name: Run the beacon chain sim
run: cargo run --release --bin simulator beacon-chain-sim run: cargo run --release --bin simulator beacon-chain-sim
check-benchmarks:
runs-on: ubuntu-latest
needs: cargo-fmt
steps:
- uses: actions/checkout@v1
- name: Typecheck benchmark code without running it
run: make check-benches

4
Makefile
View File

@ -22,6 +22,10 @@ test-debug:
cargo-fmt: cargo-fmt:
cargo fmt --all -- --check cargo fmt --all -- --check
# Typechecks benchmark code
check-benches:
cargo check --all --benches
# Runs only the ef-test vectors. # Runs only the ef-test vectors.
run-ef-tests: run-ef-tests:
cargo test --release --manifest-path=$(EF_TESTS)/Cargo.toml --features "ef_tests" cargo test --release --manifest-path=$(EF_TESTS)/Cargo.toml --features "ef_tests"

58
eth2/state_processing/benches/benches.rs
View File

@ -4,7 +4,9 @@ use criterion::Criterion;
use criterion::{black_box, criterion_group, criterion_main, Benchmark}; use criterion::{black_box, criterion_group, criterion_main, Benchmark};
use ssz::Encode; use ssz::Encode;
use state_processing::{test_utils::BlockBuilder, BlockSignatureStrategy, VerifySignatures}; use state_processing::{test_utils::BlockBuilder, BlockSignatureStrategy, VerifySignatures};
use types::{BeaconBlock, BeaconState, ChainSpec, EthSpec, MainnetEthSpec, MinimalEthSpec, Slot}; use types::{
BeaconState, ChainSpec, EthSpec, MainnetEthSpec, MinimalEthSpec, SignedBeaconBlock, Slot,
};
pub const VALIDATORS_LOW: usize = 32_768; pub const VALIDATORS_LOW: usize = 32_768;
pub const VALIDATORS_HIGH: usize = 300_032; pub const VALIDATORS_HIGH: usize = 300_032;
@ -45,7 +47,7 @@ fn worst_bench<T: EthSpec>(c: &mut Criterion, spec_desc: &str, validator_count:
fn get_average_block<T: EthSpec>( fn get_average_block<T: EthSpec>(
validator_count: usize, validator_count: usize,
spec: &ChainSpec, spec: &ChainSpec,
) -> (BeaconBlock<T>, BeaconState<T>) { ) -> (SignedBeaconBlock<T>, BeaconState<T>) {
let mut builder: BlockBuilder<T> = BlockBuilder::new(validator_count, &spec); let mut builder: BlockBuilder<T> = BlockBuilder::new(validator_count, &spec);
// builder.num_attestations = T::MaxAttestations::to_usize(); // builder.num_attestations = T::MaxAttestations::to_usize();
builder.num_attestations = 16; builder.num_attestations = 16;
@ -59,7 +61,7 @@ fn get_average_block<T: EthSpec>(
fn get_worst_block<T: EthSpec>( fn get_worst_block<T: EthSpec>(
validator_count: usize, validator_count: usize,
spec: &ChainSpec, spec: &ChainSpec,
) -> (BeaconBlock<T>, BeaconState<T>) { ) -> (SignedBeaconBlock<T>, BeaconState<T>) {
let mut builder: BlockBuilder<T> = BlockBuilder::new(validator_count, &spec); let mut builder: BlockBuilder<T> = BlockBuilder::new(validator_count, &spec);
builder.maximize_block_operations(); builder.maximize_block_operations();
@ -74,7 +76,7 @@ fn get_worst_block<T: EthSpec>(
#[allow(clippy::unit_arg)] #[allow(clippy::unit_arg)]
fn bench_block<T: EthSpec>( fn bench_block<T: EthSpec>(
c: &mut Criterion, c: &mut Criterion,
block: BeaconBlock<T>, block: SignedBeaconBlock<T>,
state: BeaconState<T>, state: BeaconState<T>,
spec: &ChainSpec, spec: &ChainSpec,
spec_desc: &str, spec_desc: &str,
@ -183,9 +185,7 @@ fn bench_block<T: EthSpec>(
black_box( black_box(
state_processing::per_block_processing::process_block_header::<T>( state_processing::per_block_processing::process_block_header::<T>(
state, state,
&block, &block.message,
None,
VerifySignatures::True,
&spec, &spec,
) )
.expect("process_block_header should succeed"), .expect("process_block_header should succeed"),
@ -231,7 +231,7 @@ fn bench_block<T: EthSpec>(
black_box( black_box(
state_processing::per_block_processing::process_attestations::<T>( state_processing::per_block_processing::process_attestations::<T>(
state, state,
&block.body.attestations, &block.message.body.attestations,
VerifySignatures::True, VerifySignatures::True,
&spec, &spec,
) )
@ -252,7 +252,7 @@ fn bench_block<T: EthSpec>(
Benchmark::new("verify_attestation", move |b| { Benchmark::new("verify_attestation", move |b| {
b.iter_batched_ref( b.iter_batched_ref(
|| { || {
let attestation = &local_block.body.attestations[0]; let attestation = &local_block.message.body.attestations[0];
(local_spec.clone(), local_state.clone(), attestation.clone()) (local_spec.clone(), local_state.clone(), attestation.clone())
}, },
@ -280,13 +280,15 @@ fn bench_block<T: EthSpec>(
Benchmark::new("get_indexed_attestation", move |b| { Benchmark::new("get_indexed_attestation", move |b| {
b.iter_batched_ref( b.iter_batched_ref(
|| { || {
let attestation = &local_block.body.attestations[0]; let attestation = &local_block.message.body.attestations[0];
let committee = local_state
(local_state.clone(), attestation.clone()) .get_beacon_committee(attestation.data.slot, attestation.data.index)
.unwrap();
(committee.committee, attestation.clone())
}, },
|(ref mut state, attestation)| { |(committee, attestation)| {
black_box( black_box(
state_processing::common::get_indexed_attestation(state, &attestation) state_processing::common::get_indexed_attestation(committee, &attestation)
.expect("should get indexed attestation"), .expect("should get indexed attestation"),
) )
}, },
@ -304,9 +306,12 @@ fn bench_block<T: EthSpec>(
Benchmark::new("is_valid_indexed_attestation_with_signature", move |b| { Benchmark::new("is_valid_indexed_attestation_with_signature", move |b| {
b.iter_batched_ref( b.iter_batched_ref(
|| { || {
let attestation = &local_block.body.attestations[0]; let attestation = &local_block.message.body.attestations[0];
let committee = local_state
.get_beacon_committee(attestation.data.slot, attestation.data.index)
.unwrap();
let indexed_attestation = state_processing::common::get_indexed_attestation( let indexed_attestation = state_processing::common::get_indexed_attestation(
&local_state, &committee.committee,
&attestation, &attestation,
) )
.expect("should get indexed attestation"); .expect("should get indexed attestation");
@ -338,9 +343,12 @@ fn bench_block<T: EthSpec>(
Benchmark::new("is_valid_indexed_attestation_without_signature", move |b| { Benchmark::new("is_valid_indexed_attestation_without_signature", move |b| {
b.iter_batched_ref( b.iter_batched_ref(
|| { || {
let attestation = &local_block.body.attestations[0]; let attestation = &local_block.message.body.attestations[0];
let committee = local_state
.get_beacon_committee(attestation.data.slot, attestation.data.index)
.unwrap();
let indexed_attestation = state_processing::common::get_indexed_attestation( let indexed_attestation = state_processing::common::get_indexed_attestation(
&local_state, &committee.committee,
&attestation, &attestation,
) )
.expect("should get indexed attestation"); .expect("should get indexed attestation");
@ -371,14 +379,16 @@ fn bench_block<T: EthSpec>(
Benchmark::new("get_attesting_indices", move |b| { Benchmark::new("get_attesting_indices", move |b| {
b.iter_batched_ref( b.iter_batched_ref(
|| { || {
let attestation = &local_block.body.attestations[0]; let attestation = &local_block.message.body.attestations[0];
let committee = local_state
.get_beacon_committee(attestation.data.slot, attestation.data.index)
.unwrap();
(local_state.clone(), attestation.clone()) (committee.committee, attestation.clone())
}, },
|(ref mut state, attestation)| { |(committee, attestation)| {
black_box(state_processing::common::get_attesting_indices( black_box(state_processing::common::get_attesting_indices::<T>(
state, committee,
&attestation.data,
&attestation.aggregation_bits, &attestation.aggregation_bits,
)) ))
}, },

13
eth2/types/benches/benches.rs
View File

@ -90,19 +90,6 @@ fn all_benches(c: &mut Criterion) {
.sample_size(10), .sample_size(10),
); );
let inner_state = state.clone();
c.bench(
&format!("{}_validators", validator_count),
Benchmark::new("clone_without_caches/beacon_state", move |b| {
b.iter_batched_ref(
|| inner_state.clone(),
|state| black_box(state.clone_without_caches()),
criterion::BatchSize::SmallInput,
)
})
.sample_size(10),
);
let inner_state = state.clone(); let inner_state = state.clone();
c.bench( c.bench(
&format!("{}_validators", validator_count), &format!("{}_validators", validator_count),

2
eth2/types/src/beacon_state/tests.rs
View File

@ -368,8 +368,8 @@ mod committees {
mod get_outstanding_deposit_len { mod get_outstanding_deposit_len {
use super::*; use super::*;
use crate::MinimalEthSpec;
use crate::test_utils::TestingBeaconStateBuilder; use crate::test_utils::TestingBeaconStateBuilder;
use crate::MinimalEthSpec;
fn state() -> BeaconState<MinimalEthSpec> { fn state() -> BeaconState<MinimalEthSpec> {
let spec = MinimalEthSpec::default_spec(); let spec = MinimalEthSpec::default_spec();

4
eth2/utils/tree_hash/benches/benches.rs
View File

@ -54,7 +54,7 @@ fn bench_suite<T: EthSpec>(c: &mut Criterion, spec_desc: &str, validator_count:
b.iter_batched_ref( b.iter_batched_ref(
|| state2.clone(), || state2.clone(),
|state| { |state| {
assert!(!state.tree_hash_cache.is_initialized()); assert!(state.tree_hash_cache.is_none());
black_box(state.update_tree_hash_cache().unwrap()) black_box(state.update_tree_hash_cache().unwrap())
}, },
criterion::BatchSize::SmallInput, criterion::BatchSize::SmallInput,
@ -72,7 +72,7 @@ fn bench_suite<T: EthSpec>(c: &mut Criterion, spec_desc: &str, validator_count:
b.iter_batched_ref( b.iter_batched_ref(
|| state3.clone(), || state3.clone(),
|state| { |state| {
assert!(state.tree_hash_cache.is_initialized()); assert!(state.tree_hash_cache.is_some());
black_box(state.update_tree_hash_cache().unwrap()) black_box(state.update_tree_hash_cache().unwrap())
}, },
criterion::BatchSize::SmallInput, criterion::BatchSize::SmallInput,