Update benchmarks, add to CI (#988)

.github/workflows/test-suite.yml

@@ -58,3 +58,10 @@ jobs:
       run: sudo npm install -g ganache-cli
     - name: Run the beacon chain sim
       run: cargo run --release --bin simulator beacon-chain-sim
+  check-benchmarks:
+    runs-on: ubuntu-latest
+    needs: cargo-fmt
+    steps:
+    - uses: actions/checkout@v1
+    - name: Typecheck benchmark code without running it
+      run: make check-benches

Makefile

@@ -22,6 +22,10 @@ test-debug:
 cargo-fmt:
 	cargo fmt --all -- --check
 
+# Typechecks benchmark code
+check-benches:
+	cargo check --all --benches
+
 # Runs only the ef-test vectors.
 run-ef-tests:
 	cargo test --release --manifest-path=$(EF_TESTS)/Cargo.toml --features "ef_tests"

eth2/state_processing/benches/benches.rs

@@ -4,7 +4,9 @@ use criterion::Criterion;
 use criterion::{black_box, criterion_group, criterion_main, Benchmark};
 use ssz::Encode;
 use state_processing::{test_utils::BlockBuilder, BlockSignatureStrategy, VerifySignatures};
-use types::{BeaconBlock, BeaconState, ChainSpec, EthSpec, MainnetEthSpec, MinimalEthSpec, Slot};
+use types::{
+    BeaconState, ChainSpec, EthSpec, MainnetEthSpec, MinimalEthSpec, SignedBeaconBlock, Slot,
+};
 
 pub const VALIDATORS_LOW: usize = 32_768;
 pub const VALIDATORS_HIGH: usize = 300_032;
@@ -45,7 +47,7 @@ fn worst_bench<T: EthSpec>(c: &mut Criterion, spec_desc: &str, validator_count:
 fn get_average_block<T: EthSpec>(
     validator_count: usize,
     spec: &ChainSpec,
-) -> (BeaconBlock<T>, BeaconState<T>) {
+) -> (SignedBeaconBlock<T>, BeaconState<T>) {
     let mut builder: BlockBuilder<T> = BlockBuilder::new(validator_count, &spec);
     // builder.num_attestations = T::MaxAttestations::to_usize();
     builder.num_attestations = 16;
@@ -59,7 +61,7 @@ fn get_average_block<T: EthSpec>(
 fn get_worst_block<T: EthSpec>(
     validator_count: usize,
     spec: &ChainSpec,
-) -> (BeaconBlock<T>, BeaconState<T>) {
+) -> (SignedBeaconBlock<T>, BeaconState<T>) {
     let mut builder: BlockBuilder<T> = BlockBuilder::new(validator_count, &spec);
     builder.maximize_block_operations();
 
@@ -74,7 +76,7 @@ fn get_worst_block<T: EthSpec>(
 #[allow(clippy::unit_arg)]
 fn bench_block<T: EthSpec>(
     c: &mut Criterion,
-    block: BeaconBlock<T>,
+    block: SignedBeaconBlock<T>,
     state: BeaconState<T>,
     spec: &ChainSpec,
     spec_desc: &str,
@@ -183,9 +185,7 @@ fn bench_block<T: EthSpec>(
                     black_box(
                         state_processing::per_block_processing::process_block_header::<T>(
                             state,
-                            &block,
-                            None,
-                            VerifySignatures::True,
+                            &block.message,
                             &spec,
                         )
                         .expect("process_block_header should succeed"),
@@ -231,7 +231,7 @@ fn bench_block<T: EthSpec>(
                     black_box(
                         state_processing::per_block_processing::process_attestations::<T>(
                             state,
-                            &block.body.attestations,
+                            &block.message.body.attestations,
                             VerifySignatures::True,
                             &spec,
                         )
@@ -252,7 +252,7 @@ fn bench_block<T: EthSpec>(
         Benchmark::new("verify_attestation", move |b| {
             b.iter_batched_ref(
                 || {
-                    let attestation = &local_block.body.attestations[0];
+                    let attestation = &local_block.message.body.attestations[0];
 
                     (local_spec.clone(), local_state.clone(), attestation.clone())
                 },
@@ -280,13 +280,15 @@ fn bench_block<T: EthSpec>(
         Benchmark::new("get_indexed_attestation", move |b| {
             b.iter_batched_ref(
                 || {
-                    let attestation = &local_block.body.attestations[0];
-
-                    (local_state.clone(), attestation.clone())
+                    let attestation = &local_block.message.body.attestations[0];
+                    let committee = local_state
+                        .get_beacon_committee(attestation.data.slot, attestation.data.index)
+                        .unwrap();
+                    (committee.committee, attestation.clone())
                 },
-                |(ref mut state, attestation)| {
+                |(committee, attestation)| {
                     black_box(
-                        state_processing::common::get_indexed_attestation(state, &attestation)
+                        state_processing::common::get_indexed_attestation(committee, &attestation)
                             .expect("should get indexed attestation"),
                     )
                 },
@@ -304,9 +306,12 @@ fn bench_block<T: EthSpec>(
         Benchmark::new("is_valid_indexed_attestation_with_signature", move |b| {
             b.iter_batched_ref(
                 || {
-                    let attestation = &local_block.body.attestations[0];
+                    let attestation = &local_block.message.body.attestations[0];
+                    let committee = local_state
+                        .get_beacon_committee(attestation.data.slot, attestation.data.index)
+                        .unwrap();
                     let indexed_attestation = state_processing::common::get_indexed_attestation(
-                        &local_state,
+                        &committee.committee,
                         &attestation,
                     )
                     .expect("should get indexed attestation");
@@ -338,9 +343,12 @@ fn bench_block<T: EthSpec>(
         Benchmark::new("is_valid_indexed_attestation_without_signature", move |b| {
             b.iter_batched_ref(
                 || {
-                    let attestation = &local_block.body.attestations[0];
+                    let attestation = &local_block.message.body.attestations[0];
+                    let committee = local_state
+                        .get_beacon_committee(attestation.data.slot, attestation.data.index)
+                        .unwrap();
                     let indexed_attestation = state_processing::common::get_indexed_attestation(
-                        &local_state,
+                        &committee.committee,
                         &attestation,
                     )
                     .expect("should get indexed attestation");
@@ -371,14 +379,16 @@ fn bench_block<T: EthSpec>(
         Benchmark::new("get_attesting_indices", move |b| {
             b.iter_batched_ref(
                 || {
-                    let attestation = &local_block.body.attestations[0];
+                    let attestation = &local_block.message.body.attestations[0];
+                    let committee = local_state
+                        .get_beacon_committee(attestation.data.slot, attestation.data.index)
+                        .unwrap();
 
-                    (local_state.clone(), attestation.clone())
+                    (committee.committee, attestation.clone())
                 },
-                |(ref mut state, attestation)| {
-                    black_box(state_processing::common::get_attesting_indices(
-                        state,
-                        &attestation.data,
+                |(committee, attestation)| {
+                    black_box(state_processing::common::get_attesting_indices::<T>(
+                        committee,
                         &attestation.aggregation_bits,
                     ))
                 },

eth2/types/benches/benches.rs

@@ -90,19 +90,6 @@ fn all_benches(c: &mut Criterion) {
         .sample_size(10),
     );
 
-    let inner_state = state.clone();
-    c.bench(
-        &format!("{}_validators", validator_count),
-        Benchmark::new("clone_without_caches/beacon_state", move |b| {
-            b.iter_batched_ref(
-                || inner_state.clone(),
-                |state| black_box(state.clone_without_caches()),
-                criterion::BatchSize::SmallInput,
-            )
-        })
-        .sample_size(10),
-    );
-
     let inner_state = state.clone();
     c.bench(
         &format!("{}_validators", validator_count),

eth2/types/src/beacon_state/tests.rs

@@ -368,8 +368,8 @@ mod committees {
 
 mod get_outstanding_deposit_len {
     use super::*;
-    use crate::MinimalEthSpec;
     use crate::test_utils::TestingBeaconStateBuilder;
+    use crate::MinimalEthSpec;
 
     fn state() -> BeaconState<MinimalEthSpec> {
         let spec = MinimalEthSpec::default_spec();

eth2/utils/tree_hash/benches/benches.rs

@@ -54,7 +54,7 @@ fn bench_suite<T: EthSpec>(c: &mut Criterion, spec_desc: &str, validator_count:
             b.iter_batched_ref(
                 || state2.clone(),
                 |state| {
-                    assert!(!state.tree_hash_cache.is_initialized());
+                    assert!(state.tree_hash_cache.is_none());
                     black_box(state.update_tree_hash_cache().unwrap())
                 },
                 criterion::BatchSize::SmallInput,
@@ -72,7 +72,7 @@ fn bench_suite<T: EthSpec>(c: &mut Criterion, spec_desc: &str, validator_count:
             b.iter_batched_ref(
                 || state3.clone(),
                 |state| {
-                    assert!(state.tree_hash_cache.is_initialized());
+                    assert!(state.tree_hash_cache.is_some());
                     black_box(state.update_tree_hash_cache().unwrap())
                 },
                 criterion::BatchSize::SmallInput,