From 5a7903a3773ae5c640a0004ef2f2ddec77322dfb Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 19 Aug 2019 16:15:55 +1000 Subject: [PATCH 01/12] Improve BeaconState safe accessors And fix a bug in the compact committees accessor. --- .../src/per_epoch_processing.rs | 45 +++----- eth2/types/src/beacon_state.rs | 104 +++++++++++------- 2 files changed, 81 insertions(+), 68 deletions(-) diff --git a/eth2/state_processing/src/per_epoch_processing.rs b/eth2/state_processing/src/per_epoch_processing.rs index 8d6153aea..71d8b20da 100644 --- a/eth2/state_processing/src/per_epoch_processing.rs +++ b/eth2/state_processing/src/per_epoch_processing.rs @@ -221,42 +221,29 @@ pub fn process_final_updates( // Update start shard. state.start_shard = state.next_epoch_start_shard(spec)?; - // This is a hack to allow us to update index roots and slashed balances for the next epoch. - // - // The indentation here is to make it obvious where the weird stuff happens. - { - state.slot += 1; - - // Set active index root - let index_epoch = next_epoch + spec.activation_exit_delay; - let indices_list = VariableList::::from( - state.get_active_validator_indices(index_epoch), - ); - state.set_active_index_root( - index_epoch, - Hash256::from_slice(&indices_list.tree_hash_root()), - spec, - )?; - - // Reset slashings - state.set_slashings(next_epoch, 0)?; - - // Set randao mix - state.set_randao_mix(next_epoch, *state.get_randao_mix(current_epoch)?)?; - - state.slot -= 1; - } + // Set active index root + let index_epoch = next_epoch + spec.activation_exit_delay; + let indices_list = VariableList::::from( + state.get_active_validator_indices(index_epoch), + ); + state.set_active_index_root( + index_epoch, + Hash256::from_slice(&indices_list.tree_hash_root()), + spec, + )?; // Set committees root - // Note: we do this out-of-order w.r.t. to the spec, because we don't want the slot to be - // incremented. It's safe because the updates to slashings and the RANDAO mix (above) don't - // affect this. state.set_compact_committee_root( next_epoch, get_compact_committees_root(state, RelativeEpoch::Next, spec)?, - spec, )?; + // Reset slashings + state.set_slashings(next_epoch, 0)?; + + // Set randao mix + state.set_randao_mix(next_epoch, *state.get_randao_mix(current_epoch)?)?; + // Set historical root accumulator if next_epoch.as_u64() % (T::SlotsPerHistoricalRoot::to_u64() / T::slots_per_epoch()) == 0 { let historical_batch = state.historical_batch(); diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index d312316f3..5b00f08b7 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -60,6 +60,22 @@ pub enum Error { SszTypesError(ssz_types::Error), } +/// Control whether an epoch-indexed field can be indexed at the next epoch or not. +#[derive(Debug, PartialEq, Clone, Copy)] +enum AllowNextEpoch { + True, + False, +} + +impl AllowNextEpoch { + fn upper_bound_of(self, current_epoch: Epoch) -> Epoch { + match self { + AllowNextEpoch::True => current_epoch + 1, + AllowNextEpoch::False => current_epoch, + } + } +} + /// The state of the `BeaconChain` at some slot. /// /// Spec v0.8.0 @@ -108,12 +124,12 @@ where pub start_shard: u64, pub randao_mixes: FixedVector, #[compare_fields(as_slice)] - active_index_roots: FixedVector, + pub active_index_roots: FixedVector, #[compare_fields(as_slice)] - compact_committees_roots: FixedVector, + pub compact_committees_roots: FixedVector, // Slashings - slashings: FixedVector, + pub slashings: FixedVector, // Attestations pub previous_epoch_attestations: VariableList, T::MaxPendingAttestations>, @@ -459,12 +475,16 @@ impl BeaconState { /// Safely obtains the index for `randao_mixes` /// - /// Spec v0.8.0 - fn get_randao_mix_index(&self, epoch: Epoch) -> Result { + /// Spec v0.8.1 + fn get_randao_mix_index( + &self, + epoch: Epoch, + allow_next_epoch: AllowNextEpoch, + ) -> Result { let current_epoch = self.current_epoch(); let len = T::EpochsPerHistoricalVector::to_u64(); - if epoch + len > current_epoch && epoch <= current_epoch { + if current_epoch < epoch + len && epoch <= allow_next_epoch.upper_bound_of(current_epoch) { Ok(epoch.as_usize() % len as usize) } else { Err(Error::EpochOutOfBounds) @@ -492,7 +512,7 @@ impl BeaconState { /// /// Spec v0.8.1 pub fn get_randao_mix(&self, epoch: Epoch) -> Result<&Hash256, Error> { - let i = self.get_randao_mix_index(epoch)?; + let i = self.get_randao_mix_index(epoch, AllowNextEpoch::False)?; Ok(&self.randao_mixes[i]) } @@ -500,21 +520,29 @@ impl BeaconState { /// /// Spec v0.8.1 pub fn set_randao_mix(&mut self, epoch: Epoch, mix: Hash256) -> Result<(), Error> { - let i = self.get_randao_mix_index(epoch)?; + let i = self.get_randao_mix_index(epoch, AllowNextEpoch::True)?; self.randao_mixes[i] = mix; Ok(()) } /// Safely obtains the index for `active_index_roots`, given some `epoch`. /// + /// If `allow_next_epoch` is `True`, then we allow an _extra_ one epoch of lookahead. + /// /// Spec v0.8.1 - fn get_active_index_root_index(&self, epoch: Epoch, spec: &ChainSpec) -> Result { + fn get_active_index_root_index( + &self, + epoch: Epoch, + spec: &ChainSpec, + allow_next_epoch: AllowNextEpoch, + ) -> Result { let current_epoch = self.current_epoch(); let lookahead = spec.activation_exit_delay; let lookback = self.active_index_roots.len() as u64 - lookahead; + let epoch_upper_bound = allow_next_epoch.upper_bound_of(current_epoch) + lookahead; - if epoch + lookback > current_epoch && current_epoch + lookahead >= epoch { + if current_epoch < epoch + lookback && epoch <= epoch_upper_bound { Ok(epoch.as_usize() % self.active_index_roots.len()) } else { Err(Error::EpochOutOfBounds) @@ -525,7 +553,7 @@ impl BeaconState { /// /// Spec v0.8.1 pub fn get_active_index_root(&self, epoch: Epoch, spec: &ChainSpec) -> Result { - let i = self.get_active_index_root_index(epoch, spec)?; + let i = self.get_active_index_root_index(epoch, spec, AllowNextEpoch::False)?; Ok(self.active_index_roots[i]) } @@ -538,7 +566,7 @@ impl BeaconState { index_root: Hash256, spec: &ChainSpec, ) -> Result<(), Error> { - let i = self.get_active_index_root_index(epoch, spec)?; + let i = self.get_active_index_root_index(epoch, spec, AllowNextEpoch::True)?; self.active_index_roots[i] = index_root; Ok(()) } @@ -552,19 +580,17 @@ impl BeaconState { /// Safely obtains the index for `compact_committees_roots`, given some `epoch`. /// - /// Spec v0.8.0 + /// Spec v0.8.1 fn get_compact_committee_root_index( &self, epoch: Epoch, - spec: &ChainSpec, + allow_next_epoch: AllowNextEpoch, ) -> Result { let current_epoch = self.current_epoch(); + let len = T::EpochsPerHistoricalVector::to_u64(); - let lookahead = spec.activation_exit_delay; - let lookback = self.compact_committees_roots.len() as u64 - lookahead; - - if epoch + lookback > current_epoch && current_epoch + lookahead >= epoch { - Ok(epoch.as_usize() % self.compact_committees_roots.len()) + if current_epoch < epoch + len && epoch <= allow_next_epoch.upper_bound_of(current_epoch) { + Ok(epoch.as_usize() % len as usize) } else { Err(Error::EpochOutOfBounds) } @@ -572,26 +598,21 @@ impl BeaconState { /// Return the `compact_committee_root` at a recent `epoch`. /// - /// Spec v0.8.0 - pub fn get_compact_committee_root( - &self, - epoch: Epoch, - spec: &ChainSpec, - ) -> Result { - let i = self.get_compact_committee_root_index(epoch, spec)?; + /// Spec v0.8.1 + pub fn get_compact_committee_root(&self, epoch: Epoch) -> Result { + let i = self.get_compact_committee_root_index(epoch, AllowNextEpoch::False)?; Ok(self.compact_committees_roots[i]) } /// Set the `compact_committee_root` at a recent `epoch`. /// - /// Spec v0.8.0 + /// Spec v0.8.1 pub fn set_compact_committee_root( &mut self, epoch: Epoch, index_root: Hash256, - spec: &ChainSpec, ) -> Result<(), Error> { - let i = self.get_compact_committee_root_index(epoch, spec)?; + let i = self.get_compact_committee_root_index(epoch, AllowNextEpoch::True)?; self.compact_committees_roots[i] = index_root; Ok(()) } @@ -642,14 +663,19 @@ impl BeaconState { /// Safely obtain the index for `slashings`, given some `epoch`. /// - /// Spec v0.8.0 - fn get_slashings_index(&self, epoch: Epoch) -> Result { + /// Spec v0.8.1 + fn get_slashings_index( + &self, + epoch: Epoch, + allow_next_epoch: AllowNextEpoch, + ) -> Result { // We allow the slashings vector to be accessed at any cached epoch at or before - // the current epoch. - if epoch <= self.current_epoch() - && epoch + T::EpochsPerSlashingsVector::to_u64() >= self.current_epoch() + 1 + // the current epoch, or the next epoch if `AllowNextEpoch::True` is passed. + let current_epoch = self.current_epoch(); + if current_epoch < epoch + T::EpochsPerSlashingsVector::to_u64() + && epoch <= allow_next_epoch.upper_bound_of(current_epoch) { - Ok((epoch.as_u64() % T::EpochsPerSlashingsVector::to_u64()) as usize) + Ok(epoch.as_usize() % T::EpochsPerSlashingsVector::to_usize()) } else { Err(Error::EpochOutOfBounds) } @@ -664,17 +690,17 @@ impl BeaconState { /// Get the total slashed balances for some epoch. /// - /// Spec v0.8.0 + /// Spec v0.8.1 pub fn get_slashings(&self, epoch: Epoch) -> Result { - let i = self.get_slashings_index(epoch)?; + let i = self.get_slashings_index(epoch, AllowNextEpoch::False)?; Ok(self.slashings[i]) } /// Set the total slashed balances for some epoch. /// - /// Spec v0.8.0 + /// Spec v0.8.1 pub fn set_slashings(&mut self, epoch: Epoch, value: u64) -> Result<(), Error> { - let i = self.get_slashings_index(epoch)?; + let i = self.get_slashings_index(epoch, AllowNextEpoch::True)?; self.slashings[i] = value; Ok(()) } From 4d2cdc94927279dce594510649df4cfedf30f187 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 27 Aug 2019 16:59:53 +1000 Subject: [PATCH 02/12] Update to spec v0.8.3 --- .../src/common/get_attesting_indices.rs | 4 +--- .../src/common/get_compact_committees_root.rs | 21 +++---------------- .../src/common/get_indexed_attestation.rs | 2 ++ .../src/per_epoch_processing.rs | 6 +++--- eth2/types/src/beacon_state.rs | 8 ------- 5 files changed, 9 insertions(+), 32 deletions(-) diff --git a/eth2/state_processing/src/common/get_attesting_indices.rs b/eth2/state_processing/src/common/get_attesting_indices.rs index f558909f6..adb71801a 100644 --- a/eth2/state_processing/src/common/get_attesting_indices.rs +++ b/eth2/state_processing/src/common/get_attesting_indices.rs @@ -17,11 +17,9 @@ pub fn get_attesting_indices( target_relative_epoch, )?; - /* TODO(freeze): re-enable this? - if bitlist.len() > committee.committee.len() { + if bitlist.len() != committee.committee.len() { return Err(BeaconStateError::InvalidBitfield); } - */ Ok(committee .committee diff --git a/eth2/state_processing/src/common/get_compact_committees_root.rs b/eth2/state_processing/src/common/get_compact_committees_root.rs index 75edb3549..b8ab4345f 100644 --- a/eth2/state_processing/src/common/get_compact_committees_root.rs +++ b/eth2/state_processing/src/common/get_compact_committees_root.rs @@ -3,7 +3,7 @@ use types::*; /// Return the compact committee root at `relative_epoch`. /// -/// Spec v0.8.0 +/// Spec v0.8.3 pub fn get_compact_committees_root( state: &BeaconState, relative_epoch: RelativeEpoch, @@ -11,28 +11,13 @@ pub fn get_compact_committees_root( ) -> Result { let mut committees = FixedVector::<_, T::ShardCount>::from_elem(CompactCommittee::::default()); - // FIXME: this is a spec bug, whereby the start shard for the epoch after the next epoch - // is mistakenly used. The start shard from the cache SHOULD work. - // Waiting on a release to fix https://github.com/ethereum/eth2.0-specs/issues/1315 - let start_shard = if relative_epoch == RelativeEpoch::Next { - state.next_epoch_start_shard(spec)? - } else { - state.get_epoch_start_shard(relative_epoch)? - }; + let start_shard = state.get_epoch_start_shard(relative_epoch)?; for committee_number in 0..state.get_committee_count(relative_epoch)? { let shard = (start_shard + committee_number) % T::ShardCount::to_u64(); - // FIXME: this is a partial workaround for the above, but it only works in the case - // where there's a committee for every shard in every epoch. It works for the minimal - // tests but not the mainnet ones. - let fake_shard = if relative_epoch == RelativeEpoch::Next { - (shard + 1) % T::ShardCount::to_u64() - } else { - shard - }; for &index in state - .get_crosslink_committee_for_shard(fake_shard, relative_epoch)? + .get_crosslink_committee_for_shard(shard, relative_epoch)? .committee { let validator = state diff --git a/eth2/state_processing/src/common/get_indexed_attestation.rs b/eth2/state_processing/src/common/get_indexed_attestation.rs index 7c08c8708..82ca92eb7 100644 --- a/eth2/state_processing/src/common/get_indexed_attestation.rs +++ b/eth2/state_processing/src/common/get_indexed_attestation.rs @@ -11,6 +11,8 @@ pub fn get_indexed_attestation( state: &BeaconState, attestation: &Attestation, ) -> Result, Error> { + // Note: we rely on both calls to `get_attesting_indices` to check the bitfield lengths + // against the committee length let attesting_indices = get_attesting_indices(state, &attestation.data, &attestation.aggregation_bits)?; diff --git a/eth2/state_processing/src/per_epoch_processing.rs b/eth2/state_processing/src/per_epoch_processing.rs index 71d8b20da..08f42a229 100644 --- a/eth2/state_processing/src/per_epoch_processing.rs +++ b/eth2/state_processing/src/per_epoch_processing.rs @@ -218,9 +218,6 @@ pub fn process_final_updates( } } - // Update start shard. - state.start_shard = state.next_epoch_start_shard(spec)?; - // Set active index root let index_epoch = next_epoch + spec.activation_exit_delay; let indices_list = VariableList::::from( @@ -252,6 +249,9 @@ pub fn process_final_updates( .push(Hash256::from_slice(&historical_batch.tree_hash_root()))?; } + // Update start shard. + state.start_shard = state.get_epoch_start_shard(RelativeEpoch::Next)?; + // Rotate current/previous epoch attestations state.previous_epoch_attestations = std::mem::replace(&mut state.current_epoch_attestations, VariableList::empty()); diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 5b00f08b7..9b623c070 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -298,14 +298,6 @@ impl BeaconState { Ok(cache.epoch_start_shard()) } - pub fn next_epoch_start_shard(&self, spec: &ChainSpec) -> Result { - let cache = self.cache(RelativeEpoch::Current)?; - let active_validator_count = cache.active_validator_count(); - let shard_delta = T::get_shard_delta(active_validator_count, spec.target_committee_size); - - Ok((self.start_shard + shard_delta) % T::ShardCount::to_u64()) - } - /// Get the slot of an attestation. /// /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. From aed2f6407dac2e644825c76734ce610badf1e637 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 27 Aug 2019 17:00:21 +1000 Subject: [PATCH 03/12] Bump EF tests to v0.8.3 --- tests/ef_tests/eth2.0-spec-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ef_tests/eth2.0-spec-tests b/tests/ef_tests/eth2.0-spec-tests index aaa1673f5..ae6dd9011 160000 --- a/tests/ef_tests/eth2.0-spec-tests +++ b/tests/ef_tests/eth2.0-spec-tests @@ -1 +1 @@ -Subproject commit aaa1673f508103e11304833e0456e4149f880065 +Subproject commit ae6dd9011df05fab8c7e651c09cf9c940973bf81 From 23a308e595cad8438aef47a1be95af2516839448 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 28 Aug 2019 18:46:16 +1000 Subject: [PATCH 04/12] BLS and SSZ static tests --- eth2/types/src/attestation_data.rs | 16 +- eth2/types/src/checkpoint.rs | 3 +- tests/ef_tests/src/cases.rs | 27 +++ .../src/cases/bls_aggregate_pubkeys.rs | 6 +- .../ef_tests/src/cases/bls_aggregate_sigs.rs | 6 +- tests/ef_tests/src/cases/bls_g2_compressed.rs | 6 +- tests/ef_tests/src/cases/bls_priv_to_pub.rs | 6 +- tests/ef_tests/src/cases/bls_sign_msg.rs | 6 +- tests/ef_tests/src/cases/ssz_static.rs | 207 +++++++++--------- tests/ef_tests/src/doc.rs | 53 +++-- tests/ef_tests/src/handler.rs | 130 +++++++++++ tests/ef_tests/src/lib.rs | 3 + tests/ef_tests/src/type_name.rs | 61 ++++++ tests/ef_tests/src/yaml_decode.rs | 14 +- tests/ef_tests/src/yaml_decode/utils.rs | 10 - tests/ef_tests/tests/tests.rs | 105 ++++++++- 16 files changed, 481 insertions(+), 178 deletions(-) create mode 100644 tests/ef_tests/src/handler.rs create mode 100644 tests/ef_tests/src/type_name.rs delete mode 100644 tests/ef_tests/src/yaml_decode/utils.rs diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data.rs index f2e63598f..4d82ce126 100644 --- a/eth2/types/src/attestation_data.rs +++ b/eth2/types/src/attestation_data.rs @@ -4,25 +4,13 @@ use crate::{Checkpoint, Crosslink, Hash256}; use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; -use tree_hash::TreeHash; -use tree_hash_derive::{SignedRoot, TreeHash}; +use tree_hash_derive::TreeHash; /// The data upon which an attestation is based. /// /// Spec v0.8.0 #[derive( - Debug, - Clone, - PartialEq, - Eq, - Serialize, - Deserialize, - Hash, - Encode, - Decode, - TreeHash, - TestRandom, - SignedRoot, + Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash, Encode, Decode, TreeHash, TestRandom, )] pub struct AttestationData { // LMD GHOST vote diff --git a/eth2/types/src/checkpoint.rs b/eth2/types/src/checkpoint.rs index dc40b336f..0c7001921 100644 --- a/eth2/types/src/checkpoint.rs +++ b/eth2/types/src/checkpoint.rs @@ -4,7 +4,7 @@ use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; use tree_hash::TreeHash; -use tree_hash_derive::{SignedRoot, TreeHash}; +use tree_hash_derive::TreeHash; /// Casper FFG checkpoint, used in attestations. /// @@ -22,7 +22,6 @@ use tree_hash_derive::{SignedRoot, TreeHash}; Decode, TreeHash, TestRandom, - SignedRoot, )] pub struct Checkpoint { pub epoch: Epoch, diff --git a/tests/ef_tests/src/cases.rs b/tests/ef_tests/src/cases.rs index 1ae4ea1d8..7f6ffb0c4 100644 --- a/tests/ef_tests/src/cases.rs +++ b/tests/ef_tests/src/cases.rs @@ -1,5 +1,6 @@ use super::*; use std::fmt::Debug; +use std::path::Path; mod bls_aggregate_pubkeys; mod bls_aggregate_sigs; @@ -53,6 +54,11 @@ pub use shuffling::*; pub use ssz_generic::*; pub use ssz_static::*; +pub trait LoadCase: Sized { + /// Load the test case from a test case directory. + fn load_from_dir(_path: &Path) -> Result; +} + pub trait Case: Debug { /// An optional field for implementing a custom description. /// @@ -68,6 +74,26 @@ pub trait Case: Debug { fn result(&self, case_index: usize) -> Result<(), Error>; } +pub trait BlsCase: serde::de::DeserializeOwned {} + +impl YamlDecode for T +where + T: BlsCase, +{ + fn yaml_decode(string: &str) -> Result { + serde_yaml::from_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) + } +} + +impl LoadCase for T +where + T: BlsCase, +{ + fn load_from_dir(path: &Path) -> Result { + Self::yaml_decode_file(&path.join("data.yaml")) + } +} + #[derive(Debug)] pub struct Cases { pub test_cases: Vec, @@ -86,6 +112,7 @@ where } } +// FIXME(michael): delete this impl YamlDecode for Cases { /// Decodes a YAML list of test cases fn yaml_decode(yaml: &str) -> Result { diff --git a/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs b/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs index 6e38743f2..c94e14495 100644 --- a/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs +++ b/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs @@ -9,11 +9,7 @@ pub struct BlsAggregatePubkeys { pub output: String, } -impl YamlDecode for BlsAggregatePubkeys { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} +impl BlsCase for BlsAggregatePubkeys {} impl Case for BlsAggregatePubkeys { fn result(&self, _case_index: usize) -> Result<(), Error> { diff --git a/tests/ef_tests/src/cases/bls_aggregate_sigs.rs b/tests/ef_tests/src/cases/bls_aggregate_sigs.rs index eeecab82c..882ad7220 100644 --- a/tests/ef_tests/src/cases/bls_aggregate_sigs.rs +++ b/tests/ef_tests/src/cases/bls_aggregate_sigs.rs @@ -9,11 +9,7 @@ pub struct BlsAggregateSigs { pub output: String, } -impl YamlDecode for BlsAggregateSigs { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} +impl BlsCase for BlsAggregateSigs {} impl Case for BlsAggregateSigs { fn result(&self, _case_index: usize) -> Result<(), Error> { diff --git a/tests/ef_tests/src/cases/bls_g2_compressed.rs b/tests/ef_tests/src/cases/bls_g2_compressed.rs index 185cb58f3..547d8d03a 100644 --- a/tests/ef_tests/src/cases/bls_g2_compressed.rs +++ b/tests/ef_tests/src/cases/bls_g2_compressed.rs @@ -15,11 +15,7 @@ pub struct BlsG2Compressed { pub output: Vec, } -impl YamlDecode for BlsG2Compressed { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} +impl BlsCase for BlsG2Compressed {} impl Case for BlsG2Compressed { fn result(&self, _case_index: usize) -> Result<(), Error> { diff --git a/tests/ef_tests/src/cases/bls_priv_to_pub.rs b/tests/ef_tests/src/cases/bls_priv_to_pub.rs index d72a43bbb..869a0891c 100644 --- a/tests/ef_tests/src/cases/bls_priv_to_pub.rs +++ b/tests/ef_tests/src/cases/bls_priv_to_pub.rs @@ -9,11 +9,7 @@ pub struct BlsPrivToPub { pub output: String, } -impl YamlDecode for BlsPrivToPub { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} +impl BlsCase for BlsPrivToPub {} impl Case for BlsPrivToPub { fn result(&self, _case_index: usize) -> Result<(), Error> { diff --git a/tests/ef_tests/src/cases/bls_sign_msg.rs b/tests/ef_tests/src/cases/bls_sign_msg.rs index e62c3550f..476ecdefb 100644 --- a/tests/ef_tests/src/cases/bls_sign_msg.rs +++ b/tests/ef_tests/src/cases/bls_sign_msg.rs @@ -16,11 +16,7 @@ pub struct BlsSign { pub output: String, } -impl YamlDecode for BlsSign { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} +impl BlsCase for BlsSign {} impl Case for BlsSign { fn result(&self, _case_index: usize) -> Result<(), Error> { diff --git a/tests/ef_tests/src/cases/ssz_static.rs b/tests/ef_tests/src/cases/ssz_static.rs index 96ba38b6a..6a949073d 100644 --- a/tests/ef_tests/src/cases/ssz_static.rs +++ b/tests/ef_tests/src/cases/ssz_static.rs @@ -3,125 +3,120 @@ use crate::case_result::compare_result; use serde_derive::Deserialize; use ssz::{Decode, Encode}; use std::fmt::Debug; -use std::marker::PhantomData; -use tree_hash::TreeHash; -use types::{ - test_utils::TestRandom, Attestation, AttestationData, AttestationDataAndCustodyBit, - AttesterSlashing, BeaconBlock, BeaconBlockBody, BeaconBlockHeader, BeaconState, Checkpoint, - CompactCommittee, Crosslink, Deposit, DepositData, Eth1Data, EthSpec, Fork, Hash256, - HistoricalBatch, IndexedAttestation, PendingAttestation, ProposerSlashing, Transfer, Validator, - VoluntaryExit, -}; - -// Enum variant names are used by Serde when deserializing the test YAML -#[allow(clippy::large_enum_variant)] -#[derive(Debug, Clone, Deserialize)] -pub enum SszStatic -where - E: EthSpec, -{ - Fork(SszStaticInner), - Crosslink(SszStaticInner), - Checkpoint(SszStaticInner), - CompactCommittee(SszStaticInner, E>), - Eth1Data(SszStaticInner), - AttestationData(SszStaticInner), - AttestationDataAndCustodyBit(SszStaticInner), - IndexedAttestation(SszStaticInner, E>), - DepositData(SszStaticInner), - BeaconBlockHeader(SszStaticInner), - Validator(SszStaticInner), - PendingAttestation(SszStaticInner, E>), - HistoricalBatch(SszStaticInner, E>), - ProposerSlashing(SszStaticInner), - AttesterSlashing(SszStaticInner, E>), - Attestation(SszStaticInner, E>), - Deposit(SszStaticInner), - VoluntaryExit(SszStaticInner), - Transfer(SszStaticInner), - BeaconBlockBody(SszStaticInner, E>), - BeaconBlock(SszStaticInner, E>), - BeaconState(SszStaticInner, E>), -} +use std::fs; +use tree_hash::{SignedRoot, TreeHash}; +use types::Hash256; #[derive(Debug, Clone, Deserialize)] -pub struct SszStaticInner -where - E: EthSpec, -{ - pub value: T, - pub serialized: String, - pub root: String, - #[serde(skip, default)] - _phantom: PhantomData, +struct SszStaticRoots { + root: String, + signing_root: Option, } -impl YamlDecode for SszStatic { +impl YamlDecode for SszStaticRoots { fn yaml_decode(yaml: &str) -> Result { - serde_yaml::from_str(yaml).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) + Ok(serde_yaml::from_str(yaml).unwrap()) } } -impl Case for SszStatic { - fn result(&self, _case_index: usize) -> Result<(), Error> { - use self::SszStatic::*; - - match *self { - Fork(ref val) => ssz_static_test(val), - Crosslink(ref val) => ssz_static_test(val), - Checkpoint(ref val) => ssz_static_test(val), - CompactCommittee(ref val) => ssz_static_test(val), - Eth1Data(ref val) => ssz_static_test(val), - AttestationData(ref val) => ssz_static_test(val), - AttestationDataAndCustodyBit(ref val) => ssz_static_test(val), - IndexedAttestation(ref val) => ssz_static_test(val), - DepositData(ref val) => ssz_static_test(val), - BeaconBlockHeader(ref val) => ssz_static_test(val), - Validator(ref val) => ssz_static_test(val), - PendingAttestation(ref val) => ssz_static_test(val), - HistoricalBatch(ref val) => ssz_static_test(val), - ProposerSlashing(ref val) => ssz_static_test(val), - AttesterSlashing(ref val) => ssz_static_test(val), - Attestation(ref val) => ssz_static_test(val), - Deposit(ref val) => ssz_static_test(val), - VoluntaryExit(ref val) => ssz_static_test(val), - Transfer(ref val) => ssz_static_test(val), - BeaconBlockBody(ref val) => ssz_static_test(val), - BeaconBlock(ref val) => ssz_static_test(val), - BeaconState(ref val) => ssz_static_test(val), - } - } +#[derive(Debug, Clone)] +pub struct SszStatic { + roots: SszStaticRoots, + serialized: Vec, + value: T, } -fn ssz_static_test(tc: &SszStaticInner) -> Result<(), Error> -where - T: Clone - + Decode - + Debug - + Encode - + PartialEq - + serde::de::DeserializeOwned - + TreeHash - + TestRandom, +#[derive(Debug, Clone)] +pub struct SszStaticSR { + roots: SszStaticRoots, + serialized: Vec, + value: T, +} + +// Trait alias for all deez bounds +pub trait SszStaticType: + serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug { - // Verify we can decode SSZ in the same way we can decode YAML. - let ssz = hex::decode(&tc.serialized[2..]) - .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?; - let expected = tc.value.clone(); - let decode_result = T::from_ssz_bytes(&ssz); - compare_result(&decode_result, &Some(expected))?; +} - // Verify we can encode the result back into original ssz bytes - let decoded = decode_result.unwrap(); - let encoded_result = decoded.as_ssz_bytes(); - compare_result::, Error>(&Ok(encoded_result), &Some(ssz))?; +impl SszStaticType for T where + T: serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug +{ +} - // Verify the TreeHash root of the decoded struct matches the test. - let expected_root = - &hex::decode(&tc.root[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?; - let expected_root = Hash256::from_slice(&expected_root); - let tree_hash_root = Hash256::from_slice(&decoded.tree_hash_root()); - compare_result::(&Ok(tree_hash_root), &Some(expected_root))?; +fn load_from_dir(path: &Path) -> Result<(SszStaticRoots, Vec, T), Error> { + // FIXME: set description/name + let roots = SszStaticRoots::yaml_decode_file(&path.join("roots.yaml"))?; + + let serialized = fs::read(&path.join("serialized.ssz")).expect("serialized.ssz exists"); + + let yaml = fs::read_to_string(&path.join("value.yaml")).expect("value.yaml exists"); + let value = + serde_yaml::from_str(&yaml).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?; + + Ok((roots, serialized, value)) +} + +impl LoadCase for SszStatic { + fn load_from_dir(path: &Path) -> Result { + load_from_dir(path).map(|(roots, serialized, value)| Self { + roots, + serialized, + value, + }) + } +} + +impl LoadCase for SszStaticSR { + fn load_from_dir(path: &Path) -> Result { + load_from_dir(path).map(|(roots, serialized, value)| Self { + roots, + serialized, + value, + }) + } +} + +fn check_serialization(value: &T, serialized: &[u8]) -> Result<(), Error> { + // Check serialization + let serialized_result = value.as_ssz_bytes(); + compare_result::, Error>(&Ok(serialized_result), &Some(serialized.to_vec()))?; + + // Check deserialization + let deserialized_result = T::from_ssz_bytes(serialized); + compare_result(&deserialized_result, &Some(value.clone()))?; Ok(()) } + +fn check_tree_hash(expected_str: &str, actual_root: Vec) -> Result<(), Error> { + let expected_root = hex::decode(&expected_str[2..]) + .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?; + let expected_root = Hash256::from_slice(&expected_root); + let tree_hash_root = Hash256::from_slice(&actual_root); + compare_result::(&Ok(tree_hash_root), &Some(expected_root)) +} + +impl Case for SszStatic { + fn result(&self, _case_index: usize) -> Result<(), Error> { + check_serialization(&self.value, &self.serialized)?; + check_tree_hash(&self.roots.root, self.value.tree_hash_root())?; + Ok(()) + } +} + +impl Case for SszStaticSR { + fn result(&self, _case_index: usize) -> Result<(), Error> { + check_serialization(&self.value, &self.serialized)?; + check_tree_hash(&self.roots.root, self.value.tree_hash_root())?; + check_tree_hash( + &self + .roots + .signing_root + .as_ref() + .expect("signed root exists"), + self.value.signed_root(), + )?; + Ok(()) + } +} diff --git a/tests/ef_tests/src/doc.rs b/tests/ef_tests/src/doc.rs index 7dfe9954c..f3a41697e 100644 --- a/tests/ef_tests/src/doc.rs +++ b/tests/ef_tests/src/doc.rs @@ -2,10 +2,14 @@ use crate::case_result::CaseResult; use crate::cases::*; use crate::doc_header::DocHeader; use crate::error::Error; -use crate::yaml_decode::{yaml_split_header_and_cases, YamlDecode}; +use crate::yaml_decode::YamlDecode; use crate::EfTest; use serde_derive::Deserialize; -use std::{fs::File, io::prelude::*, path::PathBuf}; +use std::{ + fs::File, + io::prelude::*, + path::{Path, PathBuf}, +}; use types::{MainnetEthSpec, MinimalEthSpec}; #[derive(Debug, Deserialize)] @@ -19,15 +23,13 @@ impl Doc { fn from_path(path: PathBuf) -> Self { let mut file = File::open(path.clone()).unwrap(); - let mut yaml = String::new(); - file.read_to_string(&mut yaml).unwrap(); - - let (header_yaml, cases_yaml) = yaml_split_header_and_cases(yaml.clone()); + let mut cases_yaml = String::new(); + file.read_to_string(&mut cases_yaml).unwrap(); Self { - header_yaml, cases_yaml, path, + header_yaml: String::new(), } } @@ -40,8 +42,6 @@ impl Doc { header.config.as_ref(), ) { ("ssz", "uint", _) => run_test::(self), - ("ssz", "static", "minimal") => run_test::>(self), - ("ssz", "static", "mainnet") => run_test::>(self), ("sanity", "slots", "minimal") => run_test::>(self), // FIXME: skipped due to compact committees issue ("sanity", "slots", "mainnet") => vec![], // run_test::>(self), @@ -172,14 +172,36 @@ impl Doc { } } -pub fn run_test(doc: &Doc) -> Vec +pub fn assert_tests_pass(path: &Path, results: &[CaseResult]) { + let doc = Doc { + header_yaml: String::new(), + cases_yaml: String::new(), + path: path.into(), + }; + + let (failed, skipped_bls, skipped_known_failures) = categorize_results(results); + + if failed.len() + skipped_known_failures.len() > 0 { + print_results( + &doc, + &failed, + &skipped_bls, + &skipped_known_failures, + &results, + ); + if !failed.is_empty() { + panic!("Tests failed (see above)"); + } + } else { + println!("Passed {} tests in {}", results.len(), path.display()); + } +} + +pub fn run_test(_: &Doc) -> Vec where Cases: EfTest + YamlDecode, { - // Pass only the "test_cases" YAML string to `yaml_decode`. - let test_cases: Cases = Cases::yaml_decode(&doc.cases_yaml).unwrap(); - - test_cases.test_results() + panic!("FIXME(michael): delete this") } pub fn categorize_results( @@ -208,7 +230,6 @@ pub fn print_results( skipped_known_failures: &[&CaseResult], results: &[CaseResult], ) { - let header: DocHeader = serde_yaml::from_str(&doc.header_yaml).unwrap(); println!("--------------------------------------------------"); println!( "Test {}", @@ -218,7 +239,7 @@ pub fn print_results( "Failure" } ); - println!("Title: {}", header.title); + println!("Title: TODO"); println!("File: {:?}", doc.path); println!( "{} tests, {} failed, {} skipped (known failure), {} skipped (bls), {} passed. (See below for errors)", diff --git a/tests/ef_tests/src/handler.rs b/tests/ef_tests/src/handler.rs new file mode 100644 index 000000000..1dac988ac --- /dev/null +++ b/tests/ef_tests/src/handler.rs @@ -0,0 +1,130 @@ +use crate::cases::{self, Case, Cases, LoadCase}; +use crate::type_name::TypeName; +use crate::EfTest; +use std::fs; +use std::marker::PhantomData; +use std::path::PathBuf; +use tree_hash::SignedRoot; + +pub trait Handler { + type Case: Case + LoadCase; + + fn config_name() -> &'static str { + "general" + } + + fn fork_name() -> &'static str { + "phase0" + } + + fn runner_name() -> &'static str; + + fn handler_name() -> &'static str; + + fn run() { + let handler_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("eth2.0-spec-tests") + .join("tests") + .join(Self::config_name()) + .join(Self::fork_name()) + .join(Self::runner_name()) + .join(Self::handler_name()); + + // Iterate through test suites + // TODO: parallelism + // TODO: error handling? + let test_cases = fs::read_dir(&handler_path) + .expect("open main directory") + .flat_map(|entry| { + entry + .ok() + .filter(|e| e.file_type().map(|ty| ty.is_dir()).unwrap_or(false)) + }) + .flat_map(|suite| fs::read_dir(suite.path()).expect("open suite dir")) + .flat_map(Result::ok) + .map(|test_case_dir| Self::Case::load_from_dir(&test_case_dir.path()).expect("loads")) + .collect::>(); + + let results = Cases { test_cases }.test_results(); + + crate::doc::assert_tests_pass(&handler_path, &results); + } +} + +macro_rules! bls_handler { + ($runner_name: ident, $case_name:ident, $handler_name:expr) => { + pub struct $runner_name; + + impl Handler for $runner_name { + type Case = cases::$case_name; + + fn runner_name() -> &'static str { + "bls" + } + + fn handler_name() -> &'static str { + $handler_name + } + } + }; +} + +bls_handler!( + BlsAggregatePubkeysHandler, + BlsAggregatePubkeys, + "aggregate_pubkeys" +); +bls_handler!(BlsAggregateSigsHandler, BlsAggregateSigs, "aggregate_sigs"); +bls_handler!( + BlsG2CompressedHandler, + BlsG2Compressed, + "msg_hash_compressed" +); +bls_handler!(BlsPrivToPubHandler, BlsPrivToPub, "priv_to_pub"); +bls_handler!(BlsSignMsgHandler, BlsSign, "sign_msg"); + +/// Handler for SSZ types that do not implement `SignedRoot`. +pub struct SszStaticHandler(PhantomData<(T, E)>); + +/// Handler for SSZ types that do implement `SignedRoot`. +pub struct SszStaticSRHandler(PhantomData<(T, E)>); + +impl Handler for SszStaticHandler +where + T: cases::SszStaticType + TypeName, + E: TypeName, +{ + type Case = cases::SszStatic; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "ssz_static" + } + + fn handler_name() -> &'static str { + T::name() + } +} + +impl Handler for SszStaticSRHandler +where + T: cases::SszStaticType + SignedRoot + TypeName, + E: TypeName, +{ + type Case = cases::SszStaticSR; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "ssz_static" + } + + fn handler_name() -> &'static str { + T::name() + } +} diff --git a/tests/ef_tests/src/lib.rs b/tests/ef_tests/src/lib.rs index fdd4e7b85..cc17c3ea4 100644 --- a/tests/ef_tests/src/lib.rs +++ b/tests/ef_tests/src/lib.rs @@ -4,6 +4,7 @@ pub use case_result::CaseResult; pub use cases::Case; pub use doc::Doc; pub use error::Error; +pub use handler::*; pub use yaml_decode::YamlDecode; mod bls_setting; @@ -12,6 +13,8 @@ mod cases; mod doc; mod doc_header; mod error; +mod handler; +mod type_name; mod yaml_decode; /// Defined where an object can return the results of some test(s) adhering to the Ethereum diff --git a/tests/ef_tests/src/type_name.rs b/tests/ef_tests/src/type_name.rs new file mode 100644 index 000000000..fe55a7f5f --- /dev/null +++ b/tests/ef_tests/src/type_name.rs @@ -0,0 +1,61 @@ +//! Mapping from types to canonical string identifiers used in testing. +use types::*; + +pub trait TypeName { + fn name() -> &'static str; +} + +impl TypeName for MinimalEthSpec { + fn name() -> &'static str { + "minimal" + } +} + +impl TypeName for MainnetEthSpec { + fn name() -> &'static str { + "mainnet" + } +} + +macro_rules! impl_name { + ($typ:ident) => { + impl TypeName for $typ { + fn name() -> &'static str { + stringify!($typ) + } + } + }; +} + +macro_rules! impl_name_generic { + ($typ:ident) => { + impl TypeName for $typ { + fn name() -> &'static str { + stringify!($typ) + } + } + }; +} + +impl_name_generic!(Attestation); +impl_name!(AttestationData); +impl_name!(AttestationDataAndCustodyBit); +impl_name_generic!(AttesterSlashing); +impl_name_generic!(BeaconBlock); +impl_name_generic!(BeaconBlockBody); +impl_name!(BeaconBlockHeader); +impl_name_generic!(BeaconState); +impl_name!(Checkpoint); +impl_name_generic!(CompactCommittee); +impl_name!(Crosslink); +impl_name!(Deposit); +impl_name!(DepositData); +impl_name!(Eth1Data); +impl_name!(Fork); +impl_name_generic!(HistoricalBatch); +impl_name_generic!(IndexedAttestation); +impl_name_generic!(PendingAttestation); +impl_name!(ProposerSlashing); +impl_name!(Transfer); +impl_name!(Validator); +impl_name!(VoluntaryExit); diff --git a/tests/ef_tests/src/yaml_decode.rs b/tests/ef_tests/src/yaml_decode.rs index c89dd92a9..af122fb0c 100644 --- a/tests/ef_tests/src/yaml_decode.rs +++ b/tests/ef_tests/src/yaml_decode.rs @@ -1,14 +1,20 @@ use super::*; use ethereum_types::{U128, U256}; +use std::fs; +use std::path::Path; use types::Fork; -mod utils; - -pub use utils::*; - pub trait YamlDecode: Sized { /// Decode an object from the test specification YAML. fn yaml_decode(string: &str) -> Result; + + fn yaml_decode_file(path: &Path) -> Result { + fs::read_to_string(path) + .map_err(|e| { + Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e)) + }) + .and_then(|s| Self::yaml_decode(&s)) + } } /// Basic types can general be decoded with the `parse` fn if they implement `str::FromStr`. diff --git a/tests/ef_tests/src/yaml_decode/utils.rs b/tests/ef_tests/src/yaml_decode/utils.rs deleted file mode 100644 index 7b6caac72..000000000 --- a/tests/ef_tests/src/yaml_decode/utils.rs +++ /dev/null @@ -1,10 +0,0 @@ -pub fn yaml_split_header_and_cases(mut yaml: String) -> (String, String) { - let test_cases_start = yaml.find("\ntest_cases:\n").unwrap(); - // + 1 to skip the \n we used for matching. - let mut test_cases = yaml.split_off(test_cases_start + 1); - - let end_of_first_line = test_cases.find('\n').unwrap(); - let test_cases = test_cases.split_off(end_of_first_line + 1); - - (yaml, test_cases) -} diff --git a/tests/ef_tests/tests/tests.rs b/tests/ef_tests/tests/tests.rs index deb699e78..000c53330 100644 --- a/tests/ef_tests/tests/tests.rs +++ b/tests/ef_tests/tests/tests.rs @@ -1,12 +1,20 @@ use ef_tests::*; use rayon::prelude::*; use std::path::{Path, PathBuf}; +use types::{ + Attestation, AttestationData, AttestationDataAndCustodyBit, AttesterSlashing, BeaconBlock, + BeaconBlockBody, BeaconBlockHeader, BeaconState, Checkpoint, CompactCommittee, Crosslink, + Deposit, DepositData, Eth1Data, Fork, HistoricalBatch, IndexedAttestation, MainnetEthSpec, + MinimalEthSpec, PendingAttestation, ProposerSlashing, Transfer, Validator, VoluntaryExit, +}; use walkdir::WalkDir; fn yaml_files_in_test_dir(dir: &Path) -> Vec { let base_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("eth2.0-spec-tests") .join("tests") + .join("general") + .join("phase0") .join(dir); assert!( @@ -155,12 +163,107 @@ fn sanity_slots() { #[cfg(not(feature = "fake_crypto"))] fn bls() { yaml_files_in_test_dir(&Path::new("bls")) - .into_par_iter() + .into_iter() .for_each(|file| { Doc::assert_tests_pass(file); }); } +#[test] +#[cfg(not(feature = "fake_crypto"))] +fn bls_aggregate_pubkeys() { + BlsAggregatePubkeysHandler::run(); +} + +#[test] +#[cfg(not(feature = "fake_crypto"))] +fn bls_aggregate_sigs() { + BlsAggregateSigsHandler::run(); +} + +#[test] +#[cfg(not(feature = "fake_crypto"))] +fn bls_msg_hash_g2_compressed() { + BlsG2CompressedHandler::run(); +} + +#[test] +#[cfg(not(feature = "fake_crypto"))] +fn bls_priv_to_pub() { + BlsPrivToPubHandler::run(); +} + +#[test] +#[cfg(not(feature = "fake_crypto"))] +fn bls_sign_msg() { + BlsSignMsgHandler::run(); +} + +macro_rules! ssz_static_test { + // Signed-root + ($test_name:ident, $typ:ident$(<$generics:tt>)?, SR) => { + ssz_static_test!($test_name, SszStaticSRHandler, $typ$(<$generics>)?); + }; + // Non-signed root + ($test_name:ident, $typ:ident$(<$generics:tt>)?) => { + ssz_static_test!($test_name, SszStaticHandler, $typ$(<$generics>)?); + }; + // Generic + ($test_name:ident, $handler:ident, $typ:ident<_>) => { + ssz_static_test!( + $test_name, $handler, { + ($typ, MinimalEthSpec), + ($typ, MainnetEthSpec) + } + ); + }; + // Non-generic + ($test_name:ident, $handler:ident, $typ:ident) => { + ssz_static_test!( + $test_name, $handler, { + ($typ, MinimalEthSpec), + ($typ, MainnetEthSpec) + } + ); + }; + // Base case + ($test_name:ident, $handler:ident, { $(($typ:ty, $spec:ident)),+ }) => { + #[test] + #[cfg(feature = "fake_crypto")] + fn $test_name() { + $( + $handler::<$typ, $spec>::run(); + )+ + } + }; +} + +ssz_static_test!(ssz_static_attestation, Attestation<_>, SR); +ssz_static_test!(ssz_static_attestation_data, AttestationData); +ssz_static_test!( + ssz_static_attestation_data_and_custody_bit, + AttestationDataAndCustodyBit +); +ssz_static_test!(ssz_static_attester_slashing, AttesterSlashing<_>); +ssz_static_test!(ssz_static_beacon_block, BeaconBlock<_>, SR); +ssz_static_test!(ssz_static_beacon_block_body, BeaconBlockBody<_>); +ssz_static_test!(ssz_static_beacon_block_header, BeaconBlockHeader, SR); +ssz_static_test!(ssz_static_beacon_state, BeaconState<_>); +ssz_static_test!(ssz_static_checkpoint, Checkpoint); +ssz_static_test!(ssz_static_compact_committee, CompactCommittee<_>); +ssz_static_test!(ssz_static_crosslink, Crosslink); +ssz_static_test!(ssz_static_deposit, Deposit); +ssz_static_test!(ssz_static_deposit_data, DepositData, SR); +ssz_static_test!(ssz_static_eth1_data, Eth1Data); +ssz_static_test!(ssz_static_fork, Fork); +ssz_static_test!(ssz_static_historical_batch, HistoricalBatch<_>); +ssz_static_test!(ssz_static_indexed_attestation, IndexedAttestation<_>, SR); +ssz_static_test!(ssz_static_pending_attestation, PendingAttestation<_>); +ssz_static_test!(ssz_static_proposer_slashing, ProposerSlashing); +ssz_static_test!(ssz_static_transfer, Transfer, SR); +ssz_static_test!(ssz_static_validator, Validator); +ssz_static_test!(ssz_static_voluntary_exit, VoluntaryExit, SR); + #[test] fn epoch_processing_justification_and_finalization() { yaml_files_in_test_dir(&Path::new("epoch_processing").join("justification_and_finalization")) From 81cafdc804fb58962e08c291f24e8a2f556ded7b Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 29 Aug 2019 17:41:20 +1000 Subject: [PATCH 05/12] Shuffling and sanity tests --- eth2/types/src/checkpoint.rs | 1 - eth2/utils/bls/Cargo.toml | 2 +- tests/ef_tests/src/case_result.rs | 3 + tests/ef_tests/src/cases.rs | 49 +--- tests/ef_tests/src/cases/bls_g2_compressed.rs | 13 +- tests/ef_tests/src/cases/bls_sign_msg.rs | 13 +- tests/ef_tests/src/cases/sanity_blocks.rs | 51 +++- tests/ef_tests/src/cases/sanity_slots.rs | 52 +++- tests/ef_tests/src/cases/shuffling.rs | 14 +- tests/ef_tests/src/doc.rs | 274 ------------------ tests/ef_tests/src/doc_header.rs | 12 - tests/ef_tests/src/handler.rs | 58 +++- tests/ef_tests/src/lib.rs | 4 +- tests/ef_tests/src/results.rs | 91 ++++++ tests/ef_tests/src/yaml_decode.rs | 28 ++ tests/ef_tests/tests/tests.rs | 30 +- 16 files changed, 311 insertions(+), 384 deletions(-) delete mode 100644 tests/ef_tests/src/doc.rs delete mode 100644 tests/ef_tests/src/doc_header.rs create mode 100644 tests/ef_tests/src/results.rs diff --git a/eth2/types/src/checkpoint.rs b/eth2/types/src/checkpoint.rs index 0c7001921..d5d40fa67 100644 --- a/eth2/types/src/checkpoint.rs +++ b/eth2/types/src/checkpoint.rs @@ -3,7 +3,6 @@ use crate::{Epoch, Hash256}; use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; -use tree_hash::TreeHash; use tree_hash_derive::TreeHash; /// Casper FFG checkpoint, used in attestations. diff --git a/eth2/utils/bls/Cargo.toml b/eth2/utils/bls/Cargo.toml index 5989dce07..15f087b80 100644 --- a/eth2/utils/bls/Cargo.toml +++ b/eth2/utils/bls/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Paul Hauner "] edition = "2018" [dependencies] -milagro_bls = { git = "https://github.com/sigp/milagro_bls", tag = "v0.9.0" } +milagro_bls = { git = "https://github.com/michaelsproul/milagro_bls", branch = "little-endian" } eth2_hashing = { path = "../eth2_hashing" } hex = "0.3" rand = "^0.5" diff --git a/tests/ef_tests/src/case_result.rs b/tests/ef_tests/src/case_result.rs index 88fd353a1..add428ec5 100644 --- a/tests/ef_tests/src/case_result.rs +++ b/tests/ef_tests/src/case_result.rs @@ -1,6 +1,7 @@ use super::*; use compare_fields::{CompareFields, Comparison, FieldComparison}; use std::fmt::Debug; +use std::path::PathBuf; use types::BeaconState; pub const MAX_VALUE_STRING_LEN: usize = 500; @@ -9,6 +10,7 @@ pub const MAX_VALUE_STRING_LEN: usize = 500; pub struct CaseResult { pub case_index: usize, pub desc: String, + pub path: PathBuf, pub result: Result<(), Error>, } @@ -17,6 +19,7 @@ impl CaseResult { CaseResult { case_index, desc: case.description(), + path: case.path().into(), result, } } diff --git a/tests/ef_tests/src/cases.rs b/tests/ef_tests/src/cases.rs index 7f6ffb0c4..1216b8728 100644 --- a/tests/ef_tests/src/cases.rs +++ b/tests/ef_tests/src/cases.rs @@ -67,6 +67,11 @@ pub trait Case: Debug { "no description".to_string() } + /// Path to the directory for this test case. + fn path(&self) -> &Path { + Path::new("") + } + /// Execute a test and return the result. /// /// `case_index` reports the index of the case in the set of test cases. It is not strictly @@ -76,19 +81,13 @@ pub trait Case: Debug { pub trait BlsCase: serde::de::DeserializeOwned {} -impl YamlDecode for T -where - T: BlsCase, -{ +impl YamlDecode for T { fn yaml_decode(string: &str) -> Result { serde_yaml::from_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) } } -impl LoadCase for T -where - T: BlsCase, -{ +impl LoadCase for T { fn load_from_dir(path: &Path) -> Result { Self::yaml_decode_file(&path.join("data.yaml")) } @@ -111,37 +110,3 @@ where .collect() } } - -// FIXME(michael): delete this -impl YamlDecode for Cases { - /// Decodes a YAML list of test cases - fn yaml_decode(yaml: &str) -> Result { - let mut p = 0; - let mut elems: Vec<&str> = yaml - .match_indices("\n- ") - // Skip the `\n` used for matching a new line - .map(|(i, _)| i + 1) - .map(|i| { - let yaml_element = &yaml[p..i]; - p = i; - - yaml_element - }) - .collect(); - - elems.push(&yaml[p..]); - - let test_cases = elems - .iter() - .map(|s| { - // Remove the `- ` prefix. - let s = &s[2..]; - // Remove a single level of indenting. - s.replace("\n ", "\n") - }) - .map(|s| T::yaml_decode(&s.to_string()).unwrap()) - .collect(); - - Ok(Self { test_cases }) - } -} diff --git a/tests/ef_tests/src/cases/bls_g2_compressed.rs b/tests/ef_tests/src/cases/bls_g2_compressed.rs index 547d8d03a..f8381f5a7 100644 --- a/tests/ef_tests/src/cases/bls_g2_compressed.rs +++ b/tests/ef_tests/src/cases/bls_g2_compressed.rs @@ -41,14 +41,9 @@ impl Case for BlsG2Compressed { } } -// Converts a vector to u64 (from big endian) +// Converts a vector to u64 (from little endian) fn bytes_to_u64(array: &[u8]) -> u64 { - let mut result: u64 = 0; - for (i, value) in array.iter().rev().enumerate() { - if i == 8 { - break; - } - result += u64::pow(2, i as u32 * 8) * u64::from(*value); - } - result + let mut bytes = [0u8; 8]; + bytes.copy_from_slice(array); + u64::from_le_bytes(bytes) } diff --git a/tests/ef_tests/src/cases/bls_sign_msg.rs b/tests/ef_tests/src/cases/bls_sign_msg.rs index 476ecdefb..18e90896b 100644 --- a/tests/ef_tests/src/cases/bls_sign_msg.rs +++ b/tests/ef_tests/src/cases/bls_sign_msg.rs @@ -41,16 +41,11 @@ impl Case for BlsSign { } } -// Converts a vector to u64 (from big endian) +// Converts a vector to u64 (from little endian) fn bytes_to_u64(array: &[u8]) -> u64 { - let mut result: u64 = 0; - for (i, value) in array.iter().rev().enumerate() { - if i == 8 { - break; - } - result += u64::pow(2, i as u32 * 8) * u64::from(*value); - } - result + let mut bytes = [0u8; 8]; + bytes.copy_from_slice(array); + u64::from_le_bytes(bytes) } // Increase the size of an array to 48 bytes diff --git a/tests/ef_tests/src/cases/sanity_blocks.rs b/tests/ef_tests/src/cases/sanity_blocks.rs index cd9008fda..d88d8f295 100644 --- a/tests/ef_tests/src/cases/sanity_blocks.rs +++ b/tests/ef_tests/src/cases/sanity_blocks.rs @@ -1,35 +1,72 @@ use super::*; use crate::bls_setting::BlsSetting; use crate::case_result::compare_beacon_state_results_without_caches; +use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::{ per_block_processing, per_slot_processing, BlockInvalid, BlockProcessingError, }; +use std::path::PathBuf; use types::{BeaconBlock, BeaconState, EthSpec, RelativeEpoch}; +#[derive(Debug, Clone, Deserialize)] +pub struct Metadata { + pub description: Option, + pub bls_setting: Option, + pub blocks_count: usize, +} + #[derive(Debug, Clone, Deserialize)] #[serde(bound = "E: EthSpec")] pub struct SanityBlocks { - pub description: String, - pub bls_setting: Option, + pub path: PathBuf, + pub metadata: Metadata, pub pre: BeaconState, pub blocks: Vec>, pub post: Option>, } -impl YamlDecode for SanityBlocks { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) +impl LoadCase for SanityBlocks { + fn load_from_dir(path: &Path) -> Result { + let metadata: Metadata = yaml_decode_file(&path.join("meta.yaml"))?; + let pre = ssz_decode_file(&path.join("pre.ssz"))?; + let blocks: Vec> = (0..metadata.blocks_count) + .map(|i| { + let filename = format!("blocks_{}.ssz", i); + ssz_decode_file(&path.join(filename)) + }) + .collect::>()?; + let post_file = path.join("post.ssz"); + let post = if post_file.is_file() { + Some(ssz_decode_file(&post_file)?) + } else { + None + }; + + Ok(Self { + path: path.into(), + metadata, + pre, + blocks, + post, + }) } } impl Case for SanityBlocks { fn description(&self) -> String { - self.description.clone() + self.metadata + .description + .clone() + .unwrap_or_else(String::new) + } + + fn path(&self) -> &Path { + &self.path } fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; + self.metadata.bls_setting.unwrap_or_default().check()?; let mut state = self.pre.clone(); let mut expected = self.post.clone(); diff --git a/tests/ef_tests/src/cases/sanity_slots.rs b/tests/ef_tests/src/cases/sanity_slots.rs index fbce1a06a..27c6c13c3 100644 --- a/tests/ef_tests/src/cases/sanity_slots.rs +++ b/tests/ef_tests/src/cases/sanity_slots.rs @@ -1,30 +1,70 @@ use super::*; +use crate::bls_setting::BlsSetting; use crate::case_result::compare_beacon_state_results_without_caches; +use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::per_slot_processing; +use std::path::PathBuf; use types::{BeaconState, EthSpec}; +#[derive(Debug, Clone, Default, Deserialize)] +pub struct Metadata { + pub description: Option, + pub bls_setting: Option, +} + #[derive(Debug, Clone, Deserialize)] #[serde(bound = "E: EthSpec")] pub struct SanitySlots { - pub description: String, + pub path: PathBuf, + pub metadata: Metadata, pub pre: BeaconState, - pub slots: usize, + pub slots: u64, pub post: Option>, } -impl YamlDecode for SanitySlots { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) +impl LoadCase for SanitySlots { + fn load_from_dir(path: &Path) -> Result { + let metadata_path = path.join("meta.yaml"); + let metadata: Metadata = if metadata_path.is_file() { + yaml_decode_file(&path.join("meta.yaml"))? + } else { + Metadata::default() + }; + let pre = ssz_decode_file(&path.join("pre.ssz"))?; + let slots: u64 = yaml_decode_file(&path.join("slots.yaml"))?; + let post_file = path.join("post.ssz"); + let post = if post_file.is_file() { + Some(ssz_decode_file(&post_file)?) + } else { + None + }; + + Ok(Self { + path: path.into(), + metadata, + pre, + slots, + post, + }) } } impl Case for SanitySlots { fn description(&self) -> String { - self.description.clone() + self.metadata + .description + .clone() + .unwrap_or_else(String::new) + } + + fn path(&self) -> &Path { + &self.path } fn result(&self, _case_index: usize) -> Result<(), Error> { + self.metadata.bls_setting.unwrap_or_default().check()?; + let mut state = self.pre.clone(); let mut expected = self.post.clone(); let spec = &E::default_spec(); diff --git a/tests/ef_tests/src/cases/shuffling.rs b/tests/ef_tests/src/cases/shuffling.rs index d7ff40e59..c0595e584 100644 --- a/tests/ef_tests/src/cases/shuffling.rs +++ b/tests/ef_tests/src/cases/shuffling.rs @@ -8,7 +8,7 @@ use swap_or_not_shuffle::{get_permutated_index, shuffle_list}; pub struct Shuffling { pub seed: String, pub count: usize, - pub shuffled: Vec, + pub mapping: Vec, #[serde(skip)] _phantom: PhantomData, } @@ -19,10 +19,16 @@ impl YamlDecode for Shuffling { } } +impl LoadCase for Shuffling { + fn load_from_dir(path: &Path) -> Result { + Self::yaml_decode_file(&path.join("mapping.yaml")) + } +} + impl Case for Shuffling { fn result(&self, _case_index: usize) -> Result<(), Error> { if self.count == 0 { - compare_result::<_, Error>(&Ok(vec![]), &Some(self.shuffled.clone()))?; + compare_result::<_, Error>(&Ok(vec![]), &Some(self.mapping.clone()))?; } else { let spec = T::default_spec(); let seed = hex::decode(&self.seed[2..]) @@ -34,12 +40,12 @@ impl Case for Shuffling { get_permutated_index(i, self.count, &seed, spec.shuffle_round_count).unwrap() }) .collect(); - compare_result::<_, Error>(&Ok(shuffling), &Some(self.shuffled.clone()))?; + compare_result::<_, Error>(&Ok(shuffling), &Some(self.mapping.clone()))?; // Test "shuffle_list" let input: Vec = (0..self.count).collect(); let shuffling = shuffle_list(input, spec.shuffle_round_count, &seed, false).unwrap(); - compare_result::<_, Error>(&Ok(shuffling), &Some(self.shuffled.clone()))?; + compare_result::<_, Error>(&Ok(shuffling), &Some(self.mapping.clone()))?; } Ok(()) diff --git a/tests/ef_tests/src/doc.rs b/tests/ef_tests/src/doc.rs deleted file mode 100644 index f3a41697e..000000000 --- a/tests/ef_tests/src/doc.rs +++ /dev/null @@ -1,274 +0,0 @@ -use crate::case_result::CaseResult; -use crate::cases::*; -use crate::doc_header::DocHeader; -use crate::error::Error; -use crate::yaml_decode::YamlDecode; -use crate::EfTest; -use serde_derive::Deserialize; -use std::{ - fs::File, - io::prelude::*, - path::{Path, PathBuf}, -}; -use types::{MainnetEthSpec, MinimalEthSpec}; - -#[derive(Debug, Deserialize)] -pub struct Doc { - pub header_yaml: String, - pub cases_yaml: String, - pub path: PathBuf, -} - -impl Doc { - fn from_path(path: PathBuf) -> Self { - let mut file = File::open(path.clone()).unwrap(); - - let mut cases_yaml = String::new(); - file.read_to_string(&mut cases_yaml).unwrap(); - - Self { - cases_yaml, - path, - header_yaml: String::new(), - } - } - - pub fn test_results(&self) -> Vec { - let header: DocHeader = serde_yaml::from_str(&self.header_yaml.as_str()).unwrap(); - - match ( - header.runner.as_ref(), - header.handler.as_ref(), - header.config.as_ref(), - ) { - ("ssz", "uint", _) => run_test::(self), - ("sanity", "slots", "minimal") => run_test::>(self), - // FIXME: skipped due to compact committees issue - ("sanity", "slots", "mainnet") => vec![], // run_test::>(self), - ("sanity", "blocks", "minimal") => run_test::>(self), - // FIXME: skipped due to compact committees issue - ("sanity", "blocks", "mainnet") => vec![], // run_test::>(self), - ("shuffling", "core", "minimal") => run_test::>(self), - ("shuffling", "core", "mainnet") => run_test::>(self), - ("bls", "aggregate_pubkeys", "mainnet") => run_test::(self), - ("bls", "aggregate_sigs", "mainnet") => run_test::(self), - ("bls", "msg_hash_compressed", "mainnet") => run_test::(self), - // Note this test fails due to a difference in our internal representations. It does - // not effect verification or external representation. - // - // It is skipped. - ("bls", "msg_hash_uncompressed", "mainnet") => vec![], - ("bls", "priv_to_pub", "mainnet") => run_test::(self), - ("bls", "sign_msg", "mainnet") => run_test::(self), - ("operations", "deposit", "mainnet") => { - run_test::>(self) - } - ("operations", "deposit", "minimal") => { - run_test::>(self) - } - ("operations", "transfer", "mainnet") => { - run_test::>(self) - } - ("operations", "transfer", "minimal") => { - run_test::>(self) - } - ("operations", "voluntary_exit", "mainnet") => { - run_test::>(self) - } - ("operations", "voluntary_exit", "minimal") => { - run_test::>(self) - } - ("operations", "proposer_slashing", "mainnet") => { - run_test::>(self) - } - ("operations", "proposer_slashing", "minimal") => { - run_test::>(self) - } - ("operations", "attester_slashing", "mainnet") => { - run_test::>(self) - } - ("operations", "attester_slashing", "minimal") => { - run_test::>(self) - } - ("operations", "attestation", "mainnet") => { - run_test::>(self) - } - ("operations", "attestation", "minimal") => { - run_test::>(self) - } - ("operations", "block_header", "mainnet") => { - run_test::>(self) - } - ("operations", "block_header", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "crosslinks", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "crosslinks", "mainnet") => { - run_test::>(self) - } - ("epoch_processing", "registry_updates", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "registry_updates", "mainnet") => { - run_test::>(self) - } - ("epoch_processing", "justification_and_finalization", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "justification_and_finalization", "mainnet") => { - run_test::>(self) - } - ("epoch_processing", "slashings", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "slashings", "mainnet") => { - run_test::>(self) - } - ("epoch_processing", "final_updates", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "final_updates", "mainnet") => { - vec![] - // FIXME: skipped due to compact committees issue - // run_test::>(self) - } - ("genesis", "initialization", "minimal") => { - run_test::>(self) - } - ("genesis", "initialization", "mainnet") => { - run_test::>(self) - } - ("genesis", "validity", "minimal") => run_test::>(self), - ("genesis", "validity", "mainnet") => run_test::>(self), - (runner, handler, config) => panic!( - "No implementation for runner: \"{}\", handler: \"{}\", config: \"{}\"", - runner, handler, config - ), - } - } - - pub fn assert_tests_pass(path: PathBuf) { - let doc = Self::from_path(path); - let results = doc.test_results(); - - let (failed, skipped_bls, skipped_known_failures) = categorize_results(&results); - - if failed.len() + skipped_known_failures.len() > 0 { - print_results( - &doc, - &failed, - &skipped_bls, - &skipped_known_failures, - &results, - ); - if !failed.is_empty() { - panic!("Tests failed (see above)"); - } - } else { - println!("Passed {} tests in {:?}", results.len(), doc.path); - } - } -} - -pub fn assert_tests_pass(path: &Path, results: &[CaseResult]) { - let doc = Doc { - header_yaml: String::new(), - cases_yaml: String::new(), - path: path.into(), - }; - - let (failed, skipped_bls, skipped_known_failures) = categorize_results(results); - - if failed.len() + skipped_known_failures.len() > 0 { - print_results( - &doc, - &failed, - &skipped_bls, - &skipped_known_failures, - &results, - ); - if !failed.is_empty() { - panic!("Tests failed (see above)"); - } - } else { - println!("Passed {} tests in {}", results.len(), path.display()); - } -} - -pub fn run_test(_: &Doc) -> Vec -where - Cases: EfTest + YamlDecode, -{ - panic!("FIXME(michael): delete this") -} - -pub fn categorize_results( - results: &[CaseResult], -) -> (Vec<&CaseResult>, Vec<&CaseResult>, Vec<&CaseResult>) { - let mut failed = vec![]; - let mut skipped_bls = vec![]; - let mut skipped_known_failures = vec![]; - - for case in results { - match case.result.as_ref().err() { - Some(Error::SkippedBls) => skipped_bls.push(case), - Some(Error::SkippedKnownFailure) => skipped_known_failures.push(case), - Some(_) => failed.push(case), - None => (), - } - } - - (failed, skipped_bls, skipped_known_failures) -} - -pub fn print_results( - doc: &Doc, - failed: &[&CaseResult], - skipped_bls: &[&CaseResult], - skipped_known_failures: &[&CaseResult], - results: &[CaseResult], -) { - println!("--------------------------------------------------"); - println!( - "Test {}", - if failed.is_empty() { - "Result" - } else { - "Failure" - } - ); - println!("Title: TODO"); - println!("File: {:?}", doc.path); - println!( - "{} tests, {} failed, {} skipped (known failure), {} skipped (bls), {} passed. (See below for errors)", - results.len(), - failed.len(), - skipped_known_failures.len(), - skipped_bls.len(), - results.len() - skipped_bls.len() - skipped_known_failures.len() - failed.len() - ); - println!(); - - for case in skipped_known_failures { - println!("-------"); - println!( - "case[{}] ({}) skipped because it's a known failure", - case.case_index, case.desc, - ); - } - for failure in failed { - let error = failure.result.clone().unwrap_err(); - - println!("-------"); - println!( - "case[{}] ({}) failed with {}:", - failure.case_index, - failure.desc, - error.name() - ); - println!("{}", error.message()); - } - println!(); -} diff --git a/tests/ef_tests/src/doc_header.rs b/tests/ef_tests/src/doc_header.rs deleted file mode 100644 index c0d6d3276..000000000 --- a/tests/ef_tests/src/doc_header.rs +++ /dev/null @@ -1,12 +0,0 @@ -use serde_derive::Deserialize; - -#[derive(Debug, Deserialize)] -pub struct DocHeader { - pub title: String, - pub summary: String, - pub forks_timeline: String, - pub forks: Vec, - pub config: String, - pub runner: String, - pub handler: String, -} diff --git a/tests/ef_tests/src/handler.rs b/tests/ef_tests/src/handler.rs index 1dac988ac..f66dc7b18 100644 --- a/tests/ef_tests/src/handler.rs +++ b/tests/ef_tests/src/handler.rs @@ -5,6 +5,7 @@ use std::fs; use std::marker::PhantomData; use std::path::PathBuf; use tree_hash::SignedRoot; +use types::EthSpec; pub trait Handler { type Case: Case + LoadCase; @@ -47,7 +48,8 @@ pub trait Handler { let results = Cases { test_cases }.test_results(); - crate::doc::assert_tests_pass(&handler_path, &results); + let name = format!("{}/{}", Self::runner_name(), Self::handler_name()); + crate::results::assert_tests_pass(&name, &handler_path, &results); } } @@ -128,3 +130,57 @@ where T::name() } } + +pub struct ShufflingHandler(PhantomData); + +impl Handler for ShufflingHandler { + type Case = cases::Shuffling; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "shuffling" + } + + fn handler_name() -> &'static str { + "core" + } +} + +pub struct SanityBlocksHandler(PhantomData); + +impl Handler for SanityBlocksHandler { + type Case = cases::SanityBlocks; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "sanity" + } + + fn handler_name() -> &'static str { + "blocks" + } +} + +pub struct SanitySlotsHandler(PhantomData); + +impl Handler for SanitySlotsHandler { + type Case = cases::SanitySlots; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "sanity" + } + + fn handler_name() -> &'static str { + "slots" + } +} diff --git a/tests/ef_tests/src/lib.rs b/tests/ef_tests/src/lib.rs index cc17c3ea4..06e01fc03 100644 --- a/tests/ef_tests/src/lib.rs +++ b/tests/ef_tests/src/lib.rs @@ -2,7 +2,6 @@ use types::EthSpec; pub use case_result::CaseResult; pub use cases::Case; -pub use doc::Doc; pub use error::Error; pub use handler::*; pub use yaml_decode::YamlDecode; @@ -10,10 +9,9 @@ pub use yaml_decode::YamlDecode; mod bls_setting; mod case_result; mod cases; -mod doc; -mod doc_header; mod error; mod handler; +mod results; mod type_name; mod yaml_decode; diff --git a/tests/ef_tests/src/results.rs b/tests/ef_tests/src/results.rs new file mode 100644 index 000000000..20e59f7b3 --- /dev/null +++ b/tests/ef_tests/src/results.rs @@ -0,0 +1,91 @@ +use crate::case_result::CaseResult; +use crate::error::Error; +use std::path::Path; + +pub fn assert_tests_pass(handler_name: &str, path: &Path, results: &[CaseResult]) { + let (failed, skipped_bls, skipped_known_failures) = categorize_results(results); + + if failed.len() + skipped_known_failures.len() > 0 { + print_results( + handler_name, + &failed, + &skipped_bls, + &skipped_known_failures, + &results, + ); + if !failed.is_empty() { + panic!("Tests failed (see above)"); + } + } else { + println!("Passed {} tests in {}", results.len(), path.display()); + } +} + +pub fn categorize_results( + results: &[CaseResult], +) -> (Vec<&CaseResult>, Vec<&CaseResult>, Vec<&CaseResult>) { + let mut failed = vec![]; + let mut skipped_bls = vec![]; + let mut skipped_known_failures = vec![]; + + for case in results { + match case.result.as_ref().err() { + Some(Error::SkippedBls) => skipped_bls.push(case), + Some(Error::SkippedKnownFailure) => skipped_known_failures.push(case), + Some(_) => failed.push(case), + None => (), + } + } + + (failed, skipped_bls, skipped_known_failures) +} + +pub fn print_results( + handler_name: &str, + failed: &[&CaseResult], + skipped_bls: &[&CaseResult], + skipped_known_failures: &[&CaseResult], + results: &[CaseResult], +) { + println!("--------------------------------------------------"); + println!( + "Test {}", + if failed.is_empty() { + "Result" + } else { + "Failure" + } + ); + println!("Title: {}", handler_name); + println!( + "{} tests, {} failed, {} skipped (known failure), {} skipped (bls), {} passed. (See below for errors)", + results.len(), + failed.len(), + skipped_known_failures.len(), + skipped_bls.len(), + results.len() - skipped_bls.len() - skipped_known_failures.len() - failed.len() + ); + println!(); + + for case in skipped_known_failures { + println!("-------"); + println!( + "case ({}) from {} skipped because it's a known failure", + case.desc, + case.path.display() + ); + } + for failure in failed { + let error = failure.result.clone().unwrap_err(); + + println!("-------"); + println!( + "case ({}) from {} failed with {}:", + failure.desc, + failure.path.display(), + error.name() + ); + println!("{}", error.message()); + } + println!(); +} diff --git a/tests/ef_tests/src/yaml_decode.rs b/tests/ef_tests/src/yaml_decode.rs index af122fb0c..83a162930 100644 --- a/tests/ef_tests/src/yaml_decode.rs +++ b/tests/ef_tests/src/yaml_decode.rs @@ -4,6 +4,34 @@ use std::fs; use std::path::Path; use types::Fork; +pub fn yaml_decode(string: &str) -> Result { + serde_yaml::from_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) +} + +pub fn yaml_decode_file(path: &Path) -> Result { + fs::read_to_string(path) + .map_err(|e| { + Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e)) + }) + .and_then(|s| yaml_decode(&s)) +} + +pub fn ssz_decode_file(path: &Path) -> Result { + fs::read(path) + .map_err(|e| { + Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e)) + }) + .and_then(|s| { + T::from_ssz_bytes(&s).map_err(|e| { + Error::FailedToParseTest(format!( + "Unable to parse SSZ at {}: {:?}", + path.display(), + e + )) + }) + }) +} + pub trait YamlDecode: Sized { /// Decode an object from the test specification YAML. fn yaml_decode(string: &str) -> Result; diff --git a/tests/ef_tests/tests/tests.rs b/tests/ef_tests/tests/tests.rs index 000c53330..0fb751c9e 100644 --- a/tests/ef_tests/tests/tests.rs +++ b/tests/ef_tests/tests/tests.rs @@ -48,6 +48,7 @@ fn yaml_files_in_test_dir(dir: &Path) -> Vec { paths } +/* #[test] #[cfg(feature = "fake_crypto")] fn ssz_generic() { @@ -58,6 +59,7 @@ fn ssz_generic() { }); } + #[test] #[cfg(feature = "fake_crypto")] fn ssz_static() { @@ -67,16 +69,15 @@ fn ssz_static() { Doc::assert_tests_pass(file); }); } +*/ #[test] fn shuffling() { - yaml_files_in_test_dir(&Path::new("shuffling").join("core")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + ShufflingHandler::::run(); + ShufflingHandler::::run(); } +/* #[test] fn operations_deposit() { yaml_files_in_test_dir(&Path::new("operations").join("deposit")) @@ -140,25 +141,21 @@ fn operations_block_header() { Doc::assert_tests_pass(file); }); } +*/ #[test] fn sanity_blocks() { - yaml_files_in_test_dir(&Path::new("sanity").join("blocks")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + SanityBlocksHandler::::run(); + SanityBlocksHandler::::run(); } #[test] fn sanity_slots() { - yaml_files_in_test_dir(&Path::new("sanity").join("slots")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + SanitySlotsHandler::::run(); + SanitySlotsHandler::::run(); } +/* #[test] #[cfg(not(feature = "fake_crypto"))] fn bls() { @@ -168,6 +165,7 @@ fn bls() { Doc::assert_tests_pass(file); }); } +*/ #[test] #[cfg(not(feature = "fake_crypto"))] @@ -264,6 +262,7 @@ ssz_static_test!(ssz_static_transfer, Transfer, SR); ssz_static_test!(ssz_static_validator, Validator); ssz_static_test!(ssz_static_voluntary_exit, VoluntaryExit, SR); +/* #[test] fn epoch_processing_justification_and_finalization() { yaml_files_in_test_dir(&Path::new("epoch_processing").join("justification_and_finalization")) @@ -326,3 +325,4 @@ fn genesis_validity() { Doc::assert_tests_pass(file); }); } +*/ From 6cf9b3c1a41e253de9cc3ef339f451f62a461035 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 30 Aug 2019 13:29:26 +1000 Subject: [PATCH 06/12] Epoch processing tests --- .../src/per_epoch_processing.rs | 7 +- tests/ef_tests/src/cases.rs | 13 +- tests/ef_tests/src/cases/epoch_processing.rs | 147 ++++++++++++++++++ .../src/cases/epoch_processing_crosslinks.rs | 37 ----- .../cases/epoch_processing_final_updates.rs | 41 ----- ...ocessing_justification_and_finalization.rs | 46 ------ .../epoch_processing_registry_updates.rs | 38 ----- .../src/cases/epoch_processing_slashings.rs | 50 ------ tests/ef_tests/src/cases/sanity_slots.rs | 2 +- tests/ef_tests/src/handler.rs | 20 ++- tests/ef_tests/src/lib.rs | 3 + tests/ef_tests/src/type_name.rs | 75 +++++---- tests/ef_tests/tests/tests.rs | 37 ++--- 13 files changed, 225 insertions(+), 291 deletions(-) create mode 100644 tests/ef_tests/src/cases/epoch_processing.rs delete mode 100644 tests/ef_tests/src/cases/epoch_processing_crosslinks.rs delete mode 100644 tests/ef_tests/src/cases/epoch_processing_final_updates.rs delete mode 100644 tests/ef_tests/src/cases/epoch_processing_justification_and_finalization.rs delete mode 100644 tests/ef_tests/src/cases/epoch_processing_registry_updates.rs delete mode 100644 tests/ef_tests/src/cases/epoch_processing_slashings.rs diff --git a/eth2/state_processing/src/per_epoch_processing.rs b/eth2/state_processing/src/per_epoch_processing.rs index 08f42a229..f66ce4ea2 100644 --- a/eth2/state_processing/src/per_epoch_processing.rs +++ b/eth2/state_processing/src/per_epoch_processing.rs @@ -1,8 +1,5 @@ use crate::common::get_compact_committees_root; -use apply_rewards::process_rewards_and_penalties; use errors::EpochProcessingError as Error; -use process_slashings::process_slashings; -use registry_updates::process_registry_updates; use std::collections::HashMap; use tree_hash::TreeHash; use types::*; @@ -17,6 +14,10 @@ pub mod tests; pub mod validator_statuses; pub mod winning_root; +pub use apply_rewards::process_rewards_and_penalties; +pub use process_slashings::process_slashings; +pub use registry_updates::process_registry_updates; + /// Maps a shard to a winning root. /// /// It is generated during crosslink processing and later used to reward/penalize validators. diff --git a/tests/ef_tests/src/cases.rs b/tests/ef_tests/src/cases.rs index 1216b8728..e3ec54a7f 100644 --- a/tests/ef_tests/src/cases.rs +++ b/tests/ef_tests/src/cases.rs @@ -8,11 +8,7 @@ mod bls_g2_compressed; mod bls_g2_uncompressed; mod bls_priv_to_pub; mod bls_sign_msg; -mod epoch_processing_crosslinks; -mod epoch_processing_final_updates; -mod epoch_processing_justification_and_finalization; -mod epoch_processing_registry_updates; -mod epoch_processing_slashings; +mod epoch_processing; mod genesis_initialization; mod genesis_validity; mod operations_attestation; @@ -34,11 +30,7 @@ pub use bls_g2_compressed::*; pub use bls_g2_uncompressed::*; pub use bls_priv_to_pub::*; pub use bls_sign_msg::*; -pub use epoch_processing_crosslinks::*; -pub use epoch_processing_final_updates::*; -pub use epoch_processing_justification_and_finalization::*; -pub use epoch_processing_registry_updates::*; -pub use epoch_processing_slashings::*; +pub use epoch_processing::*; pub use genesis_initialization::*; pub use genesis_validity::*; pub use operations_attestation::*; @@ -69,6 +61,7 @@ pub trait Case: Debug { /// Path to the directory for this test case. fn path(&self) -> &Path { + // FIXME(michael): remove default impl Path::new("") } diff --git a/tests/ef_tests/src/cases/epoch_processing.rs b/tests/ef_tests/src/cases/epoch_processing.rs new file mode 100644 index 000000000..ac47ab236 --- /dev/null +++ b/tests/ef_tests/src/cases/epoch_processing.rs @@ -0,0 +1,147 @@ +use super::*; +use crate::bls_setting::BlsSetting; +use crate::case_result::compare_beacon_state_results_without_caches; +use crate::type_name; +use crate::type_name::TypeName; +use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; +use serde_derive::Deserialize; +use state_processing::per_epoch_processing::{ + errors::EpochProcessingError, process_crosslinks, process_final_updates, + process_justification_and_finalization, process_registry_updates, process_slashings, + validator_statuses::ValidatorStatuses, +}; +use std::marker::PhantomData; +use std::path::{Path, PathBuf}; +use types::{BeaconState, ChainSpec, EthSpec}; + +#[derive(Debug, Clone, Default, Deserialize)] +pub struct Metadata { + pub description: Option, + pub bls_setting: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(bound = "E: EthSpec")] +pub struct EpochProcessing> { + pub path: PathBuf, + pub metadata: Metadata, + pub pre: BeaconState, + pub post: Option>, + #[serde(skip_deserializing)] + _phantom: PhantomData, +} + +pub trait EpochTransition: TypeName + Debug { + fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError>; +} + +#[derive(Debug)] +pub struct JustificationAndFinalization; +#[derive(Debug)] +pub struct Crosslinks; +#[derive(Debug)] +pub struct RegistryUpdates; +#[derive(Debug)] +pub struct Slashings; +#[derive(Debug)] +pub struct FinalUpdates; + +type_name!( + JustificationAndFinalization, + "justification_and_finalization" +); +type_name!(Crosslinks, "crosslinks"); +type_name!(RegistryUpdates, "registry_updates"); +type_name!(Slashings, "slashings"); +type_name!(FinalUpdates, "final_updates"); + +impl EpochTransition for JustificationAndFinalization { + fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { + let mut validator_statuses = ValidatorStatuses::new(state, spec)?; + validator_statuses.process_attestations(state, spec)?; + process_justification_and_finalization(state, &validator_statuses.total_balances) + } +} + +impl EpochTransition for Crosslinks { + fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { + process_crosslinks(state, spec)?; + Ok(()) + } +} + +impl EpochTransition for RegistryUpdates { + fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { + process_registry_updates(state, spec) + } +} + +impl EpochTransition for Slashings { + fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { + let mut validator_statuses = ValidatorStatuses::new(&state, spec)?; + validator_statuses.process_attestations(&state, spec)?; + process_slashings(state, validator_statuses.total_balances.current_epoch, spec)?; + Ok(()) + } +} + +impl EpochTransition for FinalUpdates { + fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { + process_final_updates(state, spec) + } +} + +impl> LoadCase for EpochProcessing { + fn load_from_dir(path: &Path) -> Result { + let metadata_path = path.join("meta.yaml"); + let metadata: Metadata = if metadata_path.is_file() { + yaml_decode_file(&metadata_path)? + } else { + Metadata::default() + }; + let pre = ssz_decode_file(&path.join("pre.ssz"))?; + let post_file = path.join("post.ssz"); + let post = if post_file.is_file() { + Some(ssz_decode_file(&post_file)?) + } else { + None + }; + + Ok(Self { + path: path.into(), + metadata, + pre, + post, + _phantom: PhantomData, + }) + } +} + +impl> Case for EpochProcessing { + fn description(&self) -> String { + self.metadata + .description + .clone() + .unwrap_or_else(String::new) + } + + fn path(&self) -> &Path { + &self.path + } + + fn result(&self, _case_index: usize) -> Result<(), Error> { + let mut state = self.pre.clone(); + let mut expected = self.post.clone(); + + let spec = &E::default_spec(); + + let mut result = (|| { + // Processing requires the epoch cache. + state.build_all_caches(spec)?; + + T::run(&mut state, spec).map(|_| state) + })(); + + compare_beacon_state_results_without_caches(&mut result, &mut expected) + } +} diff --git a/tests/ef_tests/src/cases/epoch_processing_crosslinks.rs b/tests/ef_tests/src/cases/epoch_processing_crosslinks.rs deleted file mode 100644 index f2676d122..000000000 --- a/tests/ef_tests/src/cases/epoch_processing_crosslinks.rs +++ /dev/null @@ -1,37 +0,0 @@ -use super::*; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_epoch_processing::process_crosslinks; -use types::{BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct EpochProcessingCrosslinks { - pub description: String, - pub pre: BeaconState, - pub post: Option>, -} - -impl YamlDecode for EpochProcessingCrosslinks { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for EpochProcessingCrosslinks { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - let mut state = self.pre.clone(); - let mut expected = self.post.clone(); - - // Processing requires the epoch cache. - state.build_all_caches(&E::default_spec()).unwrap(); - - let mut result = process_crosslinks(&mut state, &E::default_spec()).map(|_| state); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/epoch_processing_final_updates.rs b/tests/ef_tests/src/cases/epoch_processing_final_updates.rs deleted file mode 100644 index 69e6b8bd3..000000000 --- a/tests/ef_tests/src/cases/epoch_processing_final_updates.rs +++ /dev/null @@ -1,41 +0,0 @@ -use super::*; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_epoch_processing::process_final_updates; -use types::{BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct EpochProcessingFinalUpdates { - pub description: String, - pub pre: BeaconState, - pub post: Option>, -} - -impl YamlDecode for EpochProcessingFinalUpdates { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for EpochProcessingFinalUpdates { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - let mut state = self.pre.clone(); - let mut expected = self.post.clone(); - - let spec = &E::default_spec(); - - let mut result = (|| { - // Processing requires the epoch cache. - state.build_all_caches(spec)?; - - process_final_updates(&mut state, spec).map(|_| state) - })(); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/epoch_processing_justification_and_finalization.rs b/tests/ef_tests/src/cases/epoch_processing_justification_and_finalization.rs deleted file mode 100644 index 788301086..000000000 --- a/tests/ef_tests/src/cases/epoch_processing_justification_and_finalization.rs +++ /dev/null @@ -1,46 +0,0 @@ -use super::*; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_epoch_processing::{ - process_justification_and_finalization, validator_statuses::ValidatorStatuses, -}; -use types::{BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct EpochProcessingJustificationAndFinalization { - pub description: String, - pub pre: BeaconState, - pub post: Option>, -} - -impl YamlDecode for EpochProcessingJustificationAndFinalization { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for EpochProcessingJustificationAndFinalization { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - let mut state = self.pre.clone(); - let mut expected = self.post.clone(); - - let spec = &E::default_spec(); - - // Processing requires the epoch cache. - state.build_all_caches(spec).unwrap(); - - let mut result = (|| { - let mut validator_statuses = ValidatorStatuses::new(&state, spec)?; - validator_statuses.process_attestations(&state, spec)?; - process_justification_and_finalization(&mut state, &validator_statuses.total_balances) - .map(|_| state) - })(); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/epoch_processing_registry_updates.rs b/tests/ef_tests/src/cases/epoch_processing_registry_updates.rs deleted file mode 100644 index a01f895fe..000000000 --- a/tests/ef_tests/src/cases/epoch_processing_registry_updates.rs +++ /dev/null @@ -1,38 +0,0 @@ -use super::*; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_epoch_processing::registry_updates::process_registry_updates; -use types::{BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct EpochProcessingRegistryUpdates { - pub description: String, - pub pre: BeaconState, - pub post: Option>, -} - -impl YamlDecode for EpochProcessingRegistryUpdates { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for EpochProcessingRegistryUpdates { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - let mut state = self.pre.clone(); - let mut expected = self.post.clone(); - let spec = &E::default_spec(); - - // Processing requires the epoch cache. - state.build_all_caches(spec).unwrap(); - - let mut result = process_registry_updates(&mut state, spec).map(|_| state); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/epoch_processing_slashings.rs b/tests/ef_tests/src/cases/epoch_processing_slashings.rs deleted file mode 100644 index d2a988d92..000000000 --- a/tests/ef_tests/src/cases/epoch_processing_slashings.rs +++ /dev/null @@ -1,50 +0,0 @@ -use super::*; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_epoch_processing::{ - process_slashings::process_slashings, validator_statuses::ValidatorStatuses, -}; -use types::{BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct EpochProcessingSlashings { - pub description: String, - pub pre: BeaconState, - pub post: Option>, -} - -impl YamlDecode for EpochProcessingSlashings { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for EpochProcessingSlashings { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - let mut state = self.pre.clone(); - let mut expected = self.post.clone(); - - let spec = &E::default_spec(); - - let mut result = (|| { - // Processing requires the epoch cache. - state.build_all_caches(spec)?; - - let mut validator_statuses = ValidatorStatuses::new(&state, spec)?; - validator_statuses.process_attestations(&state, spec)?; - process_slashings( - &mut state, - validator_statuses.total_balances.current_epoch, - spec, - ) - .map(|_| state) - })(); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/sanity_slots.rs b/tests/ef_tests/src/cases/sanity_slots.rs index 27c6c13c3..a66f1c2c4 100644 --- a/tests/ef_tests/src/cases/sanity_slots.rs +++ b/tests/ef_tests/src/cases/sanity_slots.rs @@ -27,7 +27,7 @@ impl LoadCase for SanitySlots { fn load_from_dir(path: &Path) -> Result { let metadata_path = path.join("meta.yaml"); let metadata: Metadata = if metadata_path.is_file() { - yaml_decode_file(&path.join("meta.yaml"))? + yaml_decode_file(&metadata_path)? } else { Metadata::default() }; diff --git a/tests/ef_tests/src/handler.rs b/tests/ef_tests/src/handler.rs index f66dc7b18..a614afe76 100644 --- a/tests/ef_tests/src/handler.rs +++ b/tests/ef_tests/src/handler.rs @@ -1,4 +1,4 @@ -use crate::cases::{self, Case, Cases, LoadCase}; +use crate::cases::{self, Case, Cases, EpochTransition, LoadCase}; use crate::type_name::TypeName; use crate::EfTest; use std::fs; @@ -184,3 +184,21 @@ impl Handler for SanitySlotsHandler { "slots" } } + +pub struct EpochProcessingHandler(PhantomData<(E, T)>); + +impl> Handler for EpochProcessingHandler { + type Case = cases::EpochProcessing; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "epoch_processing" + } + + fn handler_name() -> &'static str { + T::name() + } +} diff --git a/tests/ef_tests/src/lib.rs b/tests/ef_tests/src/lib.rs index 06e01fc03..54e674d85 100644 --- a/tests/ef_tests/src/lib.rs +++ b/tests/ef_tests/src/lib.rs @@ -2,6 +2,9 @@ use types::EthSpec; pub use case_result::CaseResult; pub use cases::Case; +pub use cases::{ + Crosslinks, FinalUpdates, JustificationAndFinalization, RegistryUpdates, Slashings, +}; pub use error::Error; pub use handler::*; pub use yaml_decode::YamlDecode; diff --git a/tests/ef_tests/src/type_name.rs b/tests/ef_tests/src/type_name.rs index fe55a7f5f..5af0c5256 100644 --- a/tests/ef_tests/src/type_name.rs +++ b/tests/ef_tests/src/type_name.rs @@ -5,57 +5,56 @@ pub trait TypeName { fn name() -> &'static str; } -impl TypeName for MinimalEthSpec { - fn name() -> &'static str { - "minimal" - } -} - -impl TypeName for MainnetEthSpec { - fn name() -> &'static str { - "mainnet" - } -} - -macro_rules! impl_name { +#[macro_export] +macro_rules! type_name { ($typ:ident) => { + type_name!($typ, stringify!($typ)); + }; + ($typ:ident, $name:expr) => { impl TypeName for $typ { fn name() -> &'static str { - stringify!($typ) + $name } } }; } -macro_rules! impl_name_generic { +#[macro_export] +macro_rules! type_name_generic { ($typ:ident) => { + type_name_generic!($typ, stringify!($typ)); + }; + ($typ:ident, $name:expr) => { impl TypeName for $typ { fn name() -> &'static str { - stringify!($typ) + $name } } }; } -impl_name_generic!(Attestation); -impl_name!(AttestationData); -impl_name!(AttestationDataAndCustodyBit); -impl_name_generic!(AttesterSlashing); -impl_name_generic!(BeaconBlock); -impl_name_generic!(BeaconBlockBody); -impl_name!(BeaconBlockHeader); -impl_name_generic!(BeaconState); -impl_name!(Checkpoint); -impl_name_generic!(CompactCommittee); -impl_name!(Crosslink); -impl_name!(Deposit); -impl_name!(DepositData); -impl_name!(Eth1Data); -impl_name!(Fork); -impl_name_generic!(HistoricalBatch); -impl_name_generic!(IndexedAttestation); -impl_name_generic!(PendingAttestation); -impl_name!(ProposerSlashing); -impl_name!(Transfer); -impl_name!(Validator); -impl_name!(VoluntaryExit); +type_name!(MinimalEthSpec, "minimal"); +type_name!(MainnetEthSpec, "mainnet"); + +type_name_generic!(Attestation); +type_name!(AttestationData); +type_name!(AttestationDataAndCustodyBit); +type_name_generic!(AttesterSlashing); +type_name_generic!(BeaconBlock); +type_name_generic!(BeaconBlockBody); +type_name!(BeaconBlockHeader); +type_name_generic!(BeaconState); +type_name!(Checkpoint); +type_name_generic!(CompactCommittee); +type_name!(Crosslink); +type_name!(Deposit); +type_name!(DepositData); +type_name!(Eth1Data); +type_name!(Fork); +type_name_generic!(HistoricalBatch); +type_name_generic!(IndexedAttestation); +type_name_generic!(PendingAttestation); +type_name!(ProposerSlashing); +type_name!(Transfer); +type_name!(Validator); +type_name!(VoluntaryExit); diff --git a/tests/ef_tests/tests/tests.rs b/tests/ef_tests/tests/tests.rs index 0fb751c9e..848ae05fa 100644 --- a/tests/ef_tests/tests/tests.rs +++ b/tests/ef_tests/tests/tests.rs @@ -262,52 +262,37 @@ ssz_static_test!(ssz_static_transfer, Transfer, SR); ssz_static_test!(ssz_static_validator, Validator); ssz_static_test!(ssz_static_voluntary_exit, VoluntaryExit, SR); -/* #[test] fn epoch_processing_justification_and_finalization() { - yaml_files_in_test_dir(&Path::new("epoch_processing").join("justification_and_finalization")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + EpochProcessingHandler::::run(); + EpochProcessingHandler::::run(); } #[test] fn epoch_processing_crosslinks() { - yaml_files_in_test_dir(&Path::new("epoch_processing").join("crosslinks")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + EpochProcessingHandler::::run(); + EpochProcessingHandler::::run(); } #[test] fn epoch_processing_registry_updates() { - yaml_files_in_test_dir(&Path::new("epoch_processing").join("registry_updates")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + EpochProcessingHandler::::run(); + EpochProcessingHandler::::run(); } #[test] fn epoch_processing_slashings() { - yaml_files_in_test_dir(&Path::new("epoch_processing").join("slashings")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + EpochProcessingHandler::::run(); + EpochProcessingHandler::::run(); } #[test] fn epoch_processing_final_updates() { - yaml_files_in_test_dir(&Path::new("epoch_processing").join("final_updates")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + EpochProcessingHandler::::run(); + EpochProcessingHandler::::run(); } +/* #[test] fn genesis_initialization() { yaml_files_in_test_dir(&Path::new("genesis").join("initialization")) From c5a22b57d29e59f578affa938f1be50da0b2a539 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 30 Aug 2019 14:10:28 +1000 Subject: [PATCH 07/12] Genesis tests --- .../src/cases/genesis_initialization.rs | 39 ++++++++++--- tests/ef_tests/src/cases/genesis_validity.rs | 24 +++++--- tests/ef_tests/src/handler.rs | 36 ++++++++++++ tests/ef_tests/tests/tests.rs | 57 ++----------------- 4 files changed, 85 insertions(+), 71 deletions(-) diff --git a/tests/ef_tests/src/cases/genesis_initialization.rs b/tests/ef_tests/src/cases/genesis_initialization.rs index 7ae8eef59..4f0fa4296 100644 --- a/tests/ef_tests/src/cases/genesis_initialization.rs +++ b/tests/ef_tests/src/cases/genesis_initialization.rs @@ -1,34 +1,55 @@ use super::*; -use crate::bls_setting::BlsSetting; use crate::case_result::compare_beacon_state_results_without_caches; +use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::initialize_beacon_state_from_eth1; +use std::path::PathBuf; use types::{BeaconState, Deposit, EthSpec, Hash256}; +#[derive(Debug, Clone, Deserialize)] +struct Metadata { + deposits_count: usize, +} + #[derive(Debug, Clone, Deserialize)] #[serde(bound = "E: EthSpec")] pub struct GenesisInitialization { - pub description: String, - pub bls_setting: Option, + pub path: PathBuf, pub eth1_block_hash: Hash256, pub eth1_timestamp: u64, pub deposits: Vec, pub state: Option>, } -impl YamlDecode for GenesisInitialization { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) +impl LoadCase for GenesisInitialization { + fn load_from_dir(path: &Path) -> Result { + let eth1_block_hash = ssz_decode_file(&path.join("eth1_block_hash.ssz"))?; + let eth1_timestamp = yaml_decode_file(&path.join("eth1_timestamp.yaml"))?; + let meta: Metadata = yaml_decode_file(&path.join("meta.yaml"))?; + let deposits: Vec = (0..meta.deposits_count) + .map(|i| { + let filename = format!("deposits_{}.ssz", i); + ssz_decode_file(&path.join(filename)) + }) + .collect::>()?; + let state = ssz_decode_file(&path.join("state.ssz"))?; + + Ok(Self { + path: path.into(), + eth1_block_hash, + eth1_timestamp, + deposits, + state: Some(state), + }) } } impl Case for GenesisInitialization { - fn description(&self) -> String { - self.description.clone() + fn path(&self) -> &Path { + &self.path } fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; let spec = &E::default_spec(); let mut result = initialize_beacon_state_from_eth1( diff --git a/tests/ef_tests/src/cases/genesis_validity.rs b/tests/ef_tests/src/cases/genesis_validity.rs index 7ddd3e8fd..efebe5e11 100644 --- a/tests/ef_tests/src/cases/genesis_validity.rs +++ b/tests/ef_tests/src/cases/genesis_validity.rs @@ -1,31 +1,37 @@ use super::*; -use crate::bls_setting::BlsSetting; +use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::is_valid_genesis_state; +use std::path::{Path, PathBuf}; use types::{BeaconState, EthSpec}; #[derive(Debug, Clone, Deserialize)] #[serde(bound = "E: EthSpec")] pub struct GenesisValidity { - pub description: String, - pub bls_setting: Option, + pub path: PathBuf, pub genesis: BeaconState, pub is_valid: bool, } -impl YamlDecode for GenesisValidity { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) +impl LoadCase for GenesisValidity { + fn load_from_dir(path: &Path) -> Result { + let genesis = ssz_decode_file(&path.join("genesis.ssz"))?; + let is_valid = yaml_decode_file(&path.join("is_valid.yaml"))?; + + Ok(Self { + path: path.into(), + genesis, + is_valid, + }) } } impl Case for GenesisValidity { - fn description(&self) -> String { - self.description.clone() + fn path(&self) -> &Path { + &self.path } fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; let spec = &E::default_spec(); let is_valid = is_valid_genesis_state(&self.genesis, spec); diff --git a/tests/ef_tests/src/handler.rs b/tests/ef_tests/src/handler.rs index a614afe76..02518a13d 100644 --- a/tests/ef_tests/src/handler.rs +++ b/tests/ef_tests/src/handler.rs @@ -202,3 +202,39 @@ impl> Handler for EpochProcessingHa T::name() } } + +pub struct GenesisValidityHandler(PhantomData); + +impl Handler for GenesisValidityHandler { + type Case = cases::GenesisValidity; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "genesis" + } + + fn handler_name() -> &'static str { + "validity" + } +} + +pub struct GenesisInitializationHandler(PhantomData); + +impl Handler for GenesisInitializationHandler { + type Case = cases::GenesisInitialization; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "genesis" + } + + fn handler_name() -> &'static str { + "initialization" + } +} diff --git a/tests/ef_tests/tests/tests.rs b/tests/ef_tests/tests/tests.rs index 848ae05fa..37740cec0 100644 --- a/tests/ef_tests/tests/tests.rs +++ b/tests/ef_tests/tests/tests.rs @@ -1,52 +1,11 @@ use ef_tests::*; use rayon::prelude::*; -use std::path::{Path, PathBuf}; use types::{ Attestation, AttestationData, AttestationDataAndCustodyBit, AttesterSlashing, BeaconBlock, BeaconBlockBody, BeaconBlockHeader, BeaconState, Checkpoint, CompactCommittee, Crosslink, Deposit, DepositData, Eth1Data, Fork, HistoricalBatch, IndexedAttestation, MainnetEthSpec, MinimalEthSpec, PendingAttestation, ProposerSlashing, Transfer, Validator, VoluntaryExit, }; -use walkdir::WalkDir; - -fn yaml_files_in_test_dir(dir: &Path) -> Vec { - let base_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("eth2.0-spec-tests") - .join("tests") - .join("general") - .join("phase0") - .join(dir); - - assert!( - base_path.exists(), - format!( - "Unable to locate {:?}. Did you init git submodules?", - base_path - ) - ); - - let mut paths: Vec = WalkDir::new(base_path) - .into_iter() - .filter_map(|e| e.ok()) - .filter_map(|entry| { - if entry.file_type().is_file() { - match entry.file_name().to_str() { - Some(f) if f.ends_with(".yaml") => Some(entry.path().to_path_buf()), - Some(f) if f.ends_with(".yml") => Some(entry.path().to_path_buf()), - _ => None, - } - } else { - None - } - }) - .collect(); - - // Reverse the file order. Assuming files come in lexicographical order, executing tests in - // reverse means we get the "minimal" tests before the "mainnet" tests. This makes life easier - // for debugging. - paths.reverse(); - paths -} /* #[test] @@ -292,22 +251,14 @@ fn epoch_processing_final_updates() { EpochProcessingHandler::::run(); } -/* #[test] fn genesis_initialization() { - yaml_files_in_test_dir(&Path::new("genesis").join("initialization")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + GenesisInitializationHandler::::run(); } #[test] fn genesis_validity() { - yaml_files_in_test_dir(&Path::new("genesis").join("validity")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + GenesisValidityHandler::::run(); + // TODO: mainnet tests don't exist yet + // GenesisValidityHandler::::run(); } -*/ From fcf16faad38573b6c4b655d2d589f469b6d8b051 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 30 Aug 2019 16:16:38 +1000 Subject: [PATCH 08/12] Operations tests --- tests/ef_tests/src/cases.rs | 16 +- tests/ef_tests/src/cases/operations.rs | 193 ++++++++++++++++++ .../src/cases/operations_attestation.rs | 47 ----- .../src/cases/operations_attester_slashing.rs | 48 ----- .../src/cases/operations_block_header.rs | 44 ---- .../ef_tests/src/cases/operations_deposit.rs | 42 ---- tests/ef_tests/src/cases/operations_exit.rs | 45 ---- .../src/cases/operations_proposer_slashing.rs | 46 ----- .../ef_tests/src/cases/operations_transfer.rs | 47 ----- tests/ef_tests/src/handler.rs | 58 ++++-- tests/ef_tests/tests/tests.rs | 67 ++---- 11 files changed, 248 insertions(+), 405 deletions(-) create mode 100644 tests/ef_tests/src/cases/operations.rs delete mode 100644 tests/ef_tests/src/cases/operations_attestation.rs delete mode 100644 tests/ef_tests/src/cases/operations_attester_slashing.rs delete mode 100644 tests/ef_tests/src/cases/operations_block_header.rs delete mode 100644 tests/ef_tests/src/cases/operations_deposit.rs delete mode 100644 tests/ef_tests/src/cases/operations_exit.rs delete mode 100644 tests/ef_tests/src/cases/operations_proposer_slashing.rs delete mode 100644 tests/ef_tests/src/cases/operations_transfer.rs diff --git a/tests/ef_tests/src/cases.rs b/tests/ef_tests/src/cases.rs index e3ec54a7f..1192eb0a0 100644 --- a/tests/ef_tests/src/cases.rs +++ b/tests/ef_tests/src/cases.rs @@ -11,13 +11,7 @@ mod bls_sign_msg; mod epoch_processing; mod genesis_initialization; mod genesis_validity; -mod operations_attestation; -mod operations_attester_slashing; -mod operations_block_header; -mod operations_deposit; -mod operations_exit; -mod operations_proposer_slashing; -mod operations_transfer; +mod operations; mod sanity_blocks; mod sanity_slots; mod shuffling; @@ -33,13 +27,7 @@ pub use bls_sign_msg::*; pub use epoch_processing::*; pub use genesis_initialization::*; pub use genesis_validity::*; -pub use operations_attestation::*; -pub use operations_attester_slashing::*; -pub use operations_block_header::*; -pub use operations_deposit::*; -pub use operations_exit::*; -pub use operations_proposer_slashing::*; -pub use operations_transfer::*; +pub use operations::*; pub use sanity_blocks::*; pub use sanity_slots::*; pub use shuffling::*; diff --git a/tests/ef_tests/src/cases/operations.rs b/tests/ef_tests/src/cases/operations.rs new file mode 100644 index 000000000..bcc104bad --- /dev/null +++ b/tests/ef_tests/src/cases/operations.rs @@ -0,0 +1,193 @@ +use super::*; +use crate::bls_setting::BlsSetting; +use crate::case_result::compare_beacon_state_results_without_caches; +use crate::type_name::TypeName; +use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; +use serde_derive::Deserialize; +use ssz::Decode; +use state_processing::per_block_processing::{ + errors::BlockProcessingError, process_attestations, process_attester_slashings, + process_block_header, process_deposits, process_exits, process_proposer_slashings, + process_transfers, +}; +use std::path::{Path, PathBuf}; +use types::{ + Attestation, AttesterSlashing, BeaconBlock, BeaconState, ChainSpec, Deposit, EthSpec, + ProposerSlashing, Transfer, VoluntaryExit, +}; + +#[derive(Debug, Clone, Default, Deserialize)] +struct Metadata { + description: Option, + bls_setting: Option, +} + +#[derive(Debug, Clone)] +pub struct Operations> { + pub path: PathBuf, + metadata: Metadata, + pub pre: BeaconState, + pub operation: O, + pub post: Option>, +} + +pub trait Operation: Decode + TypeName + std::fmt::Debug { + fn handler_name() -> String { + Self::name().to_lowercase() + } + + fn filename() -> String { + format!("{}.ssz", Self::handler_name()) + } + + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError>; +} + +impl Operation for Attestation { + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + process_attestations(state, &[self.clone()], spec) + } +} + +impl Operation for AttesterSlashing { + fn handler_name() -> String { + "attester_slashing".into() + } + + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + process_attester_slashings(state, &[self.clone()], spec) + } +} + +impl Operation for Deposit { + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + process_deposits(state, &[self.clone()], spec) + } +} + +impl Operation for ProposerSlashing { + fn handler_name() -> String { + "proposer_slashing".into() + } + + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + process_proposer_slashings(state, &[self.clone()], spec) + } +} + +impl Operation for Transfer { + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + process_transfers(state, &[self.clone()], spec) + } +} + +impl Operation for VoluntaryExit { + fn handler_name() -> String { + "voluntary_exit".into() + } + + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + process_exits(state, &[self.clone()], spec) + } +} + +impl Operation for BeaconBlock { + fn handler_name() -> String { + "block_header".into() + } + + fn filename() -> String { + "block.ssz".into() + } + + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + process_block_header(state, self, spec, true) + } +} + +impl> LoadCase for Operations { + fn load_from_dir(path: &Path) -> Result { + let metadata_path = path.join("meta.yaml"); + let metadata: Metadata = if metadata_path.is_file() { + yaml_decode_file(&metadata_path)? + } else { + Metadata::default() + }; + let pre = ssz_decode_file(&path.join("pre.ssz"))?; + let operation = ssz_decode_file(&path.join(O::filename()))?; + let post_filename = path.join("post.ssz"); + let post = if post_filename.is_file() { + Some(ssz_decode_file(&post_filename)?) + } else { + None + }; + + Ok(Self { + path: path.into(), + metadata, + pre, + operation, + post, + }) + } +} + +impl> Case for Operations { + fn description(&self) -> String { + self.metadata + .description + .clone() + .unwrap_or_else(String::new) + } + + fn path(&self) -> &Path { + &self.path + } + + fn result(&self, _case_index: usize) -> Result<(), Error> { + self.metadata.bls_setting.unwrap_or_default().check()?; + + let spec = &E::default_spec(); + let mut state = self.pre.clone(); + let mut expected = self.post.clone(); + + // Processing requires the epoch cache. + state.build_all_caches(spec).unwrap(); + + let mut result = self.operation.apply_to(&mut state, spec).map(|()| state); + + compare_beacon_state_results_without_caches(&mut result, &mut expected) + } +} diff --git a/tests/ef_tests/src/cases/operations_attestation.rs b/tests/ef_tests/src/cases/operations_attestation.rs deleted file mode 100644 index 76cbe3f18..000000000 --- a/tests/ef_tests/src/cases/operations_attestation.rs +++ /dev/null @@ -1,47 +0,0 @@ -use super::*; -use crate::bls_setting::BlsSetting; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_block_processing::process_attestations; -use types::{Attestation, BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct OperationsAttestation { - pub description: String, - pub bls_setting: Option, - pub pre: BeaconState, - pub attestation: Attestation, - pub post: Option>, -} - -impl YamlDecode for OperationsAttestation { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(&yaml).unwrap()) - } -} - -impl Case for OperationsAttestation { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - let spec = &E::default_spec(); - - self.bls_setting.unwrap_or_default().check()?; - - let mut state = self.pre.clone(); - let attestation = self.attestation.clone(); - let mut expected = self.post.clone(); - - // Processing requires the epoch cache. - state.build_all_caches(spec).unwrap(); - - let result = process_attestations(&mut state, &[attestation], spec); - - let mut result = result.and_then(|_| Ok(state)); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/operations_attester_slashing.rs b/tests/ef_tests/src/cases/operations_attester_slashing.rs deleted file mode 100644 index c658b1af4..000000000 --- a/tests/ef_tests/src/cases/operations_attester_slashing.rs +++ /dev/null @@ -1,48 +0,0 @@ -use super::*; -use crate::bls_setting::BlsSetting; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_block_processing::process_attester_slashings; -use types::{AttesterSlashing, BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -pub struct OperationsAttesterSlashing { - pub description: String, - pub bls_setting: Option, - #[serde(bound = "E: EthSpec")] - pub pre: BeaconState, - #[serde(bound = "E: EthSpec")] - pub attester_slashing: AttesterSlashing, - #[serde(bound = "E: EthSpec")] - pub post: Option>, -} - -impl YamlDecode for OperationsAttesterSlashing { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for OperationsAttesterSlashing { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; - - let mut state = self.pre.clone(); - let attester_slashing = self.attester_slashing.clone(); - let mut expected = self.post.clone(); - - // Processing requires the epoch cache. - state.build_all_caches(&E::default_spec()).unwrap(); - - let result = - process_attester_slashings(&mut state, &[attester_slashing], &E::default_spec()); - - let mut result = result.and_then(|_| Ok(state)); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/operations_block_header.rs b/tests/ef_tests/src/cases/operations_block_header.rs deleted file mode 100644 index 8261b16d9..000000000 --- a/tests/ef_tests/src/cases/operations_block_header.rs +++ /dev/null @@ -1,44 +0,0 @@ -use super::*; -use crate::bls_setting::BlsSetting; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_block_processing::process_block_header; -use types::{BeaconBlock, BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct OperationsBlockHeader { - pub description: String, - pub bls_setting: Option, - pub pre: BeaconState, - pub block: BeaconBlock, - pub post: Option>, -} - -impl YamlDecode for OperationsBlockHeader { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for OperationsBlockHeader { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - let spec = &E::default_spec(); - - self.bls_setting.unwrap_or_default().check()?; - - let mut state = self.pre.clone(); - let mut expected = self.post.clone(); - - // Processing requires the epoch cache. - state.build_all_caches(spec).unwrap(); - - let mut result = process_block_header(&mut state, &self.block, spec, true).map(|_| state); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/operations_deposit.rs b/tests/ef_tests/src/cases/operations_deposit.rs deleted file mode 100644 index 801c02029..000000000 --- a/tests/ef_tests/src/cases/operations_deposit.rs +++ /dev/null @@ -1,42 +0,0 @@ -use super::*; -use crate::bls_setting::BlsSetting; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_block_processing::process_deposits; -use types::{BeaconState, Deposit, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct OperationsDeposit { - pub description: String, - pub bls_setting: Option, - pub pre: BeaconState, - pub deposit: Deposit, - pub post: Option>, -} - -impl YamlDecode for OperationsDeposit { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for OperationsDeposit { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; - - let mut state = self.pre.clone(); - let deposit = self.deposit.clone(); - let mut expected = self.post.clone(); - - let result = process_deposits(&mut state, &[deposit], &E::default_spec()); - - let mut result = result.and_then(|_| Ok(state)); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/operations_exit.rs b/tests/ef_tests/src/cases/operations_exit.rs deleted file mode 100644 index d7e53bcb5..000000000 --- a/tests/ef_tests/src/cases/operations_exit.rs +++ /dev/null @@ -1,45 +0,0 @@ -use super::*; -use crate::bls_setting::BlsSetting; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_block_processing::process_exits; -use types::{BeaconState, EthSpec, VoluntaryExit}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct OperationsExit { - pub description: String, - pub bls_setting: Option, - pub pre: BeaconState, - pub voluntary_exit: VoluntaryExit, - pub post: Option>, -} - -impl YamlDecode for OperationsExit { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for OperationsExit { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; - - let mut state = self.pre.clone(); - let exit = self.voluntary_exit.clone(); - let mut expected = self.post.clone(); - - // Exit processing requires the epoch cache. - state.build_all_caches(&E::default_spec()).unwrap(); - - let result = process_exits(&mut state, &[exit], &E::default_spec()); - - let mut result = result.and_then(|_| Ok(state)); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/operations_proposer_slashing.rs b/tests/ef_tests/src/cases/operations_proposer_slashing.rs deleted file mode 100644 index e52e84f39..000000000 --- a/tests/ef_tests/src/cases/operations_proposer_slashing.rs +++ /dev/null @@ -1,46 +0,0 @@ -use super::*; -use crate::bls_setting::BlsSetting; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_block_processing::process_proposer_slashings; -use types::{BeaconState, EthSpec, ProposerSlashing}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct OperationsProposerSlashing { - pub description: String, - pub bls_setting: Option, - pub pre: BeaconState, - pub proposer_slashing: ProposerSlashing, - pub post: Option>, -} - -impl YamlDecode for OperationsProposerSlashing { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for OperationsProposerSlashing { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; - - let mut state = self.pre.clone(); - let proposer_slashing = self.proposer_slashing.clone(); - let mut expected = self.post.clone(); - - // Processing requires the epoch cache. - state.build_all_caches(&E::default_spec()).unwrap(); - - let result = - process_proposer_slashings(&mut state, &[proposer_slashing], &E::default_spec()); - - let mut result = result.and_then(|_| Ok(state)); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/operations_transfer.rs b/tests/ef_tests/src/cases/operations_transfer.rs deleted file mode 100644 index 250f58769..000000000 --- a/tests/ef_tests/src/cases/operations_transfer.rs +++ /dev/null @@ -1,47 +0,0 @@ -use super::*; -use crate::bls_setting::BlsSetting; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_block_processing::process_transfers; -use types::{BeaconState, EthSpec, Transfer}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct OperationsTransfer { - pub description: String, - pub bls_setting: Option, - pub pre: BeaconState, - pub transfer: Transfer, - pub post: Option>, -} - -impl YamlDecode for OperationsTransfer { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for OperationsTransfer { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; - - let mut state = self.pre.clone(); - let transfer = self.transfer.clone(); - let mut expected = self.post.clone(); - - // Transfer processing requires the epoch cache. - state.build_all_caches(&E::default_spec()).unwrap(); - - let spec = E::default_spec(); - - let result = process_transfers(&mut state, &[transfer], &spec); - - let mut result = result.and_then(|_| Ok(state)); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/handler.rs b/tests/ef_tests/src/handler.rs index 02518a13d..ca1304136 100644 --- a/tests/ef_tests/src/handler.rs +++ b/tests/ef_tests/src/handler.rs @@ -1,4 +1,4 @@ -use crate::cases::{self, Case, Cases, EpochTransition, LoadCase}; +use crate::cases::{self, Case, Cases, EpochTransition, LoadCase, Operation}; use crate::type_name::TypeName; use crate::EfTest; use std::fs; @@ -20,7 +20,7 @@ pub trait Handler { fn runner_name() -> &'static str; - fn handler_name() -> &'static str; + fn handler_name() -> String; fn run() { let handler_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -64,8 +64,8 @@ macro_rules! bls_handler { "bls" } - fn handler_name() -> &'static str { - $handler_name + fn handler_name() -> String { + $handler_name.into() } } }; @@ -106,8 +106,8 @@ where "ssz_static" } - fn handler_name() -> &'static str { - T::name() + fn handler_name() -> String { + T::name().into() } } @@ -126,8 +126,8 @@ where "ssz_static" } - fn handler_name() -> &'static str { - T::name() + fn handler_name() -> String { + T::name().into() } } @@ -144,8 +144,8 @@ impl Handler for ShufflingHandler { "shuffling" } - fn handler_name() -> &'static str { - "core" + fn handler_name() -> String { + "core".into() } } @@ -162,8 +162,8 @@ impl Handler for SanityBlocksHandler { "sanity" } - fn handler_name() -> &'static str { - "blocks" + fn handler_name() -> String { + "blocks".into() } } @@ -180,8 +180,8 @@ impl Handler for SanitySlotsHandler { "sanity" } - fn handler_name() -> &'static str { - "slots" + fn handler_name() -> String { + "slots".into() } } @@ -198,8 +198,8 @@ impl> Handler for EpochProcessingHa "epoch_processing" } - fn handler_name() -> &'static str { - T::name() + fn handler_name() -> String { + T::name().into() } } @@ -216,8 +216,8 @@ impl Handler for GenesisValidityHandler { "genesis" } - fn handler_name() -> &'static str { - "validity" + fn handler_name() -> String { + "validity".into() } } @@ -234,7 +234,25 @@ impl Handler for GenesisInitializationHandler { "genesis" } - fn handler_name() -> &'static str { - "initialization" + fn handler_name() -> String { + "initialization".into() + } +} + +pub struct OperationsHandler(PhantomData<(E, O)>); + +impl> Handler for OperationsHandler { + type Case = cases::Operations; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "operations" + } + + fn handler_name() -> String { + O::handler_name() } } diff --git a/tests/ef_tests/tests/tests.rs b/tests/ef_tests/tests/tests.rs index 37740cec0..d663eb454 100644 --- a/tests/ef_tests/tests/tests.rs +++ b/tests/ef_tests/tests/tests.rs @@ -36,71 +36,47 @@ fn shuffling() { ShufflingHandler::::run(); } -/* #[test] fn operations_deposit() { - yaml_files_in_test_dir(&Path::new("operations").join("deposit")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + OperationsHandler::::run(); + OperationsHandler::::run(); } #[test] fn operations_transfer() { - yaml_files_in_test_dir(&Path::new("operations").join("transfer")) - .into_par_iter() - .rev() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + OperationsHandler::::run(); + // Note: there are no transfer tests for mainnet } #[test] fn operations_exit() { - yaml_files_in_test_dir(&Path::new("operations").join("voluntary_exit")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + OperationsHandler::::run(); + OperationsHandler::::run(); } #[test] fn operations_proposer_slashing() { - yaml_files_in_test_dir(&Path::new("operations").join("proposer_slashing")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + OperationsHandler::::run(); + OperationsHandler::::run(); } #[test] fn operations_attester_slashing() { - yaml_files_in_test_dir(&Path::new("operations").join("attester_slashing")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + OperationsHandler::>::run(); + OperationsHandler::>::run(); } #[test] fn operations_attestation() { - yaml_files_in_test_dir(&Path::new("operations").join("attestation")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + OperationsHandler::>::run(); + OperationsHandler::>::run(); } #[test] fn operations_block_header() { - yaml_files_in_test_dir(&Path::new("operations").join("block_header")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + OperationsHandler::>::run(); + OperationsHandler::>::run(); } -*/ #[test] fn sanity_blocks() { @@ -114,18 +90,6 @@ fn sanity_slots() { SanitySlotsHandler::::run(); } -/* -#[test] -#[cfg(not(feature = "fake_crypto"))] -fn bls() { - yaml_files_in_test_dir(&Path::new("bls")) - .into_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); -} -*/ - #[test] #[cfg(not(feature = "fake_crypto"))] fn bls_aggregate_pubkeys() { @@ -259,6 +223,5 @@ fn genesis_initialization() { #[test] fn genesis_validity() { GenesisValidityHandler::::run(); - // TODO: mainnet tests don't exist yet - // GenesisValidityHandler::::run(); + // Note: there are no genesis validity tests for mainnet } From 8d5a579aa64a4ba53c010f12913c9f45adedd56c Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 23 Aug 2019 18:33:27 +1000 Subject: [PATCH 09/12] Fix BeaconChain tests --- eth2/types/src/beacon_state/tests.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs index 67adccdda..0363e5848 100644 --- a/eth2/types/src/beacon_state/tests.rs +++ b/eth2/types/src/beacon_state/tests.rs @@ -90,11 +90,11 @@ fn test_active_index(state_slot: Slot) { // Test the start and end of the range. assert_eq!( - state.get_active_index_root_index(*range.start(), &spec), + state.get_active_index_root_index(*range.start(), &spec, AllowNextEpoch::False), Ok(modulo(*range.start())) ); assert_eq!( - state.get_active_index_root_index(*range.end(), &spec), + state.get_active_index_root_index(*range.end(), &spec, AllowNextEpoch::False), Ok(modulo(*range.end())) ); @@ -102,12 +102,12 @@ fn test_active_index(state_slot: Slot) { if state.current_epoch() > 0 { // Test is invalid on epoch zero, cannot subtract from zero. assert_eq!( - state.get_active_index_root_index(*range.start() - 1, &spec), + state.get_active_index_root_index(*range.start() - 1, &spec, AllowNextEpoch::False), Err(Error::EpochOutOfBounds) ); } assert_eq!( - state.get_active_index_root_index(*range.end() + 1, &spec), + state.get_active_index_root_index(*range.end() + 1, &spec, AllowNextEpoch::False), Err(Error::EpochOutOfBounds) ); } From f47eaf57304d627e552408aae8003197f0318f5d Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 3 Sep 2019 16:46:10 +1000 Subject: [PATCH 10/12] Parallel tests and SSZ generic --- eth2/utils/ssz_types/src/fixed_vector.rs | 17 +- tests/ef_tests/src/cases.rs | 12 +- tests/ef_tests/src/cases/epoch_processing.rs | 2 +- tests/ef_tests/src/cases/operations.rs | 3 +- tests/ef_tests/src/cases/ssz_generic.rs | 229 +++++++++++++++---- tests/ef_tests/src/cases/ssz_static.rs | 8 +- tests/ef_tests/src/handler.rs | 32 ++- tests/ef_tests/src/lib.rs | 7 - tests/ef_tests/tests/tests.rs | 25 +- 9 files changed, 246 insertions(+), 89 deletions(-) diff --git a/eth2/utils/ssz_types/src/fixed_vector.rs b/eth2/utils/ssz_types/src/fixed_vector.rs index edac77f0d..edf499adf 100644 --- a/eth2/utils/ssz_types/src/fixed_vector.rs +++ b/eth2/utils/ssz_types/src/fixed_vector.rs @@ -220,13 +220,26 @@ where fn from_ssz_bytes(bytes: &[u8]) -> Result { if bytes.is_empty() { - Ok(FixedVector::from(vec![])) + Err(ssz::DecodeError::InvalidByteLength { + len: 0, + expected: 1, + }) } else if T::is_ssz_fixed_len() { bytes .chunks(T::ssz_fixed_len()) .map(|chunk| T::from_ssz_bytes(chunk)) .collect::, _>>() - .and_then(|vec| Ok(vec.into())) + .and_then(|vec| { + if vec.len() == N::to_usize() { + Ok(vec.into()) + } else { + Err(ssz::DecodeError::BytesInvalid(format!( + "wrong number of vec elements, got: {}, expected: {}", + vec.len(), + N::to_usize() + ))) + } + }) } else { ssz::decode_list_of_variable_length_items(bytes).and_then(|vec| Ok(vec.into())) } diff --git a/tests/ef_tests/src/cases.rs b/tests/ef_tests/src/cases.rs index 1192eb0a0..ed00f0ffe 100644 --- a/tests/ef_tests/src/cases.rs +++ b/tests/ef_tests/src/cases.rs @@ -1,4 +1,5 @@ use super::*; +use rayon::prelude::*; use std::fmt::Debug; use std::path::Path; @@ -39,7 +40,7 @@ pub trait LoadCase: Sized { fn load_from_dir(_path: &Path) -> Result; } -pub trait Case: Debug { +pub trait Case: Debug + Sync { /// An optional field for implementing a custom description. /// /// Defaults to "no description". @@ -79,13 +80,10 @@ pub struct Cases { pub test_cases: Vec, } -impl EfTest for Cases -where - T: Case + Debug, -{ - fn test_results(&self) -> Vec { +impl Cases { + pub fn test_results(&self) -> Vec { self.test_cases - .iter() + .into_par_iter() .enumerate() .map(|(i, tc)| CaseResult::new(i, tc, tc.result(i))) .collect() diff --git a/tests/ef_tests/src/cases/epoch_processing.rs b/tests/ef_tests/src/cases/epoch_processing.rs index ac47ab236..d79b5fc48 100644 --- a/tests/ef_tests/src/cases/epoch_processing.rs +++ b/tests/ef_tests/src/cases/epoch_processing.rs @@ -31,7 +31,7 @@ pub struct EpochProcessing> { _phantom: PhantomData, } -pub trait EpochTransition: TypeName + Debug { +pub trait EpochTransition: TypeName + Debug + Sync { fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError>; } diff --git a/tests/ef_tests/src/cases/operations.rs b/tests/ef_tests/src/cases/operations.rs index bcc104bad..e86e6f598 100644 --- a/tests/ef_tests/src/cases/operations.rs +++ b/tests/ef_tests/src/cases/operations.rs @@ -10,6 +10,7 @@ use state_processing::per_block_processing::{ process_block_header, process_deposits, process_exits, process_proposer_slashings, process_transfers, }; +use std::fmt::Debug; use std::path::{Path, PathBuf}; use types::{ Attestation, AttesterSlashing, BeaconBlock, BeaconState, ChainSpec, Deposit, EthSpec, @@ -31,7 +32,7 @@ pub struct Operations> { pub post: Option>, } -pub trait Operation: Decode + TypeName + std::fmt::Debug { +pub trait Operation: Decode + TypeName + Debug + Sync { fn handler_name() -> String { Self::name().to_lowercase() } diff --git a/tests/ef_tests/src/cases/ssz_generic.rs b/tests/ef_tests/src/cases/ssz_generic.rs index ca49d2106..05b96ad7d 100644 --- a/tests/ef_tests/src/cases/ssz_generic.rs +++ b/tests/ef_tests/src/cases/ssz_generic.rs @@ -1,68 +1,205 @@ use super::*; -use crate::case_result::compare_result; -use ethereum_types::{U128, U256}; +use crate::cases::ssz_static::{check_serialization, check_tree_hash, SszStaticType}; +use crate::yaml_decode::yaml_decode_file; use serde_derive::Deserialize; -use ssz::Decode; -use std::fmt::Debug; +use std::fs; +use std::path::{Path, PathBuf}; +use types::typenum::*; +use types::{BitList, BitVector, FixedVector, VariableList}; #[derive(Debug, Clone, Deserialize)] -pub struct SszGeneric { - #[serde(alias = "type")] - pub type_name: String, - pub valid: bool, - pub value: Option, - pub ssz: Option, +struct Metadata { + root: String, + signing_root: Option, } -impl YamlDecode for SszGeneric { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) +#[derive(Debug, Clone)] +pub struct SszGeneric { + path: PathBuf, + handler_name: String, + case_name: String, +} + +impl LoadCase for SszGeneric { + fn load_from_dir(path: &Path) -> Result { + let components = path + .components() + .map(|c| c.as_os_str().to_string_lossy().into_owned()) + .rev() + .collect::>(); + // Test case name is last + let case_name = components[0].clone(); + // Handler name is third last, before suite name and case name + let handler_name = components[2].clone(); + Ok(Self { + path: path.into(), + handler_name, + case_name, + }) + } +} + +macro_rules! type_dispatch { + ($function:ident, + ($($arg:expr),*), + $base_ty:tt, + <$($param_ty:ty),*>, + [ $value:expr => primitive_type ] $($rest:tt)*) => { + match $value { + "bool" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* bool>, $($rest)*), + "uint8" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u8>, $($rest)*), + "uint16" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u16>, $($rest)*), + "uint32" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u32>, $($rest)*), + "uint64" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u64>, $($rest)*), + // FIXME(michael): implement tree hash for big ints + // "uint128" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* etherum_types::U128>, $($rest)*), + // "uint256" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ethereum_types::U256>, $($rest)*), + _ => { println!("unsupported: {}", $value); Ok(()) }, + } + }; + ($function:ident, + ($($arg:expr),*), + $base_ty:tt, + <$($param_ty:ty),*>, + [ $value:expr => typenum ] $($rest:tt)*) => { + match $value { + // DO YOU LIKE NUMBERS? + "0" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U0>, $($rest)*), + "1" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U1>, $($rest)*), + "2" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U2>, $($rest)*), + "3" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U3>, $($rest)*), + "4" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U4>, $($rest)*), + "5" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U5>, $($rest)*), + "6" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U6>, $($rest)*), + "7" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U7>, $($rest)*), + "8" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U8>, $($rest)*), + "9" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U9>, $($rest)*), + "16" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U16>, $($rest)*), + "31" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U31>, $($rest)*), + "32" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U32>, $($rest)*), + "64" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U64>, $($rest)*), + "128" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U128>, $($rest)*), + "256" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U256>, $($rest)*), + "512" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U512>, $($rest)*), + "513" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U513>, $($rest)*), + "1024" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U1024>, $($rest)*), + "2048" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U2048>, $($rest)*), + "4096" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U4096>, $($rest)*), + "8192" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U8192>, $($rest)*), + _ => { println!("unsupported: {}", $value); Ok(()) }, + } + }; + // No base type: apply type params to function + ($function:ident, ($($arg:expr),*), _, <$($param_ty:ty),*>,) => { + $function::<$($param_ty),*>($($arg),*) + }; + ($function:ident, ($($arg:expr),*), $base_type_name:ident, <$($param_ty:ty),*>,) => { + $function::<$base_type_name<$($param_ty),*>>($($arg),*) } } impl Case for SszGeneric { + fn path(&self) -> &Path { + &self.path + } + fn result(&self, _case_index: usize) -> Result<(), Error> { - if let Some(ssz) = &self.ssz { - match self.type_name.as_ref() { - "uint8" => ssz_generic_test::(self.valid, ssz, &self.value), - "uint16" => ssz_generic_test::(self.valid, ssz, &self.value), - "uint32" => ssz_generic_test::(self.valid, ssz, &self.value), - "uint64" => ssz_generic_test::(self.valid, ssz, &self.value), - "uint128" => ssz_generic_test::(self.valid, ssz, &self.value), - "uint256" => ssz_generic_test::(self.valid, ssz, &self.value), - _ => Err(Error::FailedToParseTest(format!( - "Unknown type: {}", - self.type_name - ))), + let parts = self.case_name.split('_').collect::>(); + + match self.handler_name.as_str() { + "basic_vector" => { + let elem_ty = parts[1]; + let length = parts[2]; + + type_dispatch!( + ssz_generic_test, + (&self.path), + FixedVector, + <>, + [elem_ty => primitive_type] + [length => typenum] + )?; } - } else { - // Skip tests that do not have an ssz field. - // - // See: https://github.com/ethereum/eth2.0-specs/issues/1079 - Ok(()) + "bitlist" => { + let limit = parts[1]; + + // FIXME(michael): mark length "no" cases as known failures + + type_dispatch!( + ssz_generic_test, + (&self.path), + BitList, + <>, + [limit => typenum] + )?; + } + "bitvector" => { + let length = parts[1]; + + type_dispatch!( + ssz_generic_test, + (&self.path), + BitVector, + <>, + [length => typenum] + )?; + } + "boolean" => { + ssz_generic_test::(&self.path)?; + } + "uints" => { + let type_name = "uint".to_owned() + parts[1]; + + type_dispatch!( + ssz_generic_test, + (&self.path), + _, + <>, + [type_name.as_str() => primitive_type] + )?; + } + // FIXME(michael): support for the containers tests + _ => panic!("unsupported handler: {}", self.handler_name), } + Ok(()) } } -/// Execute a `ssz_generic` test case. -fn ssz_generic_test(should_be_ok: bool, ssz: &str, value: &Option) -> Result<(), Error> -where - T: Decode + YamlDecode + Debug + PartialEq, -{ - let ssz = hex::decode(&ssz[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?; - - // We do not cater for the scenario where the test is valid but we are not passed any SSZ. - if should_be_ok && value.is_none() { - panic!("Unexpected test input. Cannot pass without value.") - } - - let expected = if let Some(string) = value { - Some(T::yaml_decode(string)?) +fn ssz_generic_test(path: &Path) -> Result<(), Error> { + let meta_path = path.join("meta.yaml"); + let meta: Option = if meta_path.is_file() { + Some(yaml_decode_file(&meta_path)?) } else { None }; - let decoded = T::from_ssz_bytes(&ssz); + let serialized = fs::read(&path.join("serialized.ssz")).expect("serialized.ssz exists"); - compare_result(&decoded, &expected) + let value_path = path.join("value.yaml"); + let value: Option = if value_path.is_file() { + Some(yaml_decode_file(&value_path)?) + } else { + None + }; + + // Valid + // TODO: signing root + if let Some(value) = value { + check_serialization(&value, &serialized)?; + + if let Some(ref meta) = meta { + check_tree_hash(&meta.root, value.tree_hash_root())?; + } + } + // Invalid + else { + if let Ok(decoded) = T::from_ssz_bytes(&serialized) { + return Err(Error::DidntFail(format!( + "Decoded invalid bytes into: {:?}", + decoded + ))); + } + } + + Ok(()) } diff --git a/tests/ef_tests/src/cases/ssz_static.rs b/tests/ef_tests/src/cases/ssz_static.rs index 6a949073d..f9f59cc4b 100644 --- a/tests/ef_tests/src/cases/ssz_static.rs +++ b/tests/ef_tests/src/cases/ssz_static.rs @@ -35,12 +35,12 @@ pub struct SszStaticSR { // Trait alias for all deez bounds pub trait SszStaticType: - serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + Sync { } impl SszStaticType for T where - T: serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + T: serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + Sync { } @@ -77,7 +77,7 @@ impl LoadCase for SszStaticSR { } } -fn check_serialization(value: &T, serialized: &[u8]) -> Result<(), Error> { +pub fn check_serialization(value: &T, serialized: &[u8]) -> Result<(), Error> { // Check serialization let serialized_result = value.as_ssz_bytes(); compare_result::, Error>(&Ok(serialized_result), &Some(serialized.to_vec()))?; @@ -89,7 +89,7 @@ fn check_serialization(value: &T, serialized: &[u8]) -> Result Ok(()) } -fn check_tree_hash(expected_str: &str, actual_root: Vec) -> Result<(), Error> { +pub fn check_tree_hash(expected_str: &str, actual_root: Vec) -> Result<(), Error> { let expected_root = hex::decode(&expected_str[2..]) .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?; let expected_root = Hash256::from_slice(&expected_root); diff --git a/tests/ef_tests/src/handler.rs b/tests/ef_tests/src/handler.rs index ca1304136..b6334c383 100644 --- a/tests/ef_tests/src/handler.rs +++ b/tests/ef_tests/src/handler.rs @@ -1,6 +1,6 @@ use crate::cases::{self, Case, Cases, EpochTransition, LoadCase, Operation}; +use crate::type_name; use crate::type_name::TypeName; -use crate::EfTest; use std::fs; use std::marker::PhantomData; use std::path::PathBuf; @@ -256,3 +256,33 @@ impl> Handler for OperationsHandler O::handler_name() } } + +pub struct SszGenericHandler(PhantomData); + +impl Handler for SszGenericHandler { + type Case = cases::SszGeneric; + + fn config_name() -> &'static str { + "general" + } + + fn runner_name() -> &'static str { + "ssz_generic" + } + + fn handler_name() -> String { + H::name().into() + } +} + +// Supported SSZ generic handlers +pub struct BasicVector; +type_name!(BasicVector, "basic_vector"); +pub struct Bitlist; +type_name!(Bitlist, "bitlist"); +pub struct Bitvector; +type_name!(Bitvector, "bitvector"); +pub struct Boolean; +type_name!(Boolean, "boolean"); +pub struct Uints; +type_name!(Uints, "uints"); diff --git a/tests/ef_tests/src/lib.rs b/tests/ef_tests/src/lib.rs index 54e674d85..bcf7c77a0 100644 --- a/tests/ef_tests/src/lib.rs +++ b/tests/ef_tests/src/lib.rs @@ -17,10 +17,3 @@ mod handler; mod results; mod type_name; mod yaml_decode; - -/// Defined where an object can return the results of some test(s) adhering to the Ethereum -/// Foundation testing format. -pub trait EfTest { - /// Returns the results of executing one or more tests. - fn test_results(&self) -> Vec; -} diff --git a/tests/ef_tests/tests/tests.rs b/tests/ef_tests/tests/tests.rs index d663eb454..71fa53c66 100644 --- a/tests/ef_tests/tests/tests.rs +++ b/tests/ef_tests/tests/tests.rs @@ -1,5 +1,4 @@ use ef_tests::*; -use rayon::prelude::*; use types::{ Attestation, AttestationData, AttestationDataAndCustodyBit, AttesterSlashing, BeaconBlock, BeaconBlockBody, BeaconBlockHeader, BeaconState, Checkpoint, CompactCommittee, Crosslink, @@ -7,29 +6,15 @@ use types::{ MinimalEthSpec, PendingAttestation, ProposerSlashing, Transfer, Validator, VoluntaryExit, }; -/* #[test] -#[cfg(feature = "fake_crypto")] fn ssz_generic() { - yaml_files_in_test_dir(&Path::new("ssz_generic")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + SszGenericHandler::::run(); + SszGenericHandler::::run(); + SszGenericHandler::::run(); + SszGenericHandler::::run(); + SszGenericHandler::::run(); } - -#[test] -#[cfg(feature = "fake_crypto")] -fn ssz_static() { - yaml_files_in_test_dir(&Path::new("ssz_static")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); -} -*/ - #[test] fn shuffling() { ShufflingHandler::::run(); From d511c939eb4015796573dc3466b40d5a4811735a Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 4 Sep 2019 13:03:44 +1000 Subject: [PATCH 11/12] SSZ generic tests for big uints --- eth2/utils/tree_hash/src/impls.rs | 42 ++++++++- tests/ef_tests/Cargo.toml | 2 + tests/ef_tests/src/cases.rs | 16 +--- .../src/cases/bls_aggregate_pubkeys.rs | 1 + .../ef_tests/src/cases/bls_aggregate_sigs.rs | 1 + tests/ef_tests/src/cases/bls_g2_compressed.rs | 1 + .../ef_tests/src/cases/bls_g2_uncompressed.rs | 9 +- tests/ef_tests/src/cases/bls_priv_to_pub.rs | 1 + tests/ef_tests/src/cases/bls_sign_msg.rs | 1 + tests/ef_tests/src/cases/common.rs | 72 ++++++++++++++ tests/ef_tests/src/cases/epoch_processing.rs | 2 +- .../src/cases/genesis_initialization.rs | 2 +- tests/ef_tests/src/cases/genesis_validity.rs | 2 +- tests/ef_tests/src/cases/operations.rs | 2 +- tests/ef_tests/src/cases/sanity_blocks.rs | 2 +- tests/ef_tests/src/cases/sanity_slots.rs | 2 +- tests/ef_tests/src/cases/shuffling.rs | 9 +- tests/ef_tests/src/cases/ssz_generic.rs | 20 ++-- tests/ef_tests/src/cases/ssz_static.rs | 33 ++----- tests/ef_tests/src/decode.rs | 31 +++++++ tests/ef_tests/src/lib.rs | 3 +- tests/ef_tests/src/results.rs | 3 +- tests/ef_tests/src/yaml_decode.rs | 93 ------------------- 23 files changed, 185 insertions(+), 165 deletions(-) create mode 100644 tests/ef_tests/src/cases/common.rs create mode 100644 tests/ef_tests/src/decode.rs delete mode 100644 tests/ef_tests/src/yaml_decode.rs diff --git a/eth2/utils/tree_hash/src/impls.rs b/eth2/utils/tree_hash/src/impls.rs index 88293196e..9f09f50ce 100644 --- a/eth2/utils/tree_hash/src/impls.rs +++ b/eth2/utils/tree_hash/src/impls.rs @@ -1,5 +1,5 @@ use super::*; -use ethereum_types::H256; +use ethereum_types::{H256, U128, U256}; macro_rules! impl_for_bitsize { ($type: ident, $bit_size: expr) => { @@ -73,6 +73,46 @@ macro_rules! impl_for_u8_array { impl_for_u8_array!(4); impl_for_u8_array!(32); +impl TreeHash for U128 { + fn tree_hash_type() -> TreeHashType { + TreeHashType::Basic + } + + fn tree_hash_packed_encoding(&self) -> Vec { + let mut result = vec![0; 16]; + self.to_little_endian(&mut result); + result + } + + fn tree_hash_packing_factor() -> usize { + 2 + } + + fn tree_hash_root(&self) -> Vec { + merkle_root(&self.tree_hash_packed_encoding(), 0) + } +} + +impl TreeHash for U256 { + fn tree_hash_type() -> TreeHashType { + TreeHashType::Basic + } + + fn tree_hash_packed_encoding(&self) -> Vec { + let mut result = vec![0; 32]; + self.to_little_endian(&mut result); + result + } + + fn tree_hash_packing_factor() -> usize { + 1 + } + + fn tree_hash_root(&self) -> Vec { + merkle_root(&self.tree_hash_packed_encoding(), 0) + } +} + impl TreeHash for H256 { fn tree_hash_type() -> TreeHashType { TreeHashType::Vector diff --git a/tests/ef_tests/Cargo.toml b/tests/ef_tests/Cargo.toml index ba6aca259..2f1dea11d 100644 --- a/tests/ef_tests/Cargo.toml +++ b/tests/ef_tests/Cargo.toml @@ -18,7 +18,9 @@ serde_derive = "1.0" serde_repr = "0.1" serde_yaml = "0.8" eth2_ssz = "0.1" +eth2_ssz_derive = "0.1" tree_hash = "0.1" +tree_hash_derive = "0.2" state_processing = { path = "../../eth2/state_processing" } swap_or_not_shuffle = { path = "../../eth2/utils/swap_or_not_shuffle" } types = { path = "../../eth2/types" } diff --git a/tests/ef_tests/src/cases.rs b/tests/ef_tests/src/cases.rs index ed00f0ffe..279086b68 100644 --- a/tests/ef_tests/src/cases.rs +++ b/tests/ef_tests/src/cases.rs @@ -9,6 +9,7 @@ mod bls_g2_compressed; mod bls_g2_uncompressed; mod bls_priv_to_pub; mod bls_sign_msg; +mod common; mod epoch_processing; mod genesis_initialization; mod genesis_validity; @@ -25,6 +26,7 @@ pub use bls_g2_compressed::*; pub use bls_g2_uncompressed::*; pub use bls_priv_to_pub::*; pub use bls_sign_msg::*; +pub use common::SszStaticType; pub use epoch_processing::*; pub use genesis_initialization::*; pub use genesis_validity::*; @@ -61,20 +63,6 @@ pub trait Case: Debug + Sync { fn result(&self, case_index: usize) -> Result<(), Error>; } -pub trait BlsCase: serde::de::DeserializeOwned {} - -impl YamlDecode for T { - fn yaml_decode(string: &str) -> Result { - serde_yaml::from_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) - } -} - -impl LoadCase for T { - fn load_from_dir(path: &Path) -> Result { - Self::yaml_decode_file(&path.join("data.yaml")) - } -} - #[derive(Debug)] pub struct Cases { pub test_cases: Vec, diff --git a/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs b/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs index c94e14495..13c2fea17 100644 --- a/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs +++ b/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs @@ -1,5 +1,6 @@ use super::*; use crate::case_result::compare_result; +use crate::cases::common::BlsCase; use bls::{AggregatePublicKey, PublicKey}; use serde_derive::Deserialize; diff --git a/tests/ef_tests/src/cases/bls_aggregate_sigs.rs b/tests/ef_tests/src/cases/bls_aggregate_sigs.rs index 882ad7220..22fa197df 100644 --- a/tests/ef_tests/src/cases/bls_aggregate_sigs.rs +++ b/tests/ef_tests/src/cases/bls_aggregate_sigs.rs @@ -1,5 +1,6 @@ use super::*; use crate::case_result::compare_result; +use crate::cases::common::BlsCase; use bls::{AggregateSignature, Signature}; use serde_derive::Deserialize; diff --git a/tests/ef_tests/src/cases/bls_g2_compressed.rs b/tests/ef_tests/src/cases/bls_g2_compressed.rs index f8381f5a7..1a9f1d561 100644 --- a/tests/ef_tests/src/cases/bls_g2_compressed.rs +++ b/tests/ef_tests/src/cases/bls_g2_compressed.rs @@ -1,5 +1,6 @@ use super::*; use crate::case_result::compare_result; +use crate::cases::common::BlsCase; use bls::{compress_g2, hash_on_g2}; use serde_derive::Deserialize; diff --git a/tests/ef_tests/src/cases/bls_g2_uncompressed.rs b/tests/ef_tests/src/cases/bls_g2_uncompressed.rs index 962b6aac3..3eae29967 100644 --- a/tests/ef_tests/src/cases/bls_g2_uncompressed.rs +++ b/tests/ef_tests/src/cases/bls_g2_uncompressed.rs @@ -1,5 +1,6 @@ use super::*; use crate::case_result::compare_result; +use crate::cases::common::BlsCase; use bls::hash_on_g2; use serde_derive::Deserialize; @@ -9,18 +10,14 @@ pub struct BlsG2UncompressedInput { pub domain: String, } +impl BlsCase for BlsG2UncompressedInput {} + #[derive(Debug, Clone, Deserialize)] pub struct BlsG2Uncompressed { pub input: BlsG2UncompressedInput, pub output: Vec>, } -impl YamlDecode for BlsG2Uncompressed { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - impl Case for BlsG2Uncompressed { fn result(&self, _case_index: usize) -> Result<(), Error> { // Convert message and domain to required types diff --git a/tests/ef_tests/src/cases/bls_priv_to_pub.rs b/tests/ef_tests/src/cases/bls_priv_to_pub.rs index 869a0891c..016e04dd1 100644 --- a/tests/ef_tests/src/cases/bls_priv_to_pub.rs +++ b/tests/ef_tests/src/cases/bls_priv_to_pub.rs @@ -1,5 +1,6 @@ use super::*; use crate::case_result::compare_result; +use crate::cases::common::BlsCase; use bls::{PublicKey, SecretKey}; use serde_derive::Deserialize; diff --git a/tests/ef_tests/src/cases/bls_sign_msg.rs b/tests/ef_tests/src/cases/bls_sign_msg.rs index 18e90896b..7ee109f81 100644 --- a/tests/ef_tests/src/cases/bls_sign_msg.rs +++ b/tests/ef_tests/src/cases/bls_sign_msg.rs @@ -1,5 +1,6 @@ use super::*; use crate::case_result::compare_result; +use crate::cases::common::BlsCase; use bls::{SecretKey, Signature}; use serde_derive::Deserialize; diff --git a/tests/ef_tests/src/cases/common.rs b/tests/ef_tests/src/cases/common.rs new file mode 100644 index 000000000..8e787f157 --- /dev/null +++ b/tests/ef_tests/src/cases/common.rs @@ -0,0 +1,72 @@ +use crate::cases::LoadCase; +use crate::decode::yaml_decode_file; +use crate::error::Error; +use serde_derive::Deserialize; +use ssz::{Decode, Encode}; +use ssz_derive::{Decode, Encode}; +use std::convert::TryFrom; +use std::fmt::Debug; +use std::path::Path; +use tree_hash::TreeHash; + +/// Trait for all BLS cases to eliminate some boilerplate. +pub trait BlsCase: serde::de::DeserializeOwned {} + +impl LoadCase for T { + fn load_from_dir(path: &Path) -> Result { + yaml_decode_file(&path.join("data.yaml")) + } +} + +/// Macro to wrap U128 and U256 so they deserialize correctly. +macro_rules! uint_wrapper { + ($wrapper_name:ident, $wrapped_type:ty) => { + #[derive(Debug, Clone, Copy, Default, PartialEq, Decode, Encode, Deserialize)] + #[serde(try_from = "String")] + pub struct $wrapper_name { + pub x: $wrapped_type, + } + + impl TryFrom for $wrapper_name { + type Error = String; + + fn try_from(s: String) -> Result { + <$wrapped_type>::from_dec_str(&s) + .map(|x| Self { x }) + .map_err(|e| format!("{:?}", e)) + } + } + + impl tree_hash::TreeHash for $wrapper_name { + fn tree_hash_type() -> tree_hash::TreeHashType { + <$wrapped_type>::tree_hash_type() + } + + fn tree_hash_packed_encoding(&self) -> Vec { + self.x.tree_hash_packed_encoding() + } + + fn tree_hash_packing_factor() -> usize { + <$wrapped_type>::tree_hash_packing_factor() + } + + fn tree_hash_root(&self) -> Vec { + self.x.tree_hash_root() + } + } + }; +} + +uint_wrapper!(TestU128, ethereum_types::U128); +uint_wrapper!(TestU256, ethereum_types::U256); + +/// Trait alias for all deez bounds +pub trait SszStaticType: + serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + Sync +{ +} + +impl SszStaticType for T where + T: serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + Sync +{ +} diff --git a/tests/ef_tests/src/cases/epoch_processing.rs b/tests/ef_tests/src/cases/epoch_processing.rs index d79b5fc48..2a2dde629 100644 --- a/tests/ef_tests/src/cases/epoch_processing.rs +++ b/tests/ef_tests/src/cases/epoch_processing.rs @@ -1,9 +1,9 @@ use super::*; use crate::bls_setting::BlsSetting; use crate::case_result::compare_beacon_state_results_without_caches; +use crate::decode::{ssz_decode_file, yaml_decode_file}; use crate::type_name; use crate::type_name::TypeName; -use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::per_epoch_processing::{ errors::EpochProcessingError, process_crosslinks, process_final_updates, diff --git a/tests/ef_tests/src/cases/genesis_initialization.rs b/tests/ef_tests/src/cases/genesis_initialization.rs index 4f0fa4296..bd0507b9d 100644 --- a/tests/ef_tests/src/cases/genesis_initialization.rs +++ b/tests/ef_tests/src/cases/genesis_initialization.rs @@ -1,6 +1,6 @@ use super::*; use crate::case_result::compare_beacon_state_results_without_caches; -use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; +use crate::decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::initialize_beacon_state_from_eth1; use std::path::PathBuf; diff --git a/tests/ef_tests/src/cases/genesis_validity.rs b/tests/ef_tests/src/cases/genesis_validity.rs index efebe5e11..3a1b9e267 100644 --- a/tests/ef_tests/src/cases/genesis_validity.rs +++ b/tests/ef_tests/src/cases/genesis_validity.rs @@ -1,5 +1,5 @@ use super::*; -use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; +use crate::decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::is_valid_genesis_state; use std::path::{Path, PathBuf}; diff --git a/tests/ef_tests/src/cases/operations.rs b/tests/ef_tests/src/cases/operations.rs index e86e6f598..7b4ffff98 100644 --- a/tests/ef_tests/src/cases/operations.rs +++ b/tests/ef_tests/src/cases/operations.rs @@ -1,8 +1,8 @@ use super::*; use crate::bls_setting::BlsSetting; use crate::case_result::compare_beacon_state_results_without_caches; +use crate::decode::{ssz_decode_file, yaml_decode_file}; use crate::type_name::TypeName; -use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use ssz::Decode; use state_processing::per_block_processing::{ diff --git a/tests/ef_tests/src/cases/sanity_blocks.rs b/tests/ef_tests/src/cases/sanity_blocks.rs index d88d8f295..9fadea42e 100644 --- a/tests/ef_tests/src/cases/sanity_blocks.rs +++ b/tests/ef_tests/src/cases/sanity_blocks.rs @@ -1,7 +1,7 @@ use super::*; use crate::bls_setting::BlsSetting; use crate::case_result::compare_beacon_state_results_without_caches; -use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; +use crate::decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::{ per_block_processing, per_slot_processing, BlockInvalid, BlockProcessingError, diff --git a/tests/ef_tests/src/cases/sanity_slots.rs b/tests/ef_tests/src/cases/sanity_slots.rs index a66f1c2c4..34acb1105 100644 --- a/tests/ef_tests/src/cases/sanity_slots.rs +++ b/tests/ef_tests/src/cases/sanity_slots.rs @@ -1,7 +1,7 @@ use super::*; use crate::bls_setting::BlsSetting; use crate::case_result::compare_beacon_state_results_without_caches; -use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; +use crate::decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::per_slot_processing; use std::path::PathBuf; diff --git a/tests/ef_tests/src/cases/shuffling.rs b/tests/ef_tests/src/cases/shuffling.rs index c0595e584..2fe632e84 100644 --- a/tests/ef_tests/src/cases/shuffling.rs +++ b/tests/ef_tests/src/cases/shuffling.rs @@ -1,5 +1,6 @@ use super::*; use crate::case_result::compare_result; +use crate::decode::yaml_decode_file; use serde_derive::Deserialize; use std::marker::PhantomData; use swap_or_not_shuffle::{get_permutated_index, shuffle_list}; @@ -13,15 +14,9 @@ pub struct Shuffling { _phantom: PhantomData, } -impl YamlDecode for Shuffling { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - impl LoadCase for Shuffling { fn load_from_dir(path: &Path) -> Result { - Self::yaml_decode_file(&path.join("mapping.yaml")) + yaml_decode_file(&path.join("mapping.yaml")) } } diff --git a/tests/ef_tests/src/cases/ssz_generic.rs b/tests/ef_tests/src/cases/ssz_generic.rs index 05b96ad7d..5f9cd3faf 100644 --- a/tests/ef_tests/src/cases/ssz_generic.rs +++ b/tests/ef_tests/src/cases/ssz_generic.rs @@ -1,11 +1,12 @@ use super::*; -use crate::cases::ssz_static::{check_serialization, check_tree_hash, SszStaticType}; -use crate::yaml_decode::yaml_decode_file; +use crate::cases::common::{SszStaticType, TestU128, TestU256}; +use crate::cases::ssz_static::{check_serialization, check_tree_hash}; +use crate::decode::yaml_decode_file; use serde_derive::Deserialize; use std::fs; use std::path::{Path, PathBuf}; use types::typenum::*; -use types::{BitList, BitVector, FixedVector, VariableList}; +use types::{BitList, BitVector, FixedVector}; #[derive(Debug, Clone, Deserialize)] struct Metadata { @@ -51,9 +52,8 @@ macro_rules! type_dispatch { "uint16" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u16>, $($rest)*), "uint32" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u32>, $($rest)*), "uint64" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u64>, $($rest)*), - // FIXME(michael): implement tree hash for big ints - // "uint128" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* etherum_types::U128>, $($rest)*), - // "uint256" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ethereum_types::U256>, $($rest)*), + "uint128" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* TestU128>, $($rest)*), + "uint256" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* TestU256>, $($rest)*), _ => { println!("unsupported: {}", $value); Ok(()) }, } }; @@ -121,9 +121,13 @@ impl Case for SszGeneric { )?; } "bitlist" => { - let limit = parts[1]; + let mut limit = parts[1]; - // FIXME(michael): mark length "no" cases as known failures + // Test format is inconsistent, pretend the limit is 32 (arbitrary) + // https://github.com/ethereum/eth2.0-spec-tests + if limit == "no" { + limit = "32"; + } type_dispatch!( ssz_generic_test, diff --git a/tests/ef_tests/src/cases/ssz_static.rs b/tests/ef_tests/src/cases/ssz_static.rs index f9f59cc4b..d1c9b1048 100644 --- a/tests/ef_tests/src/cases/ssz_static.rs +++ b/tests/ef_tests/src/cases/ssz_static.rs @@ -1,10 +1,10 @@ use super::*; use crate::case_result::compare_result; +use crate::cases::common::SszStaticType; +use crate::decode::yaml_decode_file; use serde_derive::Deserialize; -use ssz::{Decode, Encode}; -use std::fmt::Debug; use std::fs; -use tree_hash::{SignedRoot, TreeHash}; +use tree_hash::SignedRoot; use types::Hash256; #[derive(Debug, Clone, Deserialize)] @@ -13,12 +13,6 @@ struct SszStaticRoots { signing_root: Option, } -impl YamlDecode for SszStaticRoots { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - #[derive(Debug, Clone)] pub struct SszStatic { roots: SszStaticRoots, @@ -33,26 +27,11 @@ pub struct SszStaticSR { value: T, } -// Trait alias for all deez bounds -pub trait SszStaticType: - serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + Sync -{ -} - -impl SszStaticType for T where - T: serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + Sync -{ -} - fn load_from_dir(path: &Path) -> Result<(SszStaticRoots, Vec, T), Error> { - // FIXME: set description/name - let roots = SszStaticRoots::yaml_decode_file(&path.join("roots.yaml"))?; - + // FIXME(michael): set description/name + let roots = yaml_decode_file(&path.join("roots.yaml"))?; let serialized = fs::read(&path.join("serialized.ssz")).expect("serialized.ssz exists"); - - let yaml = fs::read_to_string(&path.join("value.yaml")).expect("value.yaml exists"); - let value = - serde_yaml::from_str(&yaml).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?; + let value = yaml_decode_file(&path.join("value.yaml"))?; Ok((roots, serialized, value)) } diff --git a/tests/ef_tests/src/decode.rs b/tests/ef_tests/src/decode.rs new file mode 100644 index 000000000..c1ea6fb3b --- /dev/null +++ b/tests/ef_tests/src/decode.rs @@ -0,0 +1,31 @@ +use super::*; +use std::fs; +use std::path::Path; + +pub fn yaml_decode(string: &str) -> Result { + serde_yaml::from_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) +} + +pub fn yaml_decode_file(path: &Path) -> Result { + fs::read_to_string(path) + .map_err(|e| { + Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e)) + }) + .and_then(|s| yaml_decode(&s)) +} + +pub fn ssz_decode_file(path: &Path) -> Result { + fs::read(path) + .map_err(|e| { + Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e)) + }) + .and_then(|s| { + T::from_ssz_bytes(&s).map_err(|e| { + Error::FailedToParseTest(format!( + "Unable to parse SSZ at {}: {:?}", + path.display(), + e + )) + }) + }) +} diff --git a/tests/ef_tests/src/lib.rs b/tests/ef_tests/src/lib.rs index bcf7c77a0..719bfc1aa 100644 --- a/tests/ef_tests/src/lib.rs +++ b/tests/ef_tests/src/lib.rs @@ -7,13 +7,12 @@ pub use cases::{ }; pub use error::Error; pub use handler::*; -pub use yaml_decode::YamlDecode; mod bls_setting; mod case_result; mod cases; +mod decode; mod error; mod handler; mod results; mod type_name; -mod yaml_decode; diff --git a/tests/ef_tests/src/results.rs b/tests/ef_tests/src/results.rs index 20e59f7b3..4f5513a9a 100644 --- a/tests/ef_tests/src/results.rs +++ b/tests/ef_tests/src/results.rs @@ -80,7 +80,8 @@ pub fn print_results( println!("-------"); println!( - "case ({}) from {} failed with {}:", + "case {} ({}) from {} failed with {}:", + failure.case_index, failure.desc, failure.path.display(), error.name() diff --git a/tests/ef_tests/src/yaml_decode.rs b/tests/ef_tests/src/yaml_decode.rs deleted file mode 100644 index 83a162930..000000000 --- a/tests/ef_tests/src/yaml_decode.rs +++ /dev/null @@ -1,93 +0,0 @@ -use super::*; -use ethereum_types::{U128, U256}; -use std::fs; -use std::path::Path; -use types::Fork; - -pub fn yaml_decode(string: &str) -> Result { - serde_yaml::from_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) -} - -pub fn yaml_decode_file(path: &Path) -> Result { - fs::read_to_string(path) - .map_err(|e| { - Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e)) - }) - .and_then(|s| yaml_decode(&s)) -} - -pub fn ssz_decode_file(path: &Path) -> Result { - fs::read(path) - .map_err(|e| { - Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e)) - }) - .and_then(|s| { - T::from_ssz_bytes(&s).map_err(|e| { - Error::FailedToParseTest(format!( - "Unable to parse SSZ at {}: {:?}", - path.display(), - e - )) - }) - }) -} - -pub trait YamlDecode: Sized { - /// Decode an object from the test specification YAML. - fn yaml_decode(string: &str) -> Result; - - fn yaml_decode_file(path: &Path) -> Result { - fs::read_to_string(path) - .map_err(|e| { - Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e)) - }) - .and_then(|s| Self::yaml_decode(&s)) - } -} - -/// Basic types can general be decoded with the `parse` fn if they implement `str::FromStr`. -macro_rules! impl_via_parse { - ($ty: ty) => { - impl YamlDecode for $ty { - fn yaml_decode(string: &str) -> Result { - string - .parse::() - .map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) - } - } - }; -} - -impl_via_parse!(u8); -impl_via_parse!(u16); -impl_via_parse!(u32); -impl_via_parse!(u64); - -/// Some `ethereum-types` methods have a `str::FromStr` implementation that expects `0x`-prefixed: -/// hex, so we use `from_dec_str` instead. -macro_rules! impl_via_from_dec_str { - ($ty: ty) => { - impl YamlDecode for $ty { - fn yaml_decode(string: &str) -> Result { - Self::from_dec_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) - } - } - }; -} - -impl_via_from_dec_str!(U128); -impl_via_from_dec_str!(U256); - -/// Types that already implement `serde::Deserialize` can be decoded using `serde_yaml`. -macro_rules! impl_via_serde_yaml { - ($ty: ty) => { - impl YamlDecode for $ty { - fn yaml_decode(string: &str) -> Result { - serde_yaml::from_str(string) - .map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) - } - } - }; -} - -impl_via_serde_yaml!(Fork); From 289f8d13b00c4984ab61324f989eff028e5cc207 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 5 Sep 2019 10:19:52 +1000 Subject: [PATCH 12/12] Cleanups and SSZ generic container tests --- tests/ef_tests/src/bls_setting.rs | 1 - tests/ef_tests/src/case_result.rs | 11 ++- tests/ef_tests/src/cases.rs | 12 +-- tests/ef_tests/src/cases/epoch_processing.rs | 4 - .../src/cases/genesis_initialization.rs | 4 - tests/ef_tests/src/cases/genesis_validity.rs | 13 +-- tests/ef_tests/src/cases/operations.rs | 8 +- tests/ef_tests/src/cases/sanity_blocks.rs | 7 -- tests/ef_tests/src/cases/sanity_slots.rs | 7 -- tests/ef_tests/src/cases/ssz_generic.rs | 79 +++++++++++++++-- tests/ef_tests/src/cases/ssz_static.rs | 1 - tests/ef_tests/src/handler.rs | 16 ++-- tests/ef_tests/tests/tests.rs | 84 ++++++++++--------- 13 files changed, 137 insertions(+), 110 deletions(-) diff --git a/tests/ef_tests/src/bls_setting.rs b/tests/ef_tests/src/bls_setting.rs index 79990c8ee..add7d8b7b 100644 --- a/tests/ef_tests/src/bls_setting.rs +++ b/tests/ef_tests/src/bls_setting.rs @@ -2,7 +2,6 @@ use self::BlsSetting::*; use crate::error::Error; use serde_repr::Deserialize_repr; -// TODO: use this in every test case #[derive(Deserialize_repr, Debug, Clone, Copy)] #[repr(u8)] pub enum BlsSetting { diff --git a/tests/ef_tests/src/case_result.rs b/tests/ef_tests/src/case_result.rs index add428ec5..9df60f402 100644 --- a/tests/ef_tests/src/case_result.rs +++ b/tests/ef_tests/src/case_result.rs @@ -1,7 +1,7 @@ use super::*; use compare_fields::{CompareFields, Comparison, FieldComparison}; use std::fmt::Debug; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use types::BeaconState; pub const MAX_VALUE_STRING_LEN: usize = 500; @@ -15,11 +15,16 @@ pub struct CaseResult { } impl CaseResult { - pub fn new(case_index: usize, case: &impl Case, result: Result<(), Error>) -> Self { + pub fn new( + case_index: usize, + path: &Path, + case: &impl Case, + result: Result<(), Error>, + ) -> Self { CaseResult { case_index, desc: case.description(), - path: case.path().into(), + path: path.into(), result, } } diff --git a/tests/ef_tests/src/cases.rs b/tests/ef_tests/src/cases.rs index 279086b68..c5b0d8c4f 100644 --- a/tests/ef_tests/src/cases.rs +++ b/tests/ef_tests/src/cases.rs @@ -1,7 +1,7 @@ use super::*; use rayon::prelude::*; use std::fmt::Debug; -use std::path::Path; +use std::path::{Path, PathBuf}; mod bls_aggregate_pubkeys; mod bls_aggregate_sigs; @@ -50,12 +50,6 @@ pub trait Case: Debug + Sync { "no description".to_string() } - /// Path to the directory for this test case. - fn path(&self) -> &Path { - // FIXME(michael): remove default impl - Path::new("") - } - /// Execute a test and return the result. /// /// `case_index` reports the index of the case in the set of test cases. It is not strictly @@ -65,7 +59,7 @@ pub trait Case: Debug + Sync { #[derive(Debug)] pub struct Cases { - pub test_cases: Vec, + pub test_cases: Vec<(PathBuf, T)>, } impl Cases { @@ -73,7 +67,7 @@ impl Cases { self.test_cases .into_par_iter() .enumerate() - .map(|(i, tc)| CaseResult::new(i, tc, tc.result(i))) + .map(|(i, (ref path, ref tc))| CaseResult::new(i, path, tc, tc.result(i))) .collect() } } diff --git a/tests/ef_tests/src/cases/epoch_processing.rs b/tests/ef_tests/src/cases/epoch_processing.rs index 2a2dde629..ece69b3fe 100644 --- a/tests/ef_tests/src/cases/epoch_processing.rs +++ b/tests/ef_tests/src/cases/epoch_processing.rs @@ -125,10 +125,6 @@ impl> Case for EpochProcessing { .unwrap_or_else(String::new) } - fn path(&self) -> &Path { - &self.path - } - fn result(&self, _case_index: usize) -> Result<(), Error> { let mut state = self.pre.clone(); let mut expected = self.post.clone(); diff --git a/tests/ef_tests/src/cases/genesis_initialization.rs b/tests/ef_tests/src/cases/genesis_initialization.rs index bd0507b9d..0fb64ccb3 100644 --- a/tests/ef_tests/src/cases/genesis_initialization.rs +++ b/tests/ef_tests/src/cases/genesis_initialization.rs @@ -45,10 +45,6 @@ impl LoadCase for GenesisInitialization { } impl Case for GenesisInitialization { - fn path(&self) -> &Path { - &self.path - } - fn result(&self, _case_index: usize) -> Result<(), Error> { let spec = &E::default_spec(); diff --git a/tests/ef_tests/src/cases/genesis_validity.rs b/tests/ef_tests/src/cases/genesis_validity.rs index 3a1b9e267..f72ac4c3e 100644 --- a/tests/ef_tests/src/cases/genesis_validity.rs +++ b/tests/ef_tests/src/cases/genesis_validity.rs @@ -2,13 +2,12 @@ use super::*; use crate::decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::is_valid_genesis_state; -use std::path::{Path, PathBuf}; +use std::path::Path; use types::{BeaconState, EthSpec}; #[derive(Debug, Clone, Deserialize)] #[serde(bound = "E: EthSpec")] pub struct GenesisValidity { - pub path: PathBuf, pub genesis: BeaconState, pub is_valid: bool, } @@ -18,19 +17,11 @@ impl LoadCase for GenesisValidity { let genesis = ssz_decode_file(&path.join("genesis.ssz"))?; let is_valid = yaml_decode_file(&path.join("is_valid.yaml"))?; - Ok(Self { - path: path.into(), - genesis, - is_valid, - }) + Ok(Self { genesis, is_valid }) } } impl Case for GenesisValidity { - fn path(&self) -> &Path { - &self.path - } - fn result(&self, _case_index: usize) -> Result<(), Error> { let spec = &E::default_spec(); diff --git a/tests/ef_tests/src/cases/operations.rs b/tests/ef_tests/src/cases/operations.rs index 7b4ffff98..89fa3ccca 100644 --- a/tests/ef_tests/src/cases/operations.rs +++ b/tests/ef_tests/src/cases/operations.rs @@ -11,7 +11,7 @@ use state_processing::per_block_processing::{ process_transfers, }; use std::fmt::Debug; -use std::path::{Path, PathBuf}; +use std::path::Path; use types::{ Attestation, AttesterSlashing, BeaconBlock, BeaconState, ChainSpec, Deposit, EthSpec, ProposerSlashing, Transfer, VoluntaryExit, @@ -25,7 +25,6 @@ struct Metadata { #[derive(Debug, Clone)] pub struct Operations> { - pub path: PathBuf, metadata: Metadata, pub pre: BeaconState, pub operation: O, @@ -156,7 +155,6 @@ impl> LoadCase for Operations { }; Ok(Self { - path: path.into(), metadata, pre, operation, @@ -173,10 +171,6 @@ impl> Case for Operations { .unwrap_or_else(String::new) } - fn path(&self) -> &Path { - &self.path - } - fn result(&self, _case_index: usize) -> Result<(), Error> { self.metadata.bls_setting.unwrap_or_default().check()?; diff --git a/tests/ef_tests/src/cases/sanity_blocks.rs b/tests/ef_tests/src/cases/sanity_blocks.rs index 9fadea42e..292f47415 100644 --- a/tests/ef_tests/src/cases/sanity_blocks.rs +++ b/tests/ef_tests/src/cases/sanity_blocks.rs @@ -6,7 +6,6 @@ use serde_derive::Deserialize; use state_processing::{ per_block_processing, per_slot_processing, BlockInvalid, BlockProcessingError, }; -use std::path::PathBuf; use types::{BeaconBlock, BeaconState, EthSpec, RelativeEpoch}; #[derive(Debug, Clone, Deserialize)] @@ -19,7 +18,6 @@ pub struct Metadata { #[derive(Debug, Clone, Deserialize)] #[serde(bound = "E: EthSpec")] pub struct SanityBlocks { - pub path: PathBuf, pub metadata: Metadata, pub pre: BeaconState, pub blocks: Vec>, @@ -44,7 +42,6 @@ impl LoadCase for SanityBlocks { }; Ok(Self { - path: path.into(), metadata, pre, blocks, @@ -61,10 +58,6 @@ impl Case for SanityBlocks { .unwrap_or_else(String::new) } - fn path(&self) -> &Path { - &self.path - } - fn result(&self, _case_index: usize) -> Result<(), Error> { self.metadata.bls_setting.unwrap_or_default().check()?; diff --git a/tests/ef_tests/src/cases/sanity_slots.rs b/tests/ef_tests/src/cases/sanity_slots.rs index 34acb1105..e9b80a252 100644 --- a/tests/ef_tests/src/cases/sanity_slots.rs +++ b/tests/ef_tests/src/cases/sanity_slots.rs @@ -4,7 +4,6 @@ use crate::case_result::compare_beacon_state_results_without_caches; use crate::decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::per_slot_processing; -use std::path::PathBuf; use types::{BeaconState, EthSpec}; #[derive(Debug, Clone, Default, Deserialize)] @@ -16,7 +15,6 @@ pub struct Metadata { #[derive(Debug, Clone, Deserialize)] #[serde(bound = "E: EthSpec")] pub struct SanitySlots { - pub path: PathBuf, pub metadata: Metadata, pub pre: BeaconState, pub slots: u64, @@ -41,7 +39,6 @@ impl LoadCase for SanitySlots { }; Ok(Self { - path: path.into(), metadata, pre, slots, @@ -58,10 +55,6 @@ impl Case for SanitySlots { .unwrap_or_else(String::new) } - fn path(&self) -> &Path { - &self.path - } - fn result(&self, _case_index: usize) -> Result<(), Error> { self.metadata.bls_setting.unwrap_or_default().check()?; diff --git a/tests/ef_tests/src/cases/ssz_generic.rs b/tests/ef_tests/src/cases/ssz_generic.rs index 5f9cd3faf..ce43f3c50 100644 --- a/tests/ef_tests/src/cases/ssz_generic.rs +++ b/tests/ef_tests/src/cases/ssz_generic.rs @@ -1,12 +1,16 @@ +#![allow(non_snake_case)] + use super::*; use crate::cases::common::{SszStaticType, TestU128, TestU256}; use crate::cases::ssz_static::{check_serialization, check_tree_hash}; use crate::decode::yaml_decode_file; use serde_derive::Deserialize; +use ssz_derive::{Decode, Encode}; use std::fs; use std::path::{Path, PathBuf}; +use tree_hash_derive::TreeHash; use types::typenum::*; -use types::{BitList, BitVector, FixedVector}; +use types::{BitList, BitVector, FixedVector, VariableList}; #[derive(Debug, Clone, Deserialize)] struct Metadata { @@ -54,7 +58,7 @@ macro_rules! type_dispatch { "uint64" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u64>, $($rest)*), "uint128" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* TestU128>, $($rest)*), "uint256" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* TestU256>, $($rest)*), - _ => { println!("unsupported: {}", $value); Ok(()) }, + _ => Err(Error::FailedToParseTest(format!("unsupported: {}", $value))), } }; ($function:ident, @@ -86,7 +90,23 @@ macro_rules! type_dispatch { "2048" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U2048>, $($rest)*), "4096" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U4096>, $($rest)*), "8192" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U8192>, $($rest)*), - _ => { println!("unsupported: {}", $value); Ok(()) }, + _ => Err(Error::FailedToParseTest(format!("unsupported: {}", $value))), + } + }; + ($function:ident, + ($($arg:expr),*), + $base_ty:tt, + <$($param_ty:ty),*>, + [ $value:expr => test_container ] $($rest:tt)*) => { + match $value { + "SingleFieldTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* SingleFieldTestStruct>, $($rest)*), + "SmallTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* SmallTestStruct>, $($rest)*), + "FixedTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* FixedTestStruct>, $($rest)*), + "VarTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* VarTestStruct>, $($rest)*), + "BitsStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* BitsStruct>, $($rest)*), + // TODO: enable ComplexTestStruct + "ComplexTestStruct" => Err(Error::SkippedKnownFailure), + _ => Err(Error::FailedToParseTest(format!("unsupported: {}", $value))), } }; // No base type: apply type params to function @@ -99,10 +119,6 @@ macro_rules! type_dispatch { } impl Case for SszGeneric { - fn path(&self) -> &Path { - &self.path - } - fn result(&self, _case_index: usize) -> Result<(), Error> { let parts = self.case_name.split('_').collect::>(); @@ -162,7 +178,17 @@ impl Case for SszGeneric { [type_name.as_str() => primitive_type] )?; } - // FIXME(michael): support for the containers tests + "containers" => { + let type_name = parts[0]; + + type_dispatch!( + ssz_generic_test, + (&self.path), + _, + <>, + [type_name => test_container] + )?; + } _ => panic!("unsupported handler: {}", self.handler_name), } Ok(()) @@ -187,7 +213,7 @@ fn ssz_generic_test(path: &Path) -> Result<(), Error> { }; // Valid - // TODO: signing root + // TODO: signing root (annoying because of traits) if let Some(value) = value { check_serialization(&value, &serialized)?; @@ -207,3 +233,38 @@ fn ssz_generic_test(path: &Path) -> Result<(), Error> { Ok(()) } + +// Containers for SSZ generic tests +#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)] +struct SingleFieldTestStruct { + A: u8, +} + +#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)] +struct SmallTestStruct { + A: u16, + B: u16, +} + +#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)] +struct FixedTestStruct { + A: u8, + B: u64, + C: u32, +} + +#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)] +struct VarTestStruct { + A: u16, + B: VariableList, + C: u8, +} + +#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)] +struct BitsStruct { + A: BitList, + B: BitVector, + C: BitVector, + D: BitList, + E: BitVector, +} diff --git a/tests/ef_tests/src/cases/ssz_static.rs b/tests/ef_tests/src/cases/ssz_static.rs index d1c9b1048..6e4a672cb 100644 --- a/tests/ef_tests/src/cases/ssz_static.rs +++ b/tests/ef_tests/src/cases/ssz_static.rs @@ -28,7 +28,6 @@ pub struct SszStaticSR { } fn load_from_dir(path: &Path) -> Result<(SszStaticRoots, Vec, T), Error> { - // FIXME(michael): set description/name let roots = yaml_decode_file(&path.join("roots.yaml"))?; let serialized = fs::read(&path.join("serialized.ssz")).expect("serialized.ssz exists"); let value = yaml_decode_file(&path.join("value.yaml"))?; diff --git a/tests/ef_tests/src/handler.rs b/tests/ef_tests/src/handler.rs index b6334c383..e5d175e11 100644 --- a/tests/ef_tests/src/handler.rs +++ b/tests/ef_tests/src/handler.rs @@ -32,19 +32,21 @@ pub trait Handler { .join(Self::handler_name()); // Iterate through test suites - // TODO: parallelism - // TODO: error handling? let test_cases = fs::read_dir(&handler_path) - .expect("open main directory") + .expect("handler dir exists") .flat_map(|entry| { entry .ok() .filter(|e| e.file_type().map(|ty| ty.is_dir()).unwrap_or(false)) }) - .flat_map(|suite| fs::read_dir(suite.path()).expect("open suite dir")) + .flat_map(|suite| fs::read_dir(suite.path()).expect("suite dir exists")) .flat_map(Result::ok) - .map(|test_case_dir| Self::Case::load_from_dir(&test_case_dir.path()).expect("loads")) - .collect::>(); + .map(|test_case_dir| { + let path = test_case_dir.path(); + let case = Self::Case::load_from_dir(&path).expect("test should load"); + (path, case) + }) + .collect(); let results = Cases { test_cases }.test_results(); @@ -286,3 +288,5 @@ pub struct Boolean; type_name!(Boolean, "boolean"); pub struct Uints; type_name!(Uints, "uints"); +pub struct Containers; +type_name!(Containers, "containers"); diff --git a/tests/ef_tests/tests/tests.rs b/tests/ef_tests/tests/tests.rs index 71fa53c66..337c54b46 100644 --- a/tests/ef_tests/tests/tests.rs +++ b/tests/ef_tests/tests/tests.rs @@ -1,19 +1,5 @@ use ef_tests::*; -use types::{ - Attestation, AttestationData, AttestationDataAndCustodyBit, AttesterSlashing, BeaconBlock, - BeaconBlockBody, BeaconBlockHeader, BeaconState, Checkpoint, CompactCommittee, Crosslink, - Deposit, DepositData, Eth1Data, Fork, HistoricalBatch, IndexedAttestation, MainnetEthSpec, - MinimalEthSpec, PendingAttestation, ProposerSlashing, Transfer, Validator, VoluntaryExit, -}; - -#[test] -fn ssz_generic() { - SszGenericHandler::::run(); - SszGenericHandler::::run(); - SszGenericHandler::::run(); - SszGenericHandler::::run(); - SszGenericHandler::::run(); -} +use types::*; #[test] fn shuffling() { @@ -105,6 +91,7 @@ fn bls_sign_msg() { BlsSignMsgHandler::run(); } +#[cfg(feature = "fake_crypto")] macro_rules! ssz_static_test { // Signed-root ($test_name:ident, $typ:ident$(<$generics:tt>)?, SR) => { @@ -135,7 +122,6 @@ macro_rules! ssz_static_test { // Base case ($test_name:ident, $handler:ident, { $(($typ:ty, $spec:ident)),+ }) => { #[test] - #[cfg(feature = "fake_crypto")] fn $test_name() { $( $handler::<$typ, $spec>::run(); @@ -144,31 +130,47 @@ macro_rules! ssz_static_test { }; } -ssz_static_test!(ssz_static_attestation, Attestation<_>, SR); -ssz_static_test!(ssz_static_attestation_data, AttestationData); -ssz_static_test!( - ssz_static_attestation_data_and_custody_bit, - AttestationDataAndCustodyBit -); -ssz_static_test!(ssz_static_attester_slashing, AttesterSlashing<_>); -ssz_static_test!(ssz_static_beacon_block, BeaconBlock<_>, SR); -ssz_static_test!(ssz_static_beacon_block_body, BeaconBlockBody<_>); -ssz_static_test!(ssz_static_beacon_block_header, BeaconBlockHeader, SR); -ssz_static_test!(ssz_static_beacon_state, BeaconState<_>); -ssz_static_test!(ssz_static_checkpoint, Checkpoint); -ssz_static_test!(ssz_static_compact_committee, CompactCommittee<_>); -ssz_static_test!(ssz_static_crosslink, Crosslink); -ssz_static_test!(ssz_static_deposit, Deposit); -ssz_static_test!(ssz_static_deposit_data, DepositData, SR); -ssz_static_test!(ssz_static_eth1_data, Eth1Data); -ssz_static_test!(ssz_static_fork, Fork); -ssz_static_test!(ssz_static_historical_batch, HistoricalBatch<_>); -ssz_static_test!(ssz_static_indexed_attestation, IndexedAttestation<_>, SR); -ssz_static_test!(ssz_static_pending_attestation, PendingAttestation<_>); -ssz_static_test!(ssz_static_proposer_slashing, ProposerSlashing); -ssz_static_test!(ssz_static_transfer, Transfer, SR); -ssz_static_test!(ssz_static_validator, Validator); -ssz_static_test!(ssz_static_voluntary_exit, VoluntaryExit, SR); +#[cfg(feature = "fake_crypto")] +mod ssz_static { + use ef_tests::{Handler, SszStaticHandler, SszStaticSRHandler}; + use types::*; + + ssz_static_test!(attestation, Attestation<_>, SR); + ssz_static_test!(attestation_data, AttestationData); + ssz_static_test!( + attestation_data_and_custody_bit, + AttestationDataAndCustodyBit + ); + ssz_static_test!(attester_slashing, AttesterSlashing<_>); + ssz_static_test!(beacon_block, BeaconBlock<_>, SR); + ssz_static_test!(beacon_block_body, BeaconBlockBody<_>); + ssz_static_test!(beacon_block_header, BeaconBlockHeader, SR); + ssz_static_test!(beacon_state, BeaconState<_>); + ssz_static_test!(checkpoint, Checkpoint); + ssz_static_test!(compact_committee, CompactCommittee<_>); + ssz_static_test!(crosslink, Crosslink); + ssz_static_test!(deposit, Deposit); + ssz_static_test!(deposit_data, DepositData, SR); + ssz_static_test!(eth1_data, Eth1Data); + ssz_static_test!(fork, Fork); + ssz_static_test!(historical_batch, HistoricalBatch<_>); + ssz_static_test!(indexed_attestation, IndexedAttestation<_>, SR); + ssz_static_test!(pending_attestation, PendingAttestation<_>); + ssz_static_test!(proposer_slashing, ProposerSlashing); + ssz_static_test!(transfer, Transfer, SR); + ssz_static_test!(validator, Validator); + ssz_static_test!(voluntary_exit, VoluntaryExit, SR); +} + +#[test] +fn ssz_generic() { + SszGenericHandler::::run(); + SszGenericHandler::::run(); + SszGenericHandler::::run(); + SszGenericHandler::::run(); + SszGenericHandler::::run(); + SszGenericHandler::::run(); +} #[test] fn epoch_processing_justification_and_finalization() {