From 81cafdc804fb58962e08c291f24e8a2f556ded7b Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 29 Aug 2019 17:41:20 +1000 Subject: [PATCH] Shuffling and sanity tests --- eth2/types/src/checkpoint.rs | 1 - eth2/utils/bls/Cargo.toml | 2 +- tests/ef_tests/src/case_result.rs | 3 + tests/ef_tests/src/cases.rs | 49 +--- tests/ef_tests/src/cases/bls_g2_compressed.rs | 13 +- tests/ef_tests/src/cases/bls_sign_msg.rs | 13 +- tests/ef_tests/src/cases/sanity_blocks.rs | 51 +++- tests/ef_tests/src/cases/sanity_slots.rs | 52 +++- tests/ef_tests/src/cases/shuffling.rs | 14 +- tests/ef_tests/src/doc.rs | 274 ------------------ tests/ef_tests/src/doc_header.rs | 12 - tests/ef_tests/src/handler.rs | 58 +++- tests/ef_tests/src/lib.rs | 4 +- tests/ef_tests/src/results.rs | 91 ++++++ tests/ef_tests/src/yaml_decode.rs | 28 ++ tests/ef_tests/tests/tests.rs | 30 +- 16 files changed, 311 insertions(+), 384 deletions(-) delete mode 100644 tests/ef_tests/src/doc.rs delete mode 100644 tests/ef_tests/src/doc_header.rs create mode 100644 tests/ef_tests/src/results.rs diff --git a/eth2/types/src/checkpoint.rs b/eth2/types/src/checkpoint.rs index 0c7001921..d5d40fa67 100644 --- a/eth2/types/src/checkpoint.rs +++ b/eth2/types/src/checkpoint.rs @@ -3,7 +3,6 @@ use crate::{Epoch, Hash256}; use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; -use tree_hash::TreeHash; use tree_hash_derive::TreeHash; /// Casper FFG checkpoint, used in attestations. diff --git a/eth2/utils/bls/Cargo.toml b/eth2/utils/bls/Cargo.toml index 5989dce07..15f087b80 100644 --- a/eth2/utils/bls/Cargo.toml +++ b/eth2/utils/bls/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Paul Hauner "] edition = "2018" [dependencies] -milagro_bls = { git = "https://github.com/sigp/milagro_bls", tag = "v0.9.0" } +milagro_bls = { git = "https://github.com/michaelsproul/milagro_bls", branch = "little-endian" } eth2_hashing = { path = "../eth2_hashing" } hex = "0.3" rand = "^0.5" diff --git a/tests/ef_tests/src/case_result.rs b/tests/ef_tests/src/case_result.rs index 88fd353a1..add428ec5 100644 --- a/tests/ef_tests/src/case_result.rs +++ b/tests/ef_tests/src/case_result.rs @@ -1,6 +1,7 @@ use super::*; use compare_fields::{CompareFields, Comparison, FieldComparison}; use std::fmt::Debug; +use std::path::PathBuf; use types::BeaconState; pub const MAX_VALUE_STRING_LEN: usize = 500; @@ -9,6 +10,7 @@ pub const MAX_VALUE_STRING_LEN: usize = 500; pub struct CaseResult { pub case_index: usize, pub desc: String, + pub path: PathBuf, pub result: Result<(), Error>, } @@ -17,6 +19,7 @@ impl CaseResult { CaseResult { case_index, desc: case.description(), + path: case.path().into(), result, } } diff --git a/tests/ef_tests/src/cases.rs b/tests/ef_tests/src/cases.rs index 7f6ffb0c4..1216b8728 100644 --- a/tests/ef_tests/src/cases.rs +++ b/tests/ef_tests/src/cases.rs @@ -67,6 +67,11 @@ pub trait Case: Debug { "no description".to_string() } + /// Path to the directory for this test case. + fn path(&self) -> &Path { + Path::new("") + } + /// Execute a test and return the result. /// /// `case_index` reports the index of the case in the set of test cases. It is not strictly @@ -76,19 +81,13 @@ pub trait Case: Debug { pub trait BlsCase: serde::de::DeserializeOwned {} -impl YamlDecode for T -where - T: BlsCase, -{ +impl YamlDecode for T { fn yaml_decode(string: &str) -> Result { serde_yaml::from_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) } } -impl LoadCase for T -where - T: BlsCase, -{ +impl LoadCase for T { fn load_from_dir(path: &Path) -> Result { Self::yaml_decode_file(&path.join("data.yaml")) } @@ -111,37 +110,3 @@ where .collect() } } - -// FIXME(michael): delete this -impl YamlDecode for Cases { - /// Decodes a YAML list of test cases - fn yaml_decode(yaml: &str) -> Result { - let mut p = 0; - let mut elems: Vec<&str> = yaml - .match_indices("\n- ") - // Skip the `\n` used for matching a new line - .map(|(i, _)| i + 1) - .map(|i| { - let yaml_element = &yaml[p..i]; - p = i; - - yaml_element - }) - .collect(); - - elems.push(&yaml[p..]); - - let test_cases = elems - .iter() - .map(|s| { - // Remove the `- ` prefix. - let s = &s[2..]; - // Remove a single level of indenting. - s.replace("\n ", "\n") - }) - .map(|s| T::yaml_decode(&s.to_string()).unwrap()) - .collect(); - - Ok(Self { test_cases }) - } -} diff --git a/tests/ef_tests/src/cases/bls_g2_compressed.rs b/tests/ef_tests/src/cases/bls_g2_compressed.rs index 547d8d03a..f8381f5a7 100644 --- a/tests/ef_tests/src/cases/bls_g2_compressed.rs +++ b/tests/ef_tests/src/cases/bls_g2_compressed.rs @@ -41,14 +41,9 @@ impl Case for BlsG2Compressed { } } -// Converts a vector to u64 (from big endian) +// Converts a vector to u64 (from little endian) fn bytes_to_u64(array: &[u8]) -> u64 { - let mut result: u64 = 0; - for (i, value) in array.iter().rev().enumerate() { - if i == 8 { - break; - } - result += u64::pow(2, i as u32 * 8) * u64::from(*value); - } - result + let mut bytes = [0u8; 8]; + bytes.copy_from_slice(array); + u64::from_le_bytes(bytes) } diff --git a/tests/ef_tests/src/cases/bls_sign_msg.rs b/tests/ef_tests/src/cases/bls_sign_msg.rs index 476ecdefb..18e90896b 100644 --- a/tests/ef_tests/src/cases/bls_sign_msg.rs +++ b/tests/ef_tests/src/cases/bls_sign_msg.rs @@ -41,16 +41,11 @@ impl Case for BlsSign { } } -// Converts a vector to u64 (from big endian) +// Converts a vector to u64 (from little endian) fn bytes_to_u64(array: &[u8]) -> u64 { - let mut result: u64 = 0; - for (i, value) in array.iter().rev().enumerate() { - if i == 8 { - break; - } - result += u64::pow(2, i as u32 * 8) * u64::from(*value); - } - result + let mut bytes = [0u8; 8]; + bytes.copy_from_slice(array); + u64::from_le_bytes(bytes) } // Increase the size of an array to 48 bytes diff --git a/tests/ef_tests/src/cases/sanity_blocks.rs b/tests/ef_tests/src/cases/sanity_blocks.rs index cd9008fda..d88d8f295 100644 --- a/tests/ef_tests/src/cases/sanity_blocks.rs +++ b/tests/ef_tests/src/cases/sanity_blocks.rs @@ -1,35 +1,72 @@ use super::*; use crate::bls_setting::BlsSetting; use crate::case_result::compare_beacon_state_results_without_caches; +use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::{ per_block_processing, per_slot_processing, BlockInvalid, BlockProcessingError, }; +use std::path::PathBuf; use types::{BeaconBlock, BeaconState, EthSpec, RelativeEpoch}; +#[derive(Debug, Clone, Deserialize)] +pub struct Metadata { + pub description: Option, + pub bls_setting: Option, + pub blocks_count: usize, +} + #[derive(Debug, Clone, Deserialize)] #[serde(bound = "E: EthSpec")] pub struct SanityBlocks { - pub description: String, - pub bls_setting: Option, + pub path: PathBuf, + pub metadata: Metadata, pub pre: BeaconState, pub blocks: Vec>, pub post: Option>, } -impl YamlDecode for SanityBlocks { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) +impl LoadCase for SanityBlocks { + fn load_from_dir(path: &Path) -> Result { + let metadata: Metadata = yaml_decode_file(&path.join("meta.yaml"))?; + let pre = ssz_decode_file(&path.join("pre.ssz"))?; + let blocks: Vec> = (0..metadata.blocks_count) + .map(|i| { + let filename = format!("blocks_{}.ssz", i); + ssz_decode_file(&path.join(filename)) + }) + .collect::>()?; + let post_file = path.join("post.ssz"); + let post = if post_file.is_file() { + Some(ssz_decode_file(&post_file)?) + } else { + None + }; + + Ok(Self { + path: path.into(), + metadata, + pre, + blocks, + post, + }) } } impl Case for SanityBlocks { fn description(&self) -> String { - self.description.clone() + self.metadata + .description + .clone() + .unwrap_or_else(String::new) + } + + fn path(&self) -> &Path { + &self.path } fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; + self.metadata.bls_setting.unwrap_or_default().check()?; let mut state = self.pre.clone(); let mut expected = self.post.clone(); diff --git a/tests/ef_tests/src/cases/sanity_slots.rs b/tests/ef_tests/src/cases/sanity_slots.rs index fbce1a06a..27c6c13c3 100644 --- a/tests/ef_tests/src/cases/sanity_slots.rs +++ b/tests/ef_tests/src/cases/sanity_slots.rs @@ -1,30 +1,70 @@ use super::*; +use crate::bls_setting::BlsSetting; use crate::case_result::compare_beacon_state_results_without_caches; +use crate::yaml_decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::per_slot_processing; +use std::path::PathBuf; use types::{BeaconState, EthSpec}; +#[derive(Debug, Clone, Default, Deserialize)] +pub struct Metadata { + pub description: Option, + pub bls_setting: Option, +} + #[derive(Debug, Clone, Deserialize)] #[serde(bound = "E: EthSpec")] pub struct SanitySlots { - pub description: String, + pub path: PathBuf, + pub metadata: Metadata, pub pre: BeaconState, - pub slots: usize, + pub slots: u64, pub post: Option>, } -impl YamlDecode for SanitySlots { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) +impl LoadCase for SanitySlots { + fn load_from_dir(path: &Path) -> Result { + let metadata_path = path.join("meta.yaml"); + let metadata: Metadata = if metadata_path.is_file() { + yaml_decode_file(&path.join("meta.yaml"))? + } else { + Metadata::default() + }; + let pre = ssz_decode_file(&path.join("pre.ssz"))?; + let slots: u64 = yaml_decode_file(&path.join("slots.yaml"))?; + let post_file = path.join("post.ssz"); + let post = if post_file.is_file() { + Some(ssz_decode_file(&post_file)?) + } else { + None + }; + + Ok(Self { + path: path.into(), + metadata, + pre, + slots, + post, + }) } } impl Case for SanitySlots { fn description(&self) -> String { - self.description.clone() + self.metadata + .description + .clone() + .unwrap_or_else(String::new) + } + + fn path(&self) -> &Path { + &self.path } fn result(&self, _case_index: usize) -> Result<(), Error> { + self.metadata.bls_setting.unwrap_or_default().check()?; + let mut state = self.pre.clone(); let mut expected = self.post.clone(); let spec = &E::default_spec(); diff --git a/tests/ef_tests/src/cases/shuffling.rs b/tests/ef_tests/src/cases/shuffling.rs index d7ff40e59..c0595e584 100644 --- a/tests/ef_tests/src/cases/shuffling.rs +++ b/tests/ef_tests/src/cases/shuffling.rs @@ -8,7 +8,7 @@ use swap_or_not_shuffle::{get_permutated_index, shuffle_list}; pub struct Shuffling { pub seed: String, pub count: usize, - pub shuffled: Vec, + pub mapping: Vec, #[serde(skip)] _phantom: PhantomData, } @@ -19,10 +19,16 @@ impl YamlDecode for Shuffling { } } +impl LoadCase for Shuffling { + fn load_from_dir(path: &Path) -> Result { + Self::yaml_decode_file(&path.join("mapping.yaml")) + } +} + impl Case for Shuffling { fn result(&self, _case_index: usize) -> Result<(), Error> { if self.count == 0 { - compare_result::<_, Error>(&Ok(vec![]), &Some(self.shuffled.clone()))?; + compare_result::<_, Error>(&Ok(vec![]), &Some(self.mapping.clone()))?; } else { let spec = T::default_spec(); let seed = hex::decode(&self.seed[2..]) @@ -34,12 +40,12 @@ impl Case for Shuffling { get_permutated_index(i, self.count, &seed, spec.shuffle_round_count).unwrap() }) .collect(); - compare_result::<_, Error>(&Ok(shuffling), &Some(self.shuffled.clone()))?; + compare_result::<_, Error>(&Ok(shuffling), &Some(self.mapping.clone()))?; // Test "shuffle_list" let input: Vec = (0..self.count).collect(); let shuffling = shuffle_list(input, spec.shuffle_round_count, &seed, false).unwrap(); - compare_result::<_, Error>(&Ok(shuffling), &Some(self.shuffled.clone()))?; + compare_result::<_, Error>(&Ok(shuffling), &Some(self.mapping.clone()))?; } Ok(()) diff --git a/tests/ef_tests/src/doc.rs b/tests/ef_tests/src/doc.rs deleted file mode 100644 index f3a41697e..000000000 --- a/tests/ef_tests/src/doc.rs +++ /dev/null @@ -1,274 +0,0 @@ -use crate::case_result::CaseResult; -use crate::cases::*; -use crate::doc_header::DocHeader; -use crate::error::Error; -use crate::yaml_decode::YamlDecode; -use crate::EfTest; -use serde_derive::Deserialize; -use std::{ - fs::File, - io::prelude::*, - path::{Path, PathBuf}, -}; -use types::{MainnetEthSpec, MinimalEthSpec}; - -#[derive(Debug, Deserialize)] -pub struct Doc { - pub header_yaml: String, - pub cases_yaml: String, - pub path: PathBuf, -} - -impl Doc { - fn from_path(path: PathBuf) -> Self { - let mut file = File::open(path.clone()).unwrap(); - - let mut cases_yaml = String::new(); - file.read_to_string(&mut cases_yaml).unwrap(); - - Self { - cases_yaml, - path, - header_yaml: String::new(), - } - } - - pub fn test_results(&self) -> Vec { - let header: DocHeader = serde_yaml::from_str(&self.header_yaml.as_str()).unwrap(); - - match ( - header.runner.as_ref(), - header.handler.as_ref(), - header.config.as_ref(), - ) { - ("ssz", "uint", _) => run_test::(self), - ("sanity", "slots", "minimal") => run_test::>(self), - // FIXME: skipped due to compact committees issue - ("sanity", "slots", "mainnet") => vec![], // run_test::>(self), - ("sanity", "blocks", "minimal") => run_test::>(self), - // FIXME: skipped due to compact committees issue - ("sanity", "blocks", "mainnet") => vec![], // run_test::>(self), - ("shuffling", "core", "minimal") => run_test::>(self), - ("shuffling", "core", "mainnet") => run_test::>(self), - ("bls", "aggregate_pubkeys", "mainnet") => run_test::(self), - ("bls", "aggregate_sigs", "mainnet") => run_test::(self), - ("bls", "msg_hash_compressed", "mainnet") => run_test::(self), - // Note this test fails due to a difference in our internal representations. It does - // not effect verification or external representation. - // - // It is skipped. - ("bls", "msg_hash_uncompressed", "mainnet") => vec![], - ("bls", "priv_to_pub", "mainnet") => run_test::(self), - ("bls", "sign_msg", "mainnet") => run_test::(self), - ("operations", "deposit", "mainnet") => { - run_test::>(self) - } - ("operations", "deposit", "minimal") => { - run_test::>(self) - } - ("operations", "transfer", "mainnet") => { - run_test::>(self) - } - ("operations", "transfer", "minimal") => { - run_test::>(self) - } - ("operations", "voluntary_exit", "mainnet") => { - run_test::>(self) - } - ("operations", "voluntary_exit", "minimal") => { - run_test::>(self) - } - ("operations", "proposer_slashing", "mainnet") => { - run_test::>(self) - } - ("operations", "proposer_slashing", "minimal") => { - run_test::>(self) - } - ("operations", "attester_slashing", "mainnet") => { - run_test::>(self) - } - ("operations", "attester_slashing", "minimal") => { - run_test::>(self) - } - ("operations", "attestation", "mainnet") => { - run_test::>(self) - } - ("operations", "attestation", "minimal") => { - run_test::>(self) - } - ("operations", "block_header", "mainnet") => { - run_test::>(self) - } - ("operations", "block_header", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "crosslinks", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "crosslinks", "mainnet") => { - run_test::>(self) - } - ("epoch_processing", "registry_updates", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "registry_updates", "mainnet") => { - run_test::>(self) - } - ("epoch_processing", "justification_and_finalization", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "justification_and_finalization", "mainnet") => { - run_test::>(self) - } - ("epoch_processing", "slashings", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "slashings", "mainnet") => { - run_test::>(self) - } - ("epoch_processing", "final_updates", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "final_updates", "mainnet") => { - vec![] - // FIXME: skipped due to compact committees issue - // run_test::>(self) - } - ("genesis", "initialization", "minimal") => { - run_test::>(self) - } - ("genesis", "initialization", "mainnet") => { - run_test::>(self) - } - ("genesis", "validity", "minimal") => run_test::>(self), - ("genesis", "validity", "mainnet") => run_test::>(self), - (runner, handler, config) => panic!( - "No implementation for runner: \"{}\", handler: \"{}\", config: \"{}\"", - runner, handler, config - ), - } - } - - pub fn assert_tests_pass(path: PathBuf) { - let doc = Self::from_path(path); - let results = doc.test_results(); - - let (failed, skipped_bls, skipped_known_failures) = categorize_results(&results); - - if failed.len() + skipped_known_failures.len() > 0 { - print_results( - &doc, - &failed, - &skipped_bls, - &skipped_known_failures, - &results, - ); - if !failed.is_empty() { - panic!("Tests failed (see above)"); - } - } else { - println!("Passed {} tests in {:?}", results.len(), doc.path); - } - } -} - -pub fn assert_tests_pass(path: &Path, results: &[CaseResult]) { - let doc = Doc { - header_yaml: String::new(), - cases_yaml: String::new(), - path: path.into(), - }; - - let (failed, skipped_bls, skipped_known_failures) = categorize_results(results); - - if failed.len() + skipped_known_failures.len() > 0 { - print_results( - &doc, - &failed, - &skipped_bls, - &skipped_known_failures, - &results, - ); - if !failed.is_empty() { - panic!("Tests failed (see above)"); - } - } else { - println!("Passed {} tests in {}", results.len(), path.display()); - } -} - -pub fn run_test(_: &Doc) -> Vec -where - Cases: EfTest + YamlDecode, -{ - panic!("FIXME(michael): delete this") -} - -pub fn categorize_results( - results: &[CaseResult], -) -> (Vec<&CaseResult>, Vec<&CaseResult>, Vec<&CaseResult>) { - let mut failed = vec![]; - let mut skipped_bls = vec![]; - let mut skipped_known_failures = vec![]; - - for case in results { - match case.result.as_ref().err() { - Some(Error::SkippedBls) => skipped_bls.push(case), - Some(Error::SkippedKnownFailure) => skipped_known_failures.push(case), - Some(_) => failed.push(case), - None => (), - } - } - - (failed, skipped_bls, skipped_known_failures) -} - -pub fn print_results( - doc: &Doc, - failed: &[&CaseResult], - skipped_bls: &[&CaseResult], - skipped_known_failures: &[&CaseResult], - results: &[CaseResult], -) { - println!("--------------------------------------------------"); - println!( - "Test {}", - if failed.is_empty() { - "Result" - } else { - "Failure" - } - ); - println!("Title: TODO"); - println!("File: {:?}", doc.path); - println!( - "{} tests, {} failed, {} skipped (known failure), {} skipped (bls), {} passed. (See below for errors)", - results.len(), - failed.len(), - skipped_known_failures.len(), - skipped_bls.len(), - results.len() - skipped_bls.len() - skipped_known_failures.len() - failed.len() - ); - println!(); - - for case in skipped_known_failures { - println!("-------"); - println!( - "case[{}] ({}) skipped because it's a known failure", - case.case_index, case.desc, - ); - } - for failure in failed { - let error = failure.result.clone().unwrap_err(); - - println!("-------"); - println!( - "case[{}] ({}) failed with {}:", - failure.case_index, - failure.desc, - error.name() - ); - println!("{}", error.message()); - } - println!(); -} diff --git a/tests/ef_tests/src/doc_header.rs b/tests/ef_tests/src/doc_header.rs deleted file mode 100644 index c0d6d3276..000000000 --- a/tests/ef_tests/src/doc_header.rs +++ /dev/null @@ -1,12 +0,0 @@ -use serde_derive::Deserialize; - -#[derive(Debug, Deserialize)] -pub struct DocHeader { - pub title: String, - pub summary: String, - pub forks_timeline: String, - pub forks: Vec, - pub config: String, - pub runner: String, - pub handler: String, -} diff --git a/tests/ef_tests/src/handler.rs b/tests/ef_tests/src/handler.rs index 1dac988ac..f66dc7b18 100644 --- a/tests/ef_tests/src/handler.rs +++ b/tests/ef_tests/src/handler.rs @@ -5,6 +5,7 @@ use std::fs; use std::marker::PhantomData; use std::path::PathBuf; use tree_hash::SignedRoot; +use types::EthSpec; pub trait Handler { type Case: Case + LoadCase; @@ -47,7 +48,8 @@ pub trait Handler { let results = Cases { test_cases }.test_results(); - crate::doc::assert_tests_pass(&handler_path, &results); + let name = format!("{}/{}", Self::runner_name(), Self::handler_name()); + crate::results::assert_tests_pass(&name, &handler_path, &results); } } @@ -128,3 +130,57 @@ where T::name() } } + +pub struct ShufflingHandler(PhantomData); + +impl Handler for ShufflingHandler { + type Case = cases::Shuffling; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "shuffling" + } + + fn handler_name() -> &'static str { + "core" + } +} + +pub struct SanityBlocksHandler(PhantomData); + +impl Handler for SanityBlocksHandler { + type Case = cases::SanityBlocks; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "sanity" + } + + fn handler_name() -> &'static str { + "blocks" + } +} + +pub struct SanitySlotsHandler(PhantomData); + +impl Handler for SanitySlotsHandler { + type Case = cases::SanitySlots; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "sanity" + } + + fn handler_name() -> &'static str { + "slots" + } +} diff --git a/tests/ef_tests/src/lib.rs b/tests/ef_tests/src/lib.rs index cc17c3ea4..06e01fc03 100644 --- a/tests/ef_tests/src/lib.rs +++ b/tests/ef_tests/src/lib.rs @@ -2,7 +2,6 @@ use types::EthSpec; pub use case_result::CaseResult; pub use cases::Case; -pub use doc::Doc; pub use error::Error; pub use handler::*; pub use yaml_decode::YamlDecode; @@ -10,10 +9,9 @@ pub use yaml_decode::YamlDecode; mod bls_setting; mod case_result; mod cases; -mod doc; -mod doc_header; mod error; mod handler; +mod results; mod type_name; mod yaml_decode; diff --git a/tests/ef_tests/src/results.rs b/tests/ef_tests/src/results.rs new file mode 100644 index 000000000..20e59f7b3 --- /dev/null +++ b/tests/ef_tests/src/results.rs @@ -0,0 +1,91 @@ +use crate::case_result::CaseResult; +use crate::error::Error; +use std::path::Path; + +pub fn assert_tests_pass(handler_name: &str, path: &Path, results: &[CaseResult]) { + let (failed, skipped_bls, skipped_known_failures) = categorize_results(results); + + if failed.len() + skipped_known_failures.len() > 0 { + print_results( + handler_name, + &failed, + &skipped_bls, + &skipped_known_failures, + &results, + ); + if !failed.is_empty() { + panic!("Tests failed (see above)"); + } + } else { + println!("Passed {} tests in {}", results.len(), path.display()); + } +} + +pub fn categorize_results( + results: &[CaseResult], +) -> (Vec<&CaseResult>, Vec<&CaseResult>, Vec<&CaseResult>) { + let mut failed = vec![]; + let mut skipped_bls = vec![]; + let mut skipped_known_failures = vec![]; + + for case in results { + match case.result.as_ref().err() { + Some(Error::SkippedBls) => skipped_bls.push(case), + Some(Error::SkippedKnownFailure) => skipped_known_failures.push(case), + Some(_) => failed.push(case), + None => (), + } + } + + (failed, skipped_bls, skipped_known_failures) +} + +pub fn print_results( + handler_name: &str, + failed: &[&CaseResult], + skipped_bls: &[&CaseResult], + skipped_known_failures: &[&CaseResult], + results: &[CaseResult], +) { + println!("--------------------------------------------------"); + println!( + "Test {}", + if failed.is_empty() { + "Result" + } else { + "Failure" + } + ); + println!("Title: {}", handler_name); + println!( + "{} tests, {} failed, {} skipped (known failure), {} skipped (bls), {} passed. (See below for errors)", + results.len(), + failed.len(), + skipped_known_failures.len(), + skipped_bls.len(), + results.len() - skipped_bls.len() - skipped_known_failures.len() - failed.len() + ); + println!(); + + for case in skipped_known_failures { + println!("-------"); + println!( + "case ({}) from {} skipped because it's a known failure", + case.desc, + case.path.display() + ); + } + for failure in failed { + let error = failure.result.clone().unwrap_err(); + + println!("-------"); + println!( + "case ({}) from {} failed with {}:", + failure.desc, + failure.path.display(), + error.name() + ); + println!("{}", error.message()); + } + println!(); +} diff --git a/tests/ef_tests/src/yaml_decode.rs b/tests/ef_tests/src/yaml_decode.rs index af122fb0c..83a162930 100644 --- a/tests/ef_tests/src/yaml_decode.rs +++ b/tests/ef_tests/src/yaml_decode.rs @@ -4,6 +4,34 @@ use std::fs; use std::path::Path; use types::Fork; +pub fn yaml_decode(string: &str) -> Result { + serde_yaml::from_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) +} + +pub fn yaml_decode_file(path: &Path) -> Result { + fs::read_to_string(path) + .map_err(|e| { + Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e)) + }) + .and_then(|s| yaml_decode(&s)) +} + +pub fn ssz_decode_file(path: &Path) -> Result { + fs::read(path) + .map_err(|e| { + Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e)) + }) + .and_then(|s| { + T::from_ssz_bytes(&s).map_err(|e| { + Error::FailedToParseTest(format!( + "Unable to parse SSZ at {}: {:?}", + path.display(), + e + )) + }) + }) +} + pub trait YamlDecode: Sized { /// Decode an object from the test specification YAML. fn yaml_decode(string: &str) -> Result; diff --git a/tests/ef_tests/tests/tests.rs b/tests/ef_tests/tests/tests.rs index 000c53330..0fb751c9e 100644 --- a/tests/ef_tests/tests/tests.rs +++ b/tests/ef_tests/tests/tests.rs @@ -48,6 +48,7 @@ fn yaml_files_in_test_dir(dir: &Path) -> Vec { paths } +/* #[test] #[cfg(feature = "fake_crypto")] fn ssz_generic() { @@ -58,6 +59,7 @@ fn ssz_generic() { }); } + #[test] #[cfg(feature = "fake_crypto")] fn ssz_static() { @@ -67,16 +69,15 @@ fn ssz_static() { Doc::assert_tests_pass(file); }); } +*/ #[test] fn shuffling() { - yaml_files_in_test_dir(&Path::new("shuffling").join("core")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + ShufflingHandler::::run(); + ShufflingHandler::::run(); } +/* #[test] fn operations_deposit() { yaml_files_in_test_dir(&Path::new("operations").join("deposit")) @@ -140,25 +141,21 @@ fn operations_block_header() { Doc::assert_tests_pass(file); }); } +*/ #[test] fn sanity_blocks() { - yaml_files_in_test_dir(&Path::new("sanity").join("blocks")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + SanityBlocksHandler::::run(); + SanityBlocksHandler::::run(); } #[test] fn sanity_slots() { - yaml_files_in_test_dir(&Path::new("sanity").join("slots")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + SanitySlotsHandler::::run(); + SanitySlotsHandler::::run(); } +/* #[test] #[cfg(not(feature = "fake_crypto"))] fn bls() { @@ -168,6 +165,7 @@ fn bls() { Doc::assert_tests_pass(file); }); } +*/ #[test] #[cfg(not(feature = "fake_crypto"))] @@ -264,6 +262,7 @@ ssz_static_test!(ssz_static_transfer, Transfer, SR); ssz_static_test!(ssz_static_validator, Validator); ssz_static_test!(ssz_static_voluntary_exit, VoluntaryExit, SR); +/* #[test] fn epoch_processing_justification_and_finalization() { yaml_files_in_test_dir(&Path::new("epoch_processing").join("justification_and_finalization")) @@ -326,3 +325,4 @@ fn genesis_validity() { Doc::assert_tests_pass(file); }); } +*/