use crate::case_result::CaseResult; use crate::cases::*; use crate::doc_header::DocHeader; use crate::error::Error; use crate::yaml_decode::{yaml_split_header_and_cases, YamlDecode}; use crate::EfTest; use serde_derive::Deserialize; use std::{fs::File, io::prelude::*, path::PathBuf}; use types::{MainnetEthSpec, MinimalEthSpec}; #[derive(Debug, Deserialize)] pub struct Doc { pub header_yaml: String, pub cases_yaml: String, pub path: PathBuf, } impl Doc { fn from_path(path: PathBuf) -> Self { let mut file = File::open(path.clone()).unwrap(); let mut yaml = String::new(); file.read_to_string(&mut yaml).unwrap(); let (header_yaml, cases_yaml) = yaml_split_header_and_cases(yaml.clone()); Self { header_yaml, cases_yaml, path, } } pub fn test_results(&self) -> Vec { let header: DocHeader = serde_yaml::from_str(&self.header_yaml.as_str()).unwrap(); match ( header.runner.as_ref(), header.handler.as_ref(), header.config.as_ref(), ) { ("ssz", "uint", _) => run_test::(self), ("ssz", "static", "minimal") => run_test::>(self), ("ssz", "static", "mainnet") => run_test::>(self), ("sanity", "slots", "minimal") => run_test::>(self), ("sanity", "slots", "mainnet") => run_test::>(self), ("sanity", "blocks", "minimal") => run_test::>(self), ("sanity", "blocks", "mainnet") => run_test::>(self), ("shuffling", "core", "minimal") => run_test::>(self), ("shuffling", "core", "mainnet") => run_test::>(self), ("bls", "aggregate_pubkeys", "mainnet") => run_test::(self), ("bls", "aggregate_sigs", "mainnet") => run_test::(self), ("bls", "msg_hash_compressed", "mainnet") => run_test::(self), // Note this test fails due to a difference in our internal representations. It does // not effect verification or external representation. // // It is skipped. ("bls", "msg_hash_uncompressed", "mainnet") => vec![], ("bls", "priv_to_pub", "mainnet") => run_test::(self), ("bls", "sign_msg", "mainnet") => run_test::(self), ("operations", "deposit", "mainnet") => { run_test::>(self) } ("operations", "deposit", "minimal") => { run_test::>(self) } ("operations", "transfer", "mainnet") => { run_test::>(self) } ("operations", "transfer", "minimal") => { run_test::>(self) } ("operations", "voluntary_exit", "mainnet") => { run_test::>(self) } ("operations", "voluntary_exit", "minimal") => { run_test::>(self) } ("operations", "proposer_slashing", "mainnet") => { run_test::>(self) } ("operations", "proposer_slashing", "minimal") => { run_test::>(self) } ("operations", "attester_slashing", "mainnet") => { run_test::>(self) } ("operations", "attester_slashing", "minimal") => { run_test::>(self) } ("operations", "attestation", "mainnet") => { run_test::>(self) } ("operations", "attestation", "minimal") => { run_test::>(self) } ("operations", "block_header", "mainnet") => { run_test::>(self) } ("operations", "block_header", "minimal") => { run_test::>(self) } ("epoch_processing", "crosslinks", "minimal") => { run_test::>(self) } ("epoch_processing", "crosslinks", "mainnet") => { run_test::>(self) } ("epoch_processing", "registry_updates", "minimal") => { run_test::>(self) } ("epoch_processing", "registry_updates", "mainnet") => { run_test::>(self) } (runner, handler, config) => panic!( "No implementation for runner: \"{}\", handler: \"{}\", config: \"{}\"", runner, handler, config ), } } pub fn assert_tests_pass(path: PathBuf) { let doc = Self::from_path(path); let results = doc.test_results(); let (failed, skipped_bls, skipped_known_failures) = categorize_results(&results); if failed.len() + skipped_known_failures.len() > 0 { print_results( &doc, &failed, &skipped_bls, &skipped_known_failures, &results, ); if !failed.is_empty() { panic!("Tests failed (see above)"); } } else { println!("Passed {} tests in {:?}", results.len(), doc.path); } } } pub fn run_test(doc: &Doc) -> Vec where Cases: EfTest + YamlDecode, { // Pass only the "test_cases" YAML string to `yaml_decode`. let test_cases: Cases = Cases::yaml_decode(&doc.cases_yaml).unwrap(); test_cases.test_results() } pub fn categorize_results( results: &[CaseResult], ) -> (Vec<&CaseResult>, Vec<&CaseResult>, Vec<&CaseResult>) { let mut failed = vec![]; let mut skipped_bls = vec![]; let mut skipped_known_failures = vec![]; for case in results { match case.result.as_ref().err() { Some(Error::SkippedBls) => skipped_bls.push(case), Some(Error::SkippedKnownFailure) => skipped_known_failures.push(case), Some(_) => failed.push(case), None => (), } } (failed, skipped_bls, skipped_known_failures) } pub fn print_results( doc: &Doc, failed: &[&CaseResult], skipped_bls: &[&CaseResult], skipped_known_failures: &[&CaseResult], results: &[CaseResult], ) { let header: DocHeader = serde_yaml::from_str(&doc.header_yaml).unwrap(); println!("--------------------------------------------------"); println!( "Test {}", if failed.is_empty() { "Result" } else { "Failure" } ); println!("Title: {}", header.title); println!("File: {:?}", doc.path); println!(); println!( "{} tests, {} failed, {} skipped (known failure), {} skipped (bls), {} passed.", results.len(), failed.len(), skipped_known_failures.len(), skipped_bls.len(), results.len() - skipped_bls.len() - skipped_known_failures.len() - failed.len() ); println!(); for case in skipped_known_failures { println!("-------"); println!( "case[{}] ({}) skipped because it's a known failure", case.case_index, case.desc, ); } for failure in failed { let error = failure.result.clone().unwrap_err(); println!("-------"); println!( "case[{}] ({}) failed with {}:", failure.case_index, failure.desc, error.name() ); println!("{}", error.message()); } println!(); }