ef_tests: skipped tests and bls_setting support

tests/ef_tests/Cargo.toml

@@ -15,6 +15,7 @@ hex = "0.3"
 rayon = "1.0"
 serde = "1.0"
 serde_derive = "1.0"
+serde_repr = "0.1"
 serde_yaml = "0.8"
 ssz = { path = "../../eth2/utils/ssz" }
 tree_hash = { path = "../../eth2/utils/tree_hash" }

tests/ef_tests/src/bls_setting.rs

@@ -0,0 +1,30 @@
+use self::BlsSetting::*;
+use crate::error::Error;
+use serde_repr::Deserialize_repr;
+
+// TODO: use this in every test case
+#[derive(Deserialize_repr, Debug, Clone, Copy)]
+#[repr(u8)]
+pub enum BlsSetting {
+    Flexible = 0,
+    Required = 1,
+    Ignored = 2,
+}
+
+impl Default for BlsSetting {
+    fn default() -> Self {
+        Flexible
+    }
+}
+
+impl BlsSetting {
+    /// Check the BLS setting and skip the test if it isn't compatible with the crypto config.
+    pub fn check(self) -> Result<(), Error> {
+        match self {
+            Flexible => Ok(()),
+            Required if !cfg!(feature = "fake_crypto") => Ok(()),
+            Ignored if cfg!(feature = "fake_crypto") => Ok(()),
+            _ => Err(Error::Skipped),
+        }
+    }
+}

tests/ef_tests/src/doc.rs

@@ -146,7 +146,14 @@ where
 
 pub fn print_failures(doc: &Doc, results: &[CaseResult]) {
     let header: DocHeader = serde_yaml::from_str(&doc.header_yaml).unwrap();
-    let failures: Vec<&CaseResult> = results.iter().filter(|r| r.result.is_err()).collect();
+    let failures: Vec<&CaseResult> = results
+        .iter()
+        .filter(|r| r.result.as_ref().err().map_or(false, |e| !e.is_skipped()))
+        .collect();
+    let skipped: Vec<&CaseResult> = results
+        .iter()
+        .filter(|r| r.result.as_ref().err().map_or(false, |e| e.is_skipped()))
+        .collect();
 
     println!("--------------------------------------------------");
     println!("Test Failure");
@@ -154,13 +161,18 @@ pub fn print_failures(doc: &Doc, results: &[CaseResult]) {
     println!("File: {:?}", doc.path);
     println!("");
     println!(
-        "{} tests, {} failures, {} passes.",
+        "{} tests, {} failures, {} skipped, {} passes.",
         results.len(),
         failures.len(),
-        results.len() - failures.len()
+        skipped.len(),
+        results.len() - skipped.len() - failures.len()
     );
     println!("");
 
+    for case in skipped {
+        println!("-------");
+        println!("case[{}] ({}) skipped", case.case_index, case.desc);
+    }
     for failure in failures {
         let error = failure.result.clone().unwrap_err();
 

tests/ef_tests/src/error.rs

@@ -6,6 +6,8 @@ pub enum Error {
     DidntFail(String),
     /// Failed to parse the test (internal error).
     FailedToParseTest(String),
+    /// Skipped the test.
+    Skipped,
 }
 
 impl Error {
@@ -14,6 +16,7 @@ impl Error {
             Error::NotEqual(_) => "NotEqual",
             Error::DidntFail(_) => "DidntFail",
             Error::FailedToParseTest(_) => "FailedToParseTest",
+            Error::Skipped => "Skipped",
         }
     }
 
@@ -22,6 +25,14 @@ impl Error {
             Error::NotEqual(m) => m.as_str(),
             Error::DidntFail(m) => m.as_str(),
             Error::FailedToParseTest(m) => m.as_str(),
+            Error::Skipped => panic!(), // "Skipped",
+        }
+    }
+
+    pub fn is_skipped(&self) -> bool {
+        match self {
+            Error::Skipped => true,
+            _ => false,
         }
     }
 }

tests/ef_tests/src/lib.rs

@@ -6,6 +6,7 @@ pub use doc::Doc;
 pub use error::Error;
 pub use yaml_decode::YamlDecode;
 
+mod bls_setting;
 mod case_result;
 mod cases;
 mod doc;