From c6499881896ae0c485df28989fe98af72bb81a18 Mon Sep 17 00:00:00 2001
From: Michael Sproul <michael@sigmaprime.io>
Date: Tue, 11 Jun 2019 18:06:15 +1000
Subject: [PATCH] ef_tests: skipped tests and bls_setting support

---
 tests/ef_tests/Cargo.toml         |  1 +
 tests/ef_tests/src/bls_setting.rs | 30 ++++++++++++++++++++++++++++++
 tests/ef_tests/src/doc.rs         | 18 +++++++++++++++---
 tests/ef_tests/src/error.rs       | 11 +++++++++++
 tests/ef_tests/src/lib.rs         |  1 +
 5 files changed, 58 insertions(+), 3 deletions(-)
 create mode 100644 tests/ef_tests/src/bls_setting.rs

diff --git a/tests/ef_tests/Cargo.toml b/tests/ef_tests/Cargo.toml
index b0a5e27c5..e199f4cb6 100644
--- a/tests/ef_tests/Cargo.toml
+++ b/tests/ef_tests/Cargo.toml
@@ -15,6 +15,7 @@ hex = "0.3"
 rayon = "1.0"
 serde = "1.0"
 serde_derive = "1.0"
+serde_repr = "0.1"
 serde_yaml = "0.8"
 ssz = { path = "../../eth2/utils/ssz" }
 tree_hash = { path = "../../eth2/utils/tree_hash" }
diff --git a/tests/ef_tests/src/bls_setting.rs b/tests/ef_tests/src/bls_setting.rs
new file mode 100644
index 000000000..056cb4748
--- /dev/null
+++ b/tests/ef_tests/src/bls_setting.rs
@@ -0,0 +1,30 @@
+use self::BlsSetting::*;
+use crate::error::Error;
+use serde_repr::Deserialize_repr;
+
+// TODO: use this in every test case
+#[derive(Deserialize_repr, Debug, Clone, Copy)]
+#[repr(u8)]
+pub enum BlsSetting {
+    Flexible = 0,
+    Required = 1,
+    Ignored = 2,
+}
+
+impl Default for BlsSetting {
+    fn default() -> Self {
+        Flexible
+    }
+}
+
+impl BlsSetting {
+    /// Check the BLS setting and skip the test if it isn't compatible with the crypto config.
+    pub fn check(self) -> Result<(), Error> {
+        match self {
+            Flexible => Ok(()),
+            Required if !cfg!(feature = "fake_crypto") => Ok(()),
+            Ignored if cfg!(feature = "fake_crypto") => Ok(()),
+            _ => Err(Error::Skipped),
+        }
+    }
+}
diff --git a/tests/ef_tests/src/doc.rs b/tests/ef_tests/src/doc.rs
index 48d46e2b7..6116340c8 100644
--- a/tests/ef_tests/src/doc.rs
+++ b/tests/ef_tests/src/doc.rs
@@ -146,7 +146,14 @@ where
 
 pub fn print_failures(doc: &Doc, results: &[CaseResult]) {
     let header: DocHeader = serde_yaml::from_str(&doc.header_yaml).unwrap();
-    let failures: Vec<&CaseResult> = results.iter().filter(|r| r.result.is_err()).collect();
+    let failures: Vec<&CaseResult> = results
+        .iter()
+        .filter(|r| r.result.as_ref().err().map_or(false, |e| !e.is_skipped()))
+        .collect();
+    let skipped: Vec<&CaseResult> = results
+        .iter()
+        .filter(|r| r.result.as_ref().err().map_or(false, |e| e.is_skipped()))
+        .collect();
 
     println!("--------------------------------------------------");
     println!("Test Failure");
@@ -154,13 +161,18 @@ pub fn print_failures(doc: &Doc, results: &[CaseResult]) {
     println!("File: {:?}", doc.path);
     println!("");
     println!(
-        "{} tests, {} failures, {} passes.",
+        "{} tests, {} failures, {} skipped, {} passes.",
         results.len(),
         failures.len(),
-        results.len() - failures.len()
+        skipped.len(),
+        results.len() - skipped.len() - failures.len()
     );
     println!("");
 
+    for case in skipped {
+        println!("-------");
+        println!("case[{}] ({}) skipped", case.case_index, case.desc);
+    }
     for failure in failures {
         let error = failure.result.clone().unwrap_err();
 
diff --git a/tests/ef_tests/src/error.rs b/tests/ef_tests/src/error.rs
index cd812d2fd..16d5192a3 100644
--- a/tests/ef_tests/src/error.rs
+++ b/tests/ef_tests/src/error.rs
@@ -6,6 +6,8 @@ pub enum Error {
     DidntFail(String),
     /// Failed to parse the test (internal error).
     FailedToParseTest(String),
+    /// Skipped the test.
+    Skipped,
 }
 
 impl Error {
@@ -14,6 +16,7 @@ impl Error {
             Error::NotEqual(_) => "NotEqual",
             Error::DidntFail(_) => "DidntFail",
             Error::FailedToParseTest(_) => "FailedToParseTest",
+            Error::Skipped => "Skipped",
         }
     }
 
@@ -22,6 +25,14 @@ impl Error {
             Error::NotEqual(m) => m.as_str(),
             Error::DidntFail(m) => m.as_str(),
             Error::FailedToParseTest(m) => m.as_str(),
+            Error::Skipped => panic!(), // "Skipped",
+        }
+    }
+
+    pub fn is_skipped(&self) -> bool {
+        match self {
+            Error::Skipped => true,
+            _ => false,
         }
     }
 }
diff --git a/tests/ef_tests/src/lib.rs b/tests/ef_tests/src/lib.rs
index 942a6dbb7..be12d45c3 100644
--- a/tests/ef_tests/src/lib.rs
+++ b/tests/ef_tests/src/lib.rs
@@ -6,6 +6,7 @@ pub use doc::Doc;
 pub use error::Error;
 pub use yaml_decode::YamlDecode;
 
+mod bls_setting;
 mod case_result;
 mod cases;
 mod doc;