Improve ef_tests, pass all ssz_static tests

tests/ef_tests/Cargo.toml

@@ -11,10 +11,13 @@ fake_crypto = ["bls/fake_crypto"]
 bls = { path = "../../eth2/utils/bls" }
 ethereum-types = "0.5"
 hex = "0.3"
+rayon = "1.0"
 serde = "1.0"
 serde_derive = "1.0"
 serde_yaml = "0.8"
 ssz = { path = "../../eth2/utils/ssz" }
 tree_hash = { path = "../../eth2/utils/tree_hash" }
+cached_tree_hash = { path = "../../eth2/utils/cached_tree_hash" }
 types = { path = "../../eth2/types" }
+walkdir = "2"
 yaml-rust = { git = "https://github.com/sigp/yaml-rust", branch = "escape_all_str"}

tests/ef_tests/src/cases/ssz_static.rs

@@ -1,5 +1,7 @@
 use super::*;
 use crate::case_result::compare_result;
+use cached_tree_hash::{CachedTreeHash, TreeHashCache};
+use rayon::prelude::*;
 use serde_derive::Deserialize;
 use ssz::Decode;
 use std::fmt::Debug;
@@ -48,7 +50,7 @@ impl SszStatic {
 impl EfTest for Cases<SszStatic> {
     fn test_results<E: EthSpec>(&self) -> Vec<CaseResult> {
         self.test_cases
-            .iter()
+            .par_iter()
             .enumerate()
             .map(|(i, tc)| {
                 let result = match tc.type_name.as_ref() {
@@ -88,7 +90,7 @@ impl EfTest for Cases<SszStatic> {
 
 fn ssz_static_test<T>(tc: &SszStatic) -> Result<(), Error>
 where
-    T: Decode + Debug + PartialEq<T> + serde::de::DeserializeOwned + TreeHash,
+    T: Decode + Debug + PartialEq<T> + serde::de::DeserializeOwned + TreeHash + CachedTreeHash,
 {
     // Verify we can decode SSZ in the same way we can decode YAML.
     let ssz = hex::decode(&tc.serialized[2..])
@@ -97,12 +99,18 @@ where
     let decode_result = T::from_ssz_bytes(&ssz);
     compare_result(&decode_result, &Some(expected))?;
 
-    // Verify the tree hash root is identical to the decoded struct.
+    // Verify the TreeHash root of the decoded struct matches the test.
-    let root_bytes =
+    let decoded = decode_result.unwrap();
+    let expected_root =
         &hex::decode(&tc.root[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
-    let expected_root = Hash256::from_slice(&root_bytes);
+    let expected_root = Hash256::from_slice(&expected_root);
-    let root = Hash256::from_slice(&decode_result.unwrap().tree_hash_root());
+    let tree_hash_root = Hash256::from_slice(&decoded.tree_hash_root());
-    compare_result::<Hash256, Error>(&Ok(root), &Some(expected_root))?;
+    compare_result::<Hash256, Error>(&Ok(tree_hash_root), &Some(expected_root))?;
+
+    // Verify a _new_ CachedTreeHash root of the decoded struct matches the test.
+    let cache = TreeHashCache::new(&decoded).unwrap();
+    let cached_tree_hash_root = Hash256::from_slice(cache.tree_hash_root().unwrap());
+    compare_result::<Hash256, Error>(&Ok(cached_tree_hash_root), &Some(expected_root))?;
 
     Ok(())
 }

tests/ef_tests/src/doc.rs

@@ -1,12 +1,12 @@
 use crate::case_result::CaseResult;
 use crate::cases::*;
 use crate::doc_header::DocHeader;
-use crate::eth_specs::MinimalEthSpec;
+use crate::eth_specs::{MainnetEthSpec, MinimalEthSpec};
 use crate::yaml_decode::{extract_yaml_by_key, YamlDecode};
 use crate::EfTest;
 use serde_derive::Deserialize;
 use std::{fs::File, io::prelude::*, path::PathBuf};
-use types::{EthSpec, FoundationEthSpec};
+use types::EthSpec;
 
 #[derive(Debug, Deserialize)]
 pub struct Doc {
@@ -32,8 +32,9 @@ impl Doc {
             header.handler.as_ref(),
             header.config.as_ref(),
         ) {
-            ("ssz", "uint", _) => run_test::<SszGeneric, FoundationEthSpec>(&self.yaml),
+            ("ssz", "uint", _) => run_test::<SszGeneric, MainnetEthSpec>(&self.yaml),
             ("ssz", "static", "minimal") => run_test::<SszStatic, MinimalEthSpec>(&self.yaml),
+            ("ssz", "static", "mainnet") => run_test::<SszStatic, MainnetEthSpec>(&self.yaml),
             (runner, handler, config) => panic!(
                 "No implementation for runner: \"{}\", handler: \"{}\", config: \"{}\"",
                 runner, handler, config
@@ -48,6 +49,8 @@ impl Doc {
         if results.iter().any(|r| r.result.is_err()) {
             print_failures(&doc, &results);
             panic!("Tests failed (see above)");
+        } else {
+            println!("Passed {} tests in {:?}", results.len(), doc.path);
         }
     }
 }

tests/ef_tests/src/eth_specs.rs

@@ -1,5 +1,8 @@
-use types::{EthSpec, typenum::{U64, U8}, ChainSpec, FewValidatorsEthSpec};
+use serde_derive::{Deserialize, Serialize};
-use serde_derive::{Serialize, Deserialize};
+use types::{
+    typenum::{U64, U8},
+    ChainSpec, EthSpec, FewValidatorsEthSpec, FoundationEthSpec,
+};
 
 /// "Minimal" testing specification, as defined here:
 ///
@@ -21,3 +24,5 @@ impl EthSpec for MinimalEthSpec {
         FewValidatorsEthSpec::spec()
     }
 }
+
+pub type MainnetEthSpec = FoundationEthSpec;

tests/ef_tests/tests/tests.rs

@@ -1,51 +1,45 @@
 use ef_tests::*;
+use rayon::prelude::*;
 use std::path::PathBuf;
+use walkdir::WalkDir;
 
-fn test_file(trailing_path: &str) -> PathBuf {
+fn yaml_files_in_test_dir(dir: &str) -> Vec<PathBuf> {
-    let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    let mut base_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-    file_path_buf.push(format!("eth2.0-spec-tests/tests/{}", trailing_path));
+    base_path.push("eth2.0-spec-tests");
+    base_path.push("tests");
+    base_path.push(dir);
 
-    file_path_buf
+    WalkDir::new(base_path)
+        .into_iter()
+        .filter_map(|e| e.ok())
+        .filter_map(|entry| {
+            if entry.file_type().is_file() {
+                match entry.file_name().to_str() {
+                    Some(f) if f.ends_with(".yaml") => Some(entry.path().to_path_buf()),
+                    Some(f) if f.ends_with(".yml") => Some(entry.path().to_path_buf()),
+                    _ => None,
                 }
-
+            } else {
-mod ssz_generic {
+                None
-    use super::*;
+            }
-
+        })
-    fn ssz_generic_file(file: &str) -> PathBuf {
+        .collect()
-        let mut path = test_file("ssz_generic");
-        path.push(file);
-
-        path
 }
 
 #[test]
-    fn uint_bounds() {
+fn ssz_generic() {
-        Doc::assert_tests_pass(ssz_generic_file("uint/uint_bounds.yaml"));
+    yaml_files_in_test_dir("ssz_generic")
+        .into_par_iter()
+        .for_each(|file| {
+            Doc::assert_tests_pass(file);
+        });
 }
 
 #[test]
-    fn uint_random() {
+fn ssz_static() {
-        Doc::assert_tests_pass(ssz_generic_file("uint/uint_random.yaml"));
+    yaml_files_in_test_dir("ssz_static")
-    }
+        .into_par_iter()
-
+        .for_each(|file| {
-    #[test]
+            Doc::assert_tests_pass(file);
-    fn uint_wrong_length() {
+        });
-        Doc::assert_tests_pass(ssz_generic_file("uint/uint_wrong_length.yaml"));
-    }
-}
-
-mod ssz_static {
-    use super::*;
-
-    fn ssz_generic_file(file: &str) -> PathBuf {
-        let mut path = test_file("ssz_static");
-        path.push(file);
-
-        path
-    }
-
-    #[test]
-    fn minimal_nil() {
-        Doc::assert_tests_pass(ssz_generic_file("core/ssz_minimal_nil.yaml"));
-    }
 }