Parallel tests and SSZ generic

eth2/utils/ssz_types/src/fixed_vector.rs

@@ -220,13 +220,26 @@ where
 
     fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, ssz::DecodeError> {
         if bytes.is_empty() {
-            Ok(FixedVector::from(vec![]))
+            Err(ssz::DecodeError::InvalidByteLength {
+                len: 0,
+                expected: 1,
+            })
         } else if T::is_ssz_fixed_len() {
             bytes
                 .chunks(T::ssz_fixed_len())
                 .map(|chunk| T::from_ssz_bytes(chunk))
                 .collect::<Result<Vec<T>, _>>()
-                .and_then(|vec| Ok(vec.into()))
+                .and_then(|vec| {
+                    if vec.len() == N::to_usize() {
+                        Ok(vec.into())
+                    } else {
+                        Err(ssz::DecodeError::BytesInvalid(format!(
+                            "wrong number of vec elements, got: {}, expected: {}",
+                            vec.len(),
+                            N::to_usize()
+                        )))
+                    }
+                })
         } else {
             ssz::decode_list_of_variable_length_items(bytes).and_then(|vec| Ok(vec.into()))
         }

tests/ef_tests/src/cases.rs

@@ -1,4 +1,5 @@
 use super::*;
+use rayon::prelude::*;
 use std::fmt::Debug;
 use std::path::Path;
 
@@ -39,7 +40,7 @@ pub trait LoadCase: Sized {
     fn load_from_dir(_path: &Path) -> Result<Self, Error>;
 }
 
-pub trait Case: Debug {
+pub trait Case: Debug + Sync {
     /// An optional field for implementing a custom description.
     ///
     /// Defaults to "no description".
@@ -79,13 +80,10 @@ pub struct Cases<T> {
     pub test_cases: Vec<T>,
 }
 
-impl<T> EfTest for Cases<T>
+impl<T: Case> Cases<T> {
-where
+    pub fn test_results(&self) -> Vec<CaseResult> {
-    T: Case + Debug,
-{
-    fn test_results(&self) -> Vec<CaseResult> {
         self.test_cases
-            .iter()
+            .into_par_iter()
             .enumerate()
             .map(|(i, tc)| CaseResult::new(i, tc, tc.result(i)))
             .collect()

tests/ef_tests/src/cases/epoch_processing.rs

@@ -31,7 +31,7 @@ pub struct EpochProcessing<E: EthSpec, T: EpochTransition<E>> {
     _phantom: PhantomData<T>,
 }
 
-pub trait EpochTransition<E: EthSpec>: TypeName + Debug {
+pub trait EpochTransition<E: EthSpec>: TypeName + Debug + Sync {
     fn run(state: &mut BeaconState<E>, spec: &ChainSpec) -> Result<(), EpochProcessingError>;
 }
 

tests/ef_tests/src/cases/operations.rs

@@ -10,6 +10,7 @@ use state_processing::per_block_processing::{
     process_block_header, process_deposits, process_exits, process_proposer_slashings,
     process_transfers,
 };
+use std::fmt::Debug;
 use std::path::{Path, PathBuf};
 use types::{
     Attestation, AttesterSlashing, BeaconBlock, BeaconState, ChainSpec, Deposit, EthSpec,
@@ -31,7 +32,7 @@ pub struct Operations<E: EthSpec, O: Operation<E>> {
     pub post: Option<BeaconState<E>>,
 }
 
-pub trait Operation<E: EthSpec>: Decode + TypeName + std::fmt::Debug {
+pub trait Operation<E: EthSpec>: Decode + TypeName + Debug + Sync {
     fn handler_name() -> String {
         Self::name().to_lowercase()
     }

tests/ef_tests/src/cases/ssz_generic.rs

@@ -1,68 +1,205 @@
 use super::*;
-use crate::case_result::compare_result;
+use crate::cases::ssz_static::{check_serialization, check_tree_hash, SszStaticType};
-use ethereum_types::{U128, U256};
+use crate::yaml_decode::yaml_decode_file;
 use serde_derive::Deserialize;
-use ssz::Decode;
+use std::fs;
-use std::fmt::Debug;
+use std::path::{Path, PathBuf};
+use types::typenum::*;
+use types::{BitList, BitVector, FixedVector, VariableList};
 
 #[derive(Debug, Clone, Deserialize)]
-pub struct SszGeneric {
+struct Metadata {
-    #[serde(alias = "type")]
+    root: String,
-    pub type_name: String,
+    signing_root: Option<String>,
-    pub valid: bool,
-    pub value: Option<String>,
-    pub ssz: Option<String>,
 }
 
-impl YamlDecode for SszGeneric {
+#[derive(Debug, Clone)]
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
+pub struct SszGeneric {
-        Ok(serde_yaml::from_str(yaml).unwrap())
+    path: PathBuf,
+    handler_name: String,
+    case_name: String,
+}
+
+impl LoadCase for SszGeneric {
+    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+        let components = path
+            .components()
+            .map(|c| c.as_os_str().to_string_lossy().into_owned())
+            .rev()
+            .collect::<Vec<_>>();
+        // Test case name is last
+        let case_name = components[0].clone();
+        // Handler name is third last, before suite name and case name
+        let handler_name = components[2].clone();
+        Ok(Self {
+            path: path.into(),
+            handler_name,
+            case_name,
+        })
+    }
+}
+
+macro_rules! type_dispatch {
+    ($function:ident,
+     ($($arg:expr),*),
+     $base_ty:tt,
+     <$($param_ty:ty),*>,
+     [ $value:expr => primitive_type ] $($rest:tt)*) => {
+        match $value {
+            "bool" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* bool>, $($rest)*),
+            "uint8" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u8>, $($rest)*),
+            "uint16" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u16>, $($rest)*),
+            "uint32" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u32>, $($rest)*),
+            "uint64" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u64>, $($rest)*),
+            // FIXME(michael): implement tree hash for big ints
+            // "uint128" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* etherum_types::U128>, $($rest)*),
+            // "uint256" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ethereum_types::U256>, $($rest)*),
+            _ => { println!("unsupported: {}", $value); Ok(()) },
+        }
+    };
+    ($function:ident,
+     ($($arg:expr),*),
+     $base_ty:tt,
+     <$($param_ty:ty),*>,
+     [ $value:expr => typenum ] $($rest:tt)*) => {
+        match $value {
+            // DO YOU LIKE NUMBERS?
+            "0" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U0>, $($rest)*),
+            "1" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U1>, $($rest)*),
+            "2" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U2>, $($rest)*),
+            "3" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U3>, $($rest)*),
+            "4" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U4>, $($rest)*),
+            "5" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U5>, $($rest)*),
+            "6" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U6>, $($rest)*),
+            "7" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U7>, $($rest)*),
+            "8" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U8>, $($rest)*),
+            "9" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U9>, $($rest)*),
+            "16" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U16>, $($rest)*),
+            "31" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U31>, $($rest)*),
+            "32" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U32>, $($rest)*),
+            "64" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U64>, $($rest)*),
+            "128" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U128>, $($rest)*),
+            "256" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U256>, $($rest)*),
+            "512" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U512>, $($rest)*),
+            "513" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U513>, $($rest)*),
+            "1024" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U1024>, $($rest)*),
+            "2048" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U2048>, $($rest)*),
+            "4096" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U4096>, $($rest)*),
+            "8192" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U8192>, $($rest)*),
+            _ => { println!("unsupported: {}", $value); Ok(()) },
+        }
+    };
+    // No base type: apply type params to function
+    ($function:ident, ($($arg:expr),*), _, <$($param_ty:ty),*>,) => {
+        $function::<$($param_ty),*>($($arg),*)
+    };
+    ($function:ident, ($($arg:expr),*), $base_type_name:ident, <$($param_ty:ty),*>,) => {
+        $function::<$base_type_name<$($param_ty),*>>($($arg),*)
     }
 }
 
 impl Case for SszGeneric {
+    fn path(&self) -> &Path {
+        &self.path
+    }
+
     fn result(&self, _case_index: usize) -> Result<(), Error> {
-        if let Some(ssz) = &self.ssz {
+        let parts = self.case_name.split('_').collect::<Vec<_>>();
-            match self.type_name.as_ref() {
+
-                "uint8" => ssz_generic_test::<u8>(self.valid, ssz, &self.value),
+        match self.handler_name.as_str() {
-                "uint16" => ssz_generic_test::<u16>(self.valid, ssz, &self.value),
+            "basic_vector" => {
-                "uint32" => ssz_generic_test::<u32>(self.valid, ssz, &self.value),
+                let elem_ty = parts[1];
-                "uint64" => ssz_generic_test::<u64>(self.valid, ssz, &self.value),
+                let length = parts[2];
-                "uint128" => ssz_generic_test::<U128>(self.valid, ssz, &self.value),
+
-                "uint256" => ssz_generic_test::<U256>(self.valid, ssz, &self.value),
+                type_dispatch!(
-                _ => Err(Error::FailedToParseTest(format!(
+                    ssz_generic_test,
-                    "Unknown type: {}",
+                    (&self.path),
-                    self.type_name
+                    FixedVector,
-                ))),
+                    <>,
+                    [elem_ty => primitive_type]
+                    [length => typenum]
+                )?;
+            }
+            "bitlist" => {
+                let limit = parts[1];
+
+                // FIXME(michael): mark length "no" cases as known failures
+
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path),
+                    BitList,
+                    <>,
+                    [limit => typenum]
+                )?;
+            }
+            "bitvector" => {
+                let length = parts[1];
+
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path),
+                    BitVector,
+                    <>,
+                    [length => typenum]
+                )?;
+            }
+            "boolean" => {
+                ssz_generic_test::<bool>(&self.path)?;
+            }
+            "uints" => {
+                let type_name = "uint".to_owned() + parts[1];
+
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path),
+                    _,
+                    <>,
+                    [type_name.as_str() => primitive_type]
+                )?;
+            }
+            // FIXME(michael): support for the containers tests
+            _ => panic!("unsupported handler: {}", self.handler_name),
         }
-        } else {
-            // Skip tests that do not have an ssz field.
-            //
-            // See: https://github.com/ethereum/eth2.0-specs/issues/1079
         Ok(())
     }
-    }
 }
 
-/// Execute a `ssz_generic` test case.
+fn ssz_generic_test<T: SszStaticType>(path: &Path) -> Result<(), Error> {
-fn ssz_generic_test<T>(should_be_ok: bool, ssz: &str, value: &Option<String>) -> Result<(), Error>
+    let meta_path = path.join("meta.yaml");
-where
+    let meta: Option<Metadata> = if meta_path.is_file() {
-    T: Decode + YamlDecode + Debug + PartialEq<T>,
+        Some(yaml_decode_file(&meta_path)?)
-{
-    let ssz = hex::decode(&ssz[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
-
-    // We do not cater for the scenario where the test is valid but we are not passed any SSZ.
-    if should_be_ok && value.is_none() {
-        panic!("Unexpected test input. Cannot pass without value.")
-    }
-
-    let expected = if let Some(string) = value {
-        Some(T::yaml_decode(string)?)
     } else {
         None
     };
 
-    let decoded = T::from_ssz_bytes(&ssz);
+    let serialized = fs::read(&path.join("serialized.ssz")).expect("serialized.ssz exists");
 
-    compare_result(&decoded, &expected)
+    let value_path = path.join("value.yaml");
+    let value: Option<T> = if value_path.is_file() {
+        Some(yaml_decode_file(&value_path)?)
+    } else {
+        None
+    };
+
+    // Valid
+    // TODO: signing root
+    if let Some(value) = value {
+        check_serialization(&value, &serialized)?;
+
+        if let Some(ref meta) = meta {
+            check_tree_hash(&meta.root, value.tree_hash_root())?;
+        }
+    }
+    // Invalid
+    else {
+        if let Ok(decoded) = T::from_ssz_bytes(&serialized) {
+            return Err(Error::DidntFail(format!(
+                "Decoded invalid bytes into: {:?}",
+                decoded
+            )));
+        }
+    }
+
+    Ok(())
 }

tests/ef_tests/src/cases/ssz_static.rs

@@ -35,12 +35,12 @@ pub struct SszStaticSR<T> {
 
 // Trait alias for all deez bounds
 pub trait SszStaticType:
-    serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug
+    serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + Sync
 {
 }
 
 impl<T> SszStaticType for T where
-    T: serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug
+    T: serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + Sync
 {
 }
 
@@ -77,7 +77,7 @@ impl<T: SszStaticType + SignedRoot> LoadCase for SszStaticSR<T> {
     }
 }
 
-fn check_serialization<T: SszStaticType>(value: &T, serialized: &[u8]) -> Result<(), Error> {
+pub fn check_serialization<T: SszStaticType>(value: &T, serialized: &[u8]) -> Result<(), Error> {
     // Check serialization
     let serialized_result = value.as_ssz_bytes();
     compare_result::<Vec<u8>, Error>(&Ok(serialized_result), &Some(serialized.to_vec()))?;
@@ -89,7 +89,7 @@ fn check_serialization<T: SszStaticType>(value: &T, serialized: &[u8]) -> Result
     Ok(())
 }
 
-fn check_tree_hash(expected_str: &str, actual_root: Vec<u8>) -> Result<(), Error> {
+pub fn check_tree_hash(expected_str: &str, actual_root: Vec<u8>) -> Result<(), Error> {
     let expected_root = hex::decode(&expected_str[2..])
         .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
     let expected_root = Hash256::from_slice(&expected_root);

tests/ef_tests/src/handler.rs

@@ -1,6 +1,6 @@
 use crate::cases::{self, Case, Cases, EpochTransition, LoadCase, Operation};
+use crate::type_name;
 use crate::type_name::TypeName;
-use crate::EfTest;
 use std::fs;
 use std::marker::PhantomData;
 use std::path::PathBuf;
@@ -256,3 +256,33 @@ impl<E: EthSpec + TypeName, O: Operation<E>> Handler for OperationsHandler<E, O>
         O::handler_name()
     }
 }
+
+pub struct SszGenericHandler<H>(PhantomData<H>);
+
+impl<H: TypeName> Handler for SszGenericHandler<H> {
+    type Case = cases::SszGeneric;
+
+    fn config_name() -> &'static str {
+        "general"
+    }
+
+    fn runner_name() -> &'static str {
+        "ssz_generic"
+    }
+
+    fn handler_name() -> String {
+        H::name().into()
+    }
+}
+
+// Supported SSZ generic handlers
+pub struct BasicVector;
+type_name!(BasicVector, "basic_vector");
+pub struct Bitlist;
+type_name!(Bitlist, "bitlist");
+pub struct Bitvector;
+type_name!(Bitvector, "bitvector");
+pub struct Boolean;
+type_name!(Boolean, "boolean");
+pub struct Uints;
+type_name!(Uints, "uints");

tests/ef_tests/src/lib.rs

@@ -17,10 +17,3 @@ mod handler;
 mod results;
 mod type_name;
 mod yaml_decode;
-
-/// Defined where an object can return the results of some test(s) adhering to the Ethereum
-/// Foundation testing format.
-pub trait EfTest {
-    /// Returns the results of executing one or more tests.
-    fn test_results(&self) -> Vec<CaseResult>;
-}

tests/ef_tests/tests/tests.rs

@@ -1,5 +1,4 @@
 use ef_tests::*;
-use rayon::prelude::*;
 use types::{
     Attestation, AttestationData, AttestationDataAndCustodyBit, AttesterSlashing, BeaconBlock,
     BeaconBlockBody, BeaconBlockHeader, BeaconState, Checkpoint, CompactCommittee, Crosslink,
@@ -7,29 +6,15 @@ use types::{
     MinimalEthSpec, PendingAttestation, ProposerSlashing, Transfer, Validator, VoluntaryExit,
 };
 
-/*
 #[test]
-#[cfg(feature = "fake_crypto")]
 fn ssz_generic() {
-    yaml_files_in_test_dir(&Path::new("ssz_generic"))
+    SszGenericHandler::<BasicVector>::run();
-        .into_par_iter()
+    SszGenericHandler::<Bitlist>::run();
-        .for_each(|file| {
+    SszGenericHandler::<Bitvector>::run();
-            Doc::assert_tests_pass(file);
+    SszGenericHandler::<Boolean>::run();
-        });
+    SszGenericHandler::<Uints>::run();
 }
 
-
-#[test]
-#[cfg(feature = "fake_crypto")]
-fn ssz_static() {
-    yaml_files_in_test_dir(&Path::new("ssz_static"))
-        .into_par_iter()
-        .for_each(|file| {
-            Doc::assert_tests_pass(file);
-        });
-}
-*/
-
 #[test]
 fn shuffling() {
     ShufflingHandler::<MinimalEthSpec>::run();