Merging interop into api-alignment; fixing conflicts.

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "tests/ef_tests/eth2.0-spec-tests"]
-	path = tests/ef_tests/eth2.0-spec-tests
-	url = https://github.com/ethereum/eth2.0-spec-tests

Makefile

@@ -0,0 +1,31 @@
+TESTS_TAG := v0.8.3
+TESTS = general minimal mainnet
+
+TESTS_BASE_DIR := ./tests/ef_tests
+REPO_NAME := eth2.0-spec-tests
+OUTPUT_DIR := $(TESTS_BASE_DIR)/$(REPO_NAME)
+
+BASE_URL := https://github.com/ethereum/$(REPO_NAME)/releases/download/$(SPEC_VERSION)
+
+release:
+	cargo build --all --release
+
+clean_ef_tests:
+	rm -r $(OUTPUT_DIR)
+
+ef_tests: download_tests extract_tests
+	mkdir $(OUTPUT_DIR)
+	for test in $(TESTS); do \
+		tar -C $(OUTPUT_DIR) -xvf $(TESTS_BASE_DIR)/$$test.tar ;\
+		rm $(TESTS_BASE_DIR)/$$test.tar ;\
+	done
+
+extract_tests:
+	for test in $(TESTS); do \
+		gzip -df $(TESTS_BASE_DIR)/$$test.tar.gz ;\
+	done
+
+download_tests:
+	for test in $(TESTS); do \
+		wget -P $(TESTS_BASE_DIR) $(BASE_URL)/$$test.tar.gz; \
+	done

account_manager/src/main.rs

@@ -125,9 +125,13 @@ fn main() {
                 }
             }
         }
-        _ => panic!(
-            "The account manager must be run with a subcommand. See help for more information."
-        ),
+        _ => {
+            crit!(
+                log,
+                "The account manager must be run with a subcommand. See help for more information."
+            );
+            return;
+        }
     }
 }
 

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -178,7 +178,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             genesis_state.genesis_time,
             Duration::from_millis(spec.milliseconds_per_slot),
         )
-        .ok_or_else(|| Error::SlotClockDidNotStart)?;
+        .map_err(|_| Error::SlotClockDidNotStart)?;
 
         info!(log, "Beacon chain initialized from genesis";
               "validator_count" => genesis_state.validators.len(),
@@ -220,7 +220,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             state.genesis_time,
             Duration::from_millis(spec.milliseconds_per_slot),
         )
-        .ok_or_else(|| Error::SlotClockDidNotStart)?;
+        .map_err(|_| Error::SlotClockDidNotStart)?;
 
         let last_finalized_root = p.canonical_head.beacon_state.finalized_checkpoint.root;
         let last_finalized_block = &p.canonical_head.beacon_block;
@@ -459,6 +459,15 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         None
     }
 
+    /// Returns the block canonical root of the current canonical chain at a given slot.
+    ///
+    /// Returns None if a block doesn't exist at the slot.
+    pub fn root_at_slot(&self, target_slot: Slot) -> Option<Hash256> {
+        self.rev_iter_block_roots()
+            .find(|(_root, slot)| *slot == target_slot)
+            .map(|(root, _slot)| root)
+    }
+
     /// Reads the slot clock (see `self.read_slot_clock()` and returns the number of slots since
     /// genesis.
     pub fn slots_since_genesis(&self) -> Option<SlotHeight> {
@@ -1017,7 +1026,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         };
 
         // Load the parent blocks state from the database, returning an error if it is not found.
-        // It is an error because if know the parent block we should also know the parent state.
+        // It is an error because if we know the parent block we should also know the parent state.
         let parent_state_root = parent_block.state_root;
         let parent_state = self
             .store

beacon_node/client/src/lib.rs

@@ -16,7 +16,7 @@ use slog::{crit, error, info, o};
 use slot_clock::SlotClock;
 use std::marker::PhantomData;
 use std::sync::Arc;
-use std::time::{Duration, Instant};
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
 use tokio::runtime::TaskExecutor;
 use tokio::timer::Interval;
 use types::EthSpec;
@@ -177,8 +177,18 @@ where
                 .map_err(error::Error::from)?,
         );
 
-        if beacon_chain.slot().is_err() {
-            panic!("Cannot start client before genesis!")
+        let since_epoch = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .map_err(|e| format!("Unable to read system time: {}", e))?;
+        let since_genesis = Duration::from_secs(beacon_chain.head().beacon_state.genesis_time);
+
+        if since_genesis > since_epoch {
+            info!(
+                log,
+                "Starting node prior to genesis";
+                "now" => since_epoch.as_secs(),
+                "genesis_seconds" => since_genesis.as_secs(),
+            );
         }
 
         let network_config = &client_config.network;

beacon_node/client/src/notifier.rs

@@ -34,10 +34,10 @@ pub fn run<T: BeaconChainTypes>(client: &Client<T>, executor: TaskExecutor, exit
         // Panics if libp2p is poisoned.
         let connected_peer_count = libp2p.lock().swarm.connected_peers();
 
-        debug!(log, "Libp2p connected peer status"; "peer_count" => connected_peer_count);
+        debug!(log, "Connected peer status"; "peer_count" => connected_peer_count);
 
         if connected_peer_count <= WARN_PEER_COUNT {
-            warn!(log, "Low libp2p peer count"; "peer_count" => connected_peer_count);
+            warn!(log, "Low peer count"; "peer_count" => connected_peer_count);
         }
 
         Ok(())

beacon_node/eth2-libp2p/Cargo.toml

@@ -7,8 +7,8 @@ edition = "2018"
 [dependencies]
 clap = "2.32.0"
 #SigP repository 
-libp2p =  { git = "https://github.com/SigP/rust-libp2p", rev = "61036890d574f5b46573952b20def2baafd6a6e9" }
-enr =  { git = "https://github.com/SigP/rust-libp2p/", rev = "61036890d574f5b46573952b20def2baafd6a6e9", features = ["serde"] }
+libp2p =  { git = "https://github.com/SigP/rust-libp2p", rev = "76f7475e4b7063e663ad03c7524cf091f9961968" }
+enr =  { git = "https://github.com/SigP/rust-libp2p/", rev = "76f7475e4b7063e663ad03c7524cf091f9961968", features = ["serde"] }
 types = { path =  "../../eth2/types" }
 serde = "1.0"
 serde_derive = "1.0"

beacon_node/eth2-libp2p/src/behaviour.rs

@@ -15,7 +15,7 @@ use libp2p::{
     tokio_io::{AsyncRead, AsyncWrite},
     NetworkBehaviour, PeerId,
 };
-use slog::{debug, o, trace};
+use slog::{debug, o};
 use std::num::NonZeroU32;
 use std::time::Duration;
 
@@ -90,13 +90,15 @@ impl<TSubstream: AsyncRead + AsyncWrite> NetworkBehaviourEventProcess<GossipsubE
 {
     fn inject_event(&mut self, event: GossipsubEvent) {
         match event {
-            GossipsubEvent::Message(gs_msg) => {
-                trace!(self.log, "Received GossipEvent");
-
+            GossipsubEvent::Message(propagation_source, gs_msg) => {
+                let id = gs_msg.id();
                 let msg = PubsubMessage::from_topics(&gs_msg.topics, gs_msg.data);
 
+                // Note: We are keeping track here of the peer that sent us the message, not the
+                // peer that originally published the message.
                 self.events.push(BehaviourEvent::GossipMessage {
-                    source: gs_msg.source,
+                    id,
+                    source: propagation_source,
                     topics: gs_msg.topics,
                     message: msg,
                 });
@@ -199,6 +201,13 @@ impl<TSubstream: AsyncRead + AsyncWrite> Behaviour<TSubstream> {
         }
     }
 
+    /// Forwards a message that is waiting in gossipsub's mcache. Messages are only propagated
+    /// once validated by the beacon chain.
+    pub fn propagate_message(&mut self, propagation_source: &PeerId, message_id: String) {
+        self.gossipsub
+            .propagate_message(&message_id, propagation_source);
+    }
+
     /* Eth2 RPC behaviour functions */
 
     /// Sends an RPC Request/Response via the RPC protocol.
@@ -214,12 +223,21 @@ impl<TSubstream: AsyncRead + AsyncWrite> Behaviour<TSubstream> {
 
 /// The types of events than can be obtained from polling the behaviour.
 pub enum BehaviourEvent {
+    /// A received RPC event and the peer that it was received from.
     RPC(PeerId, RPCEvent),
+    /// We have completed an initial connection to a new peer.
     PeerDialed(PeerId),
+    /// A peer has disconnected.
     PeerDisconnected(PeerId),
+    /// A gossipsub message has been received.
     GossipMessage {
+        /// The gossipsub message id. Used when propagating blocks after validation.
+        id: String,
+        /// The peer from which we received this message, not the peer that published it.
         source: PeerId,
+        /// The topics that this message was sent on.
         topics: Vec<TopicHash>,
+        /// The message itself.
         message: PubsubMessage,
     },
 }

beacon_node/eth2-libp2p/src/config.rs

@@ -74,7 +74,8 @@ impl Default for Config {
             // parameter.
             gs_config: GossipsubConfigBuilder::new()
                 .max_transmit_size(1_048_576)
-                .heartbeat_interval(Duration::from_secs(20))
+                .heartbeat_interval(Duration::from_secs(20)) // TODO: Reduce for mainnet
+                .propagate_messages(false) // require validation before propagation
                 .build(),
             boot_nodes: vec![],
             libp2p_nodes: vec![],

beacon_node/eth2-libp2p/src/discovery.rs

@@ -114,7 +114,7 @@ impl<TSubstream> Discovery<TSubstream> {
         self.find_peers();
     }
 
-    /// Add an Enr to the routing table of the discovery mechanism.
+    /// Add an ENR to the routing table of the discovery mechanism.
     pub fn add_enr(&mut self, enr: Enr) {
         self.discovery.add_enr(enr);
     }
@@ -169,6 +169,7 @@ where
 
     fn inject_connected(&mut self, peer_id: PeerId, _endpoint: ConnectedPoint) {
         self.connected_peers.insert(peer_id);
+        // TODO: Drop peers if over max_peer limit
 
         metrics::inc_counter(&metrics::PEER_CONNECT_EVENT_COUNT);
         metrics::set_gauge(&metrics::PEERS_CONNECTED, self.connected_peers() as i64);

beacon_node/eth2-libp2p/src/rpc/codec/base.rs

@@ -101,13 +101,15 @@ where
     type Error = <TCodec as Decoder>::Error;
 
     fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
+        // if we have only received the response code, wait for more bytes
+        if src.len() == 1 {
+            return Ok(None);
+        }
+        // using the response code determine which kind of payload needs to be decoded.
         let response_code = {
             if let Some(resp_code) = self.response_code {
                 resp_code
             } else {
-                // buffer should not be empty
-                debug_assert!(!src.is_empty());
-
                 let resp_byte = src.split_to(1);
                 let mut resp_code_byte = [0; 1];
                 resp_code_byte.copy_from_slice(&resp_byte);

beacon_node/eth2-libp2p/src/rpc/codec/ssz.rs

@@ -4,7 +4,7 @@ use crate::rpc::{
     protocol::{ProtocolId, RPCError},
 };
 use crate::rpc::{ErrorMessage, RPCErrorResponse, RPCRequest, RPCResponse};
-use bytes::{Bytes, BytesMut};
+use bytes::{BufMut, Bytes, BytesMut};
 use ssz::{Decode, Encode};
 use tokio::codec::{Decoder, Encoder};
 use unsigned_varint::codec::UviBytes;
@@ -56,6 +56,10 @@ impl Encoder for SSZInboundCodec {
                 .inner
                 .encode(Bytes::from(bytes), dst)
                 .map_err(RPCError::from);
+        } else {
+            // payload is empty, add a 0-byte length prefix
+            dst.reserve(1);
+            dst.put_u8(0);
         }
         Ok(())
     }
@@ -152,45 +156,49 @@ impl Decoder for SSZOutboundCodec {
     type Error = RPCError;
 
     fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
-        match self.inner.decode(src).map_err(RPCError::from) {
-            Ok(Some(packet)) => match self.protocol.message_name.as_str() {
+        if src.len() == 1 && src[0] == 0_u8 {
+            // the object is empty. We return the empty object if this is the case
+            match self.protocol.message_name.as_str() {
                 "hello" => match self.protocol.version.as_str() {
-                    "1" => Ok(Some(RPCResponse::Hello(HelloMessage::from_ssz_bytes(
-                        &packet,
-                    )?))),
+                    "1" => Err(RPCError::Custom(
+                        "Hello stream terminated unexpectedly".into(),
+                    )), // cannot have an empty HELLO message. The stream has terminated unexpectedly
                     _ => unreachable!("Cannot negotiate an unknown version"),
                 },
                 "goodbye" => Err(RPCError::InvalidProtocol("GOODBYE doesn't have a response")),
                 "beacon_blocks" => match self.protocol.version.as_str() {
-                    "1" => Ok(Some(RPCResponse::BeaconBlocks(packet.to_vec()))),
+                    "1" => Ok(Some(RPCResponse::BeaconBlocks(Vec::new()))),
                     _ => unreachable!("Cannot negotiate an unknown version"),
                 },
                 "recent_beacon_blocks" => match self.protocol.version.as_str() {
-                    "1" => Ok(Some(RPCResponse::RecentBeaconBlocks(packet.to_vec()))),
+                    "1" => Ok(Some(RPCResponse::RecentBeaconBlocks(Vec::new()))),
                     _ => unreachable!("Cannot negotiate an unknown version"),
                 },
                 _ => unreachable!("Cannot negotiate an unknown protocol"),
-            },
-            Ok(None) => {
-                // the object sent could be a empty. We return the empty object if this is the case
-                match self.protocol.message_name.as_str() {
+            }
+        } else {
+            match self.inner.decode(src).map_err(RPCError::from) {
+                Ok(Some(packet)) => match self.protocol.message_name.as_str() {
                     "hello" => match self.protocol.version.as_str() {
-                        "1" => Ok(None), // cannot have an empty HELLO message. The stream has terminated unexpectedly
+                        "1" => Ok(Some(RPCResponse::Hello(HelloMessage::from_ssz_bytes(
+                            &packet,
+                        )?))),
                         _ => unreachable!("Cannot negotiate an unknown version"),
                     },
                     "goodbye" => Err(RPCError::InvalidProtocol("GOODBYE doesn't have a response")),
                     "beacon_blocks" => match self.protocol.version.as_str() {
-                        "1" => Ok(Some(RPCResponse::BeaconBlocks(Vec::new()))),
+                        "1" => Ok(Some(RPCResponse::BeaconBlocks(packet.to_vec()))),
                         _ => unreachable!("Cannot negotiate an unknown version"),
                     },
                     "recent_beacon_blocks" => match self.protocol.version.as_str() {
-                        "1" => Ok(Some(RPCResponse::RecentBeaconBlocks(Vec::new()))),
+                        "1" => Ok(Some(RPCResponse::RecentBeaconBlocks(packet.to_vec()))),
                         _ => unreachable!("Cannot negotiate an unknown version"),
                     },
                     _ => unreachable!("Cannot negotiate an unknown protocol"),
-                }
+                },
+                Ok(None) => Ok(None), // waiting for more bytes
+                Err(e) => Err(e),
             }
-            Err(e) => Err(e),
         }
     }
 }

beacon_node/eth2-libp2p/src/rpc/methods.rs

@@ -1,6 +1,5 @@
 //!Available RPC methods types and ids.
 
-use ssz::{impl_decode_via_from, impl_encode_via_from};
 use ssz_derive::{Decode, Encode};
 use types::{Epoch, Hash256, Slot};
 
@@ -66,8 +65,38 @@ impl Into<u64> for GoodbyeReason {
     }
 }
 
-impl_encode_via_from!(GoodbyeReason, u64);
-impl_decode_via_from!(GoodbyeReason, u64);
+impl ssz::Encode for GoodbyeReason {
+    fn is_ssz_fixed_len() -> bool {
+        <u64 as ssz::Encode>::is_ssz_fixed_len()
+    }
+
+    fn ssz_fixed_len() -> usize {
+        <u64 as ssz::Encode>::ssz_fixed_len()
+    }
+
+    fn ssz_bytes_len(&self) -> usize {
+        0_u64.ssz_bytes_len()
+    }
+
+    fn ssz_append(&self, buf: &mut Vec<u8>) {
+        let conv: u64 = self.clone().into();
+        conv.ssz_append(buf)
+    }
+}
+
+impl ssz::Decode for GoodbyeReason {
+    fn is_ssz_fixed_len() -> bool {
+        <u64 as ssz::Decode>::is_ssz_fixed_len()
+    }
+
+    fn ssz_fixed_len() -> usize {
+        <u64 as ssz::Decode>::ssz_fixed_len()
+    }
+
+    fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, ssz::DecodeError> {
+        u64::from_ssz_bytes(bytes).and_then(|n| Ok(n.into()))
+    }
+}
 
 /// Request a number of beacon block roots from a peer.
 #[derive(Encode, Decode, Clone, Debug, PartialEq)]
@@ -157,3 +186,53 @@ impl ErrorMessage {
         String::from_utf8(self.error_message.clone()).unwrap_or_else(|_| "".into())
     }
 }
+
+impl std::fmt::Display for HelloMessage {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "Hello Message: Fork Version: {:?}, Finalized Root: {}, Finalized Epoch: {}, Head Root: {}, Head Slot: {}", self.fork_version, self.finalized_root, self.finalized_epoch, self.head_root, self.head_slot)
+    }
+}
+
+impl std::fmt::Display for RPCResponse {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            RPCResponse::Hello(hello) => write!(f, "{}", hello),
+            RPCResponse::BeaconBlocks(data) => write!(f, "<BeaconBlocks>, len: {}", data.len()),
+            RPCResponse::RecentBeaconBlocks(data) => {
+                write!(f, "<RecentBeaconBlocks>, len: {}", data.len())
+            }
+        }
+    }
+}
+
+impl std::fmt::Display for RPCErrorResponse {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            RPCErrorResponse::Success(res) => write!(f, "{}", res),
+            RPCErrorResponse::InvalidRequest(err) => write!(f, "Invalid Request: {:?}", err),
+            RPCErrorResponse::ServerError(err) => write!(f, "Server Error: {:?}", err),
+            RPCErrorResponse::Unknown(err) => write!(f, "Unknown Error: {:?}", err),
+        }
+    }
+}
+
+impl std::fmt::Display for GoodbyeReason {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            GoodbyeReason::ClientShutdown => write!(f, "Client Shutdown"),
+            GoodbyeReason::IrrelevantNetwork => write!(f, "Irrelevant Network"),
+            GoodbyeReason::Fault => write!(f, "Fault"),
+            GoodbyeReason::Unknown => write!(f, "Unknown Reason"),
+        }
+    }
+}
+
+impl std::fmt::Display for BeaconBlocksRequest {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "Head Block Root: {},  Start Slot: {}, Count: {}, Step: {}",
+            self.head_block_root, self.start_slot, self.count, self.step
+        )
+    }
+}

beacon_node/eth2-libp2p/src/rpc/mod.rs

@@ -47,6 +47,16 @@ impl RPCEvent {
     }
 }
 
+impl std::fmt::Display for RPCEvent {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            RPCEvent::Request(id, req) => write!(f, "RPC Request(Id: {}, {})", id, req),
+            RPCEvent::Response(id, res) => write!(f, "RPC Response(Id: {}, {})", id, res),
+            RPCEvent::Error(id, err) => write!(f, "RPC Request(Id: {}, Error: {:?})", id, err),
+        }
+    }
+}
+
 /// Implements the libp2p `NetworkBehaviour` trait and therefore manages network-level
 /// logic.
 pub struct RPC<TSubstream> {

beacon_node/eth2-libp2p/src/rpc/protocol.rs

@@ -288,3 +288,14 @@ impl std::error::Error for RPCError {
         }
     }
 }
+
+impl std::fmt::Display for RPCRequest {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            RPCRequest::Hello(hello) => write!(f, "Hello Message: {}", hello),
+            RPCRequest::Goodbye(reason) => write!(f, "Goodbye: {}", reason),
+            RPCRequest::BeaconBlocks(req) => write!(f, "Beacon Blocks: {}", req),
+            RPCRequest::RecentBeaconBlocks(req) => write!(f, "Recent Beacon Blocks: {:?}", req),
+        }
+    }
+}

beacon_node/eth2-libp2p/src/service.rs

@@ -79,15 +79,32 @@ impl Service {
             }
         };
 
-        // attempt to connect to user-input libp2p nodes
-        for multiaddr in config.libp2p_nodes {
+        // helper closure for dialing peers
+        let mut dial_addr = |multiaddr: Multiaddr| {
             match Swarm::dial_addr(&mut swarm, multiaddr.clone()) {
                 Ok(()) => debug!(log, "Dialing libp2p peer"; "address" => format!("{}", multiaddr)),
                 Err(err) => debug!(
                     log,
-                    "Could not connect to peer"; "address" => format!("{}", multiaddr), "Error" => format!("{:?}", err)
+                    "Could not connect to peer"; "address" => format!("{}", multiaddr), "error" => format!("{:?}", err)
                 ),
             };
+        };
+
+        // attempt to connect to user-input libp2p nodes
+        for multiaddr in config.libp2p_nodes {
+            dial_addr(multiaddr);
+        }
+
+        // attempt to connect to any specified boot-nodes
+        for bootnode_enr in config.boot_nodes {
+            for multiaddr in bootnode_enr.multiaddr() {
+                // ignore udp multiaddr if it exists
+                let components = multiaddr.iter().collect::<Vec<_>>();
+                if let Protocol::Udp(_) = components[1] {
+                    continue;
+                }
+                dial_addr(multiaddr);
+            }
         }
 
         // subscribe to default gossipsub topics
@@ -145,16 +162,16 @@ impl Stream for Service {
     fn poll(&mut self) -> Poll<Option<Self::Item>, Self::Error> {
         loop {
             match self.swarm.poll() {
-                //Behaviour events
                 Ok(Async::Ready(Some(event))) => match event {
-                    // TODO: Stub here for debugging
                     BehaviourEvent::GossipMessage {
+                        id,
                         source,
                         topics,
                         message,
                     } => {
                         trace!(self.log, "Gossipsub message received"; "service" => "Swarm");
                         return Ok(Async::Ready(Some(Libp2pEvent::PubsubMessage {
+                            id,
                             source,
                             topics,
                             message,
@@ -222,6 +239,7 @@ pub enum Libp2pEvent {
     PeerDisconnected(PeerId),
     /// Received pubsub message.
     PubsubMessage {
+        id: String,
         source: PeerId,
         topics: Vec<TopicHash>,
         message: PubsubMessage,

beacon_node/network/Cargo.toml

@@ -19,3 +19,4 @@ futures = "0.1.25"
 error-chain = "0.12.0"
 tokio = "0.1.16"
 parking_lot = "0.9.0"
+smallvec = "0.6.10"

beacon_node/network/src/message_handler.rs

@@ -1,6 +1,6 @@
 use crate::error;
 use crate::service::NetworkMessage;
-use crate::sync::SimpleSync;
+use crate::sync::MessageProcessor;
 use beacon_chain::{BeaconChain, BeaconChainTypes};
 use eth2_libp2p::{
     behaviour::PubsubMessage,
@@ -9,18 +9,22 @@ use eth2_libp2p::{
 };
 use futures::future::Future;
 use futures::stream::Stream;
-use slog::{debug, trace, warn};
+use slog::{debug, o, trace, warn};
 use ssz::{Decode, DecodeError};
 use std::sync::Arc;
 use tokio::sync::mpsc;
 use types::{Attestation, AttesterSlashing, BeaconBlock, ProposerSlashing, VoluntaryExit};
 
-/// Handles messages received from the network and client and organises syncing.
+/// Handles messages received from the network and client and organises syncing. This
+/// functionality of this struct is to validate an decode messages from the network before
+/// passing them to the internal message processor. The message processor spawns a syncing thread
+/// which manages which blocks need to be requested and processed.
 pub struct MessageHandler<T: BeaconChainTypes> {
-    /// Currently loaded and initialised beacon chain.
-    _chain: Arc<BeaconChain<T>>,
-    /// The syncing framework.
-    sync: SimpleSync<T>,
+    /// A channel to the network service to allow for gossip propagation.
+    network_send: mpsc::UnboundedSender<NetworkMessage>,
+    /// Processes validated and decoded messages from the network. Has direct access to the
+    /// sync manager.
+    message_processor: MessageProcessor<T>,
     /// The `MessageHandler` logger.
     log: slog::Logger,
 }
@@ -34,8 +38,9 @@ pub enum HandlerMessage {
     PeerDisconnected(PeerId),
     /// An RPC response/request has been received.
     RPC(PeerId, RPCEvent),
-    /// A gossip message has been received.
-    PubsubMessage(PeerId, PubsubMessage),
+    /// A gossip message has been received. The fields are: message id, the peer that sent us this
+    /// message and the message itself.
+    PubsubMessage(String, PeerId, PubsubMessage),
 }
 
 impl<T: BeaconChainTypes + 'static> MessageHandler<T> {
@@ -46,17 +51,20 @@ impl<T: BeaconChainTypes + 'static> MessageHandler<T> {
         executor: &tokio::runtime::TaskExecutor,
         log: slog::Logger,
     ) -> error::Result<mpsc::UnboundedSender<HandlerMessage>> {
-        trace!(log, "Service starting");
+        let message_handler_log = log.new(o!("Service"=> "Message Handler"));
+        trace!(message_handler_log, "Service starting");
 
         let (handler_send, handler_recv) = mpsc::unbounded_channel();
-        // Initialise sync and begin processing in thread
-        let sync = SimpleSync::new(beacon_chain.clone(), network_send, &log);
+
+        // Initialise a message instance, which itself spawns the syncing thread.
+        let message_processor =
+            MessageProcessor::new(executor, beacon_chain, network_send.clone(), &log);
 
         // generate the Message handler
         let mut handler = MessageHandler {
-            _chain: beacon_chain.clone(),
-            sync,
-            log: log.clone(),
+            network_send,
+            message_processor,
+            log: message_handler_log,
         };
 
         // spawn handler task and move the message handler instance into the spawned thread
@@ -65,7 +73,11 @@ impl<T: BeaconChainTypes + 'static> MessageHandler<T> {
                 .for_each(move |msg| Ok(handler.handle_message(msg)))
                 .map_err(move |_| {
                     debug!(log, "Network message handler terminated.");
-                }),
+                }), /*
+                    .then(move |_| {
+                        debug!(log.clone(), "Message handler shutdown");
+                    }),
+                    */
         );
 
         Ok(handler_send)
@@ -76,19 +88,19 @@ impl<T: BeaconChainTypes + 'static> MessageHandler<T> {
         match message {
             // we have initiated a connection to a peer
             HandlerMessage::PeerDialed(peer_id) => {
-                self.sync.on_connect(peer_id);
+                self.message_processor.on_connect(peer_id);
             }
             // A peer has disconnected
             HandlerMessage::PeerDisconnected(peer_id) => {
-                self.sync.on_disconnect(peer_id);
+                self.message_processor.on_disconnect(peer_id);
             }
             // An RPC message request/response has been received
             HandlerMessage::RPC(peer_id, rpc_event) => {
                 self.handle_rpc_message(peer_id, rpc_event);
             }
             // An RPC message request/response has been received
-            HandlerMessage::PubsubMessage(peer_id, gossip) => {
-                self.handle_gossip(peer_id, gossip);
+            HandlerMessage::PubsubMessage(id, peer_id, gossip) => {
+                self.handle_gossip(id, peer_id, gossip);
             }
         }
     }
@@ -108,7 +120,7 @@ impl<T: BeaconChainTypes + 'static> MessageHandler<T> {
     fn handle_rpc_request(&mut self, peer_id: PeerId, request_id: RequestId, request: RPCRequest) {
         match request {
             RPCRequest::Hello(hello_message) => {
-                self.sync
+                self.message_processor
                     .on_hello_request(peer_id, request_id, hello_message)
             }
             RPCRequest::Goodbye(goodbye_reason) => {
@@ -117,13 +129,13 @@ impl<T: BeaconChainTypes + 'static> MessageHandler<T> {
                     "peer" => format!("{:?}", peer_id),
                     "reason" => format!("{:?}", goodbye_reason),
                 );
-                self.sync.on_disconnect(peer_id);
+                self.message_processor.on_disconnect(peer_id);
             }
             RPCRequest::BeaconBlocks(request) => self
-                .sync
+                .message_processor
                 .on_beacon_blocks_request(peer_id, request_id, request),
             RPCRequest::RecentBeaconBlocks(request) => self
-                .sync
+                .message_processor
                 .on_recent_beacon_blocks_request(peer_id, request_id, request),
         }
     }
@@ -150,12 +162,13 @@ impl<T: BeaconChainTypes + 'static> MessageHandler<T> {
             RPCErrorResponse::Success(response) => {
                 match response {
                     RPCResponse::Hello(hello_message) => {
-                        self.sync.on_hello_response(peer_id, hello_message);
+                        self.message_processor
+                            .on_hello_response(peer_id, hello_message);
                     }
                     RPCResponse::BeaconBlocks(response) => {
                         match self.decode_beacon_blocks(&response) {
                             Ok(beacon_blocks) => {
-                                self.sync.on_beacon_blocks_response(
+                                self.message_processor.on_beacon_blocks_response(
                                     peer_id,
                                     request_id,
                                     beacon_blocks,
@@ -170,7 +183,7 @@ impl<T: BeaconChainTypes + 'static> MessageHandler<T> {
                     RPCResponse::RecentBeaconBlocks(response) => {
                         match self.decode_beacon_blocks(&response) {
                             Ok(beacon_blocks) => {
-                                self.sync.on_recent_beacon_blocks_response(
+                                self.message_processor.on_recent_beacon_blocks_response(
                                     peer_id,
                                     request_id,
                                     beacon_blocks,
@@ -194,24 +207,37 @@ impl<T: BeaconChainTypes + 'static> MessageHandler<T> {
     }
 
     /// Handle RPC messages
-    fn handle_gossip(&mut self, peer_id: PeerId, gossip_message: PubsubMessage) {
+    fn handle_gossip(&mut self, id: String, peer_id: PeerId, gossip_message: PubsubMessage) {
         match gossip_message {
             PubsubMessage::Block(message) => match self.decode_gossip_block(message) {
                 Ok(block) => {
-                    let _should_forward_on = self.sync.on_block_gossip(peer_id, block);
+                    let should_forward_on = self
+                        .message_processor
+                        .on_block_gossip(peer_id.clone(), block);
+                    // TODO: Apply more sophisticated validation and decoding logic
+                    if should_forward_on {
+                        self.propagate_message(id, peer_id.clone());
+                    }
                 }
                 Err(e) => {
                     debug!(self.log, "Invalid gossiped beacon block"; "peer_id" => format!("{}", peer_id), "Error" => format!("{:?}", e));
                 }
             },
             PubsubMessage::Attestation(message) => match self.decode_gossip_attestation(message) {
-                Ok(attestation) => self.sync.on_attestation_gossip(peer_id, attestation),
+                Ok(attestation) => {
+                    // TODO: Apply more sophisticated validation and decoding logic
+                    self.propagate_message(id, peer_id.clone());
+                    self.message_processor
+                        .on_attestation_gossip(peer_id, attestation);
+                }
                 Err(e) => {
                     debug!(self.log, "Invalid gossiped attestation"; "peer_id" => format!("{}", peer_id), "Error" => format!("{:?}", e));
                 }
             },
             PubsubMessage::VoluntaryExit(message) => match self.decode_gossip_exit(message) {
                 Ok(_exit) => {
+                    // TODO: Apply more sophisticated validation and decoding logic
+                    self.propagate_message(id, peer_id.clone());
                     // TODO: Handle exits
                     debug!(self.log, "Received a voluntary exit"; "peer_id" => format!("{}", peer_id) );
                 }
@@ -222,6 +248,8 @@ impl<T: BeaconChainTypes + 'static> MessageHandler<T> {
             PubsubMessage::ProposerSlashing(message) => {
                 match self.decode_gossip_proposer_slashing(message) {
                     Ok(_slashing) => {
+                        // TODO: Apply more sophisticated validation and decoding logic
+                        self.propagate_message(id, peer_id.clone());
                         // TODO: Handle proposer slashings
                         debug!(self.log, "Received a proposer slashing"; "peer_id" => format!("{}", peer_id) );
                     }
@@ -233,6 +261,8 @@ impl<T: BeaconChainTypes + 'static> MessageHandler<T> {
             PubsubMessage::AttesterSlashing(message) => {
                 match self.decode_gossip_attestation_slashing(message) {
                     Ok(_slashing) => {
+                        // TODO: Apply more sophisticated validation and decoding logic
+                        self.propagate_message(id, peer_id.clone());
                         // TODO: Handle attester slashings
                         debug!(self.log, "Received an attester slashing"; "peer_id" => format!("{}", peer_id) );
                     }
@@ -248,6 +278,21 @@ impl<T: BeaconChainTypes + 'static> MessageHandler<T> {
         }
     }
 
+    /// Informs the network service that the message should be forwarded to other peers.
+    fn propagate_message(&mut self, message_id: String, propagation_source: PeerId) {
+        self.network_send
+            .try_send(NetworkMessage::Propagate {
+                propagation_source,
+                message_id,
+            })
+            .unwrap_or_else(|_| {
+                warn!(
+                    self.log,
+                    "Could not send propagation request to the network service"
+                )
+            });
+    }
+
     /* Decoding of gossipsub objects from the network.
      *
      * The decoding is done in the message handler as it has access to to a `BeaconChain` and can

beacon_node/network/src/service.rs

@@ -34,13 +34,8 @@ impl<T: BeaconChainTypes + 'static> Service<T> {
         // build the network channel
         let (network_send, network_recv) = mpsc::unbounded_channel::<NetworkMessage>();
         // launch message handler thread
-        let message_handler_log = log.new(o!("Service" => "MessageHandler"));
-        let message_handler_send = MessageHandler::spawn(
-            beacon_chain,
-            network_send.clone(),
-            executor,
-            message_handler_log,
-        )?;
+        let message_handler_send =
+            MessageHandler::spawn(beacon_chain, network_send.clone(), executor, log.clone())?;
 
         let network_log = log.new(o!("Service" => "Network"));
         // launch libp2p service
@@ -159,12 +154,23 @@ fn network_service(
             // poll the network channel
             match network_recv.poll() {
                 Ok(Async::Ready(Some(message))) => match message {
-                    NetworkMessage::Send(peer_id, outgoing_message) => match outgoing_message {
-                        OutgoingMessage::RPC(rpc_event) => {
-                            trace!(log, "Sending RPC Event: {:?}", rpc_event);
-                            libp2p_service.lock().swarm.send_rpc(peer_id, rpc_event);
-                        }
-                    },
+                    NetworkMessage::RPC(peer_id, rpc_event) => {
+                        trace!(log, "{}", rpc_event);
+                        libp2p_service.lock().swarm.send_rpc(peer_id, rpc_event);
+                    }
+                    NetworkMessage::Propagate {
+                        propagation_source,
+                        message_id,
+                    } => {
+                        trace!(log, "Propagating gossipsub message";
+                        "propagation_peer" => format!("{:?}", propagation_source),
+                        "message_id" => format!("{}", message_id),
+                        );
+                        libp2p_service
+                            .lock()
+                            .swarm
+                            .propagate_message(&propagation_source, message_id);
+                    }
                     NetworkMessage::Publish { topics, message } => {
                         debug!(log, "Sending pubsub message"; "topics" => format!("{:?}",topics));
                         libp2p_service.lock().swarm.publish(&topics, message);
@@ -185,7 +191,7 @@ fn network_service(
             match libp2p_service.lock().poll() {
                 Ok(Async::Ready(Some(event))) => match event {
                     Libp2pEvent::RPC(peer_id, rpc_event) => {
-                        trace!(log, "RPC Event: RPC message received: {:?}", rpc_event);
+                        trace!(log, "{}", rpc_event);
                         message_handler_send
                             .try_send(HandlerMessage::RPC(peer_id, rpc_event))
                             .map_err(|_| "Failed to send RPC to handler")?;
@@ -203,13 +209,14 @@ fn network_service(
                             .map_err(|_| "Failed to send PeerDisconnected to handler")?;
                     }
                     Libp2pEvent::PubsubMessage {
-                        source, message, ..
+                        id,
+                        source,
+                        message,
+                        ..
                     } => {
-                        //TODO: Decide if we need to propagate the topic upwards. (Potentially for
-                        //attestations)
                         message_handler_send
-                            .try_send(HandlerMessage::PubsubMessage(source, message))
-                            .map_err(|_| " failed to send pubsub message to handler")?;
+                            .try_send(HandlerMessage::PubsubMessage(id, source, message))
+                            .map_err(|_| "Failed to send pubsub message to handler")?;
                     }
                 },
                 Ok(Async::Ready(None)) => unreachable!("Stream never ends"),
@@ -225,19 +232,16 @@ fn network_service(
 /// Types of messages that the network service can receive.
 #[derive(Debug)]
 pub enum NetworkMessage {
-    /// Send a message to libp2p service.
-    //TODO: Define typing for messages across the wire
-    Send(PeerId, OutgoingMessage),
-    /// Publish a message to pubsub mechanism.
+    /// Send an RPC message to the libp2p service.
+    RPC(PeerId, RPCEvent),
+    /// Publish a message to gossipsub.
     Publish {
         topics: Vec<Topic>,
         message: PubsubMessage,
     },
-}
-
-/// Type of outgoing messages that can be sent through the network service.
-#[derive(Debug)]
-pub enum OutgoingMessage {
-    /// Send an RPC request/response.
-    RPC(RPCEvent),
+    /// Propagate a received gossipsub message
+    Propagate {
+        propagation_source: PeerId,
+        message_id: String,
+    },
 }

beacon_node/network/src/sync/mod.rs

@@ -4,7 +4,7 @@ mod manager;
 /// Stores the various syncing methods for the beacon chain.
 mod simple_sync;
 
-pub use simple_sync::SimpleSync;
+pub use simple_sync::MessageProcessor;
 
 /// Currently implemented sync methods.
 pub enum SyncMethod {

beacon_node/network/src/sync/simple_sync.rs

@@ -1,23 +1,23 @@
-use super::manager::{ImportManager, ImportManagerOutcome};
-use crate::service::{NetworkMessage, OutgoingMessage};
+use super::manager::SyncMessage;
+use crate::service::NetworkMessage;
 use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome};
 use eth2_libp2p::rpc::methods::*;
 use eth2_libp2p::rpc::{RPCEvent, RPCRequest, RPCResponse, RequestId};
 use eth2_libp2p::PeerId;
 use slog::{debug, info, o, trace, warn};
 use ssz::Encode;
-use std::ops::Sub;
 use std::sync::Arc;
 use store::Store;
-use tokio::sync::mpsc;
+use tokio::sync::{mpsc, oneshot};
 use types::{Attestation, BeaconBlock, Epoch, EthSpec, Hash256, Slot};
 
+//TODO: Put a maximum limit on the number of block that can be requested.
+//TODO: Rate limit requests
+
 /// If a block is more than `FUTURE_SLOT_TOLERANCE` slots ahead of our slot clock, we drop it.
 /// Otherwise we queue it.
 pub(crate) const FUTURE_SLOT_TOLERANCE: u64 = 1;
 
-/// The number of slots behind our head that we still treat a peer as a fully synced peer.
-const FULL_PEER_TOLERANCE: u64 = 10;
 const SHOULD_FORWARD_GOSSIP_BLOCK: bool = true;
 const SHOULD_NOT_FORWARD_GOSSIP_BLOCK: bool = false;
 
@@ -49,45 +49,63 @@ impl<T: BeaconChainTypes> From<&Arc<BeaconChain<T>>> for PeerSyncInfo {
     }
 }
 
-/// The current syncing state.
-#[derive(PartialEq)]
-pub enum SyncState {
-    _Idle,
-    _Downloading,
-    _Stopped,
-}
-
-/// Simple Syncing protocol.
-pub struct SimpleSync<T: BeaconChainTypes> {
+/// Processes validated messages from the network. It relays necessary data to the syncing thread
+/// and processes blocks from the pubsub network.
+pub struct MessageProcessor<T: BeaconChainTypes> {
     /// A reference to the underlying beacon chain.
     chain: Arc<BeaconChain<T>>,
-    manager: ImportManager<T>,
+    /// A channel to the syncing thread.
+    sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+    /// A oneshot channel for destroying the sync thread.
+    _sync_exit: oneshot::Sender<()>,
+    /// A nextwork context to return and handle RPC requests.
     network: NetworkContext,
+    /// The `RPCHandler` logger.
     log: slog::Logger,
 }
 
-impl<T: BeaconChainTypes> SimpleSync<T> {
-    /// Instantiate a `SimpleSync` instance, with no peers and an empty queue.
+impl<T: BeaconChainTypes> MessageProcessor<T> {
+    /// Instantiate a `MessageProcessor` instance
     pub fn new(
+        executor: &tokio::runtime::TaskExecutor,
         beacon_chain: Arc<BeaconChain<T>>,
         network_send: mpsc::UnboundedSender<NetworkMessage>,
         log: &slog::Logger,
     ) -> Self {
         let sync_logger = log.new(o!("Service"=> "Sync"));
+        let sync_network_context = NetworkContext::new(network_send.clone(), sync_logger.clone());
 
-        SimpleSync {
-            chain: beacon_chain.clone(),
-            manager: ImportManager::new(beacon_chain, log),
+        // spawn the sync thread
+        let (sync_send, _sync_exit) = super::manager::spawn(
+            executor,
+            Arc::downgrade(&beacon_chain),
+            sync_network_context,
+            sync_logger,
+        );
+
+        MessageProcessor {
+            chain: beacon_chain,
+            sync_send,
+            _sync_exit,
             network: NetworkContext::new(network_send, log.clone()),
-            log: sync_logger,
+            log: log.clone(),
         }
     }
 
+    fn send_to_sync(&mut self, message: SyncMessage<T::EthSpec>) {
+        self.sync_send.try_send(message).unwrap_or_else(|_| {
+            warn!(
+                self.log,
+                "Could not send message to the sync service";
+            )
+        });
+    }
+
     /// Handle a peer disconnect.
     ///
     /// Removes the peer from the manager.
     pub fn on_disconnect(&mut self, peer_id: PeerId) {
-        self.manager.peer_disconnect(&peer_id);
+        self.send_to_sync(SyncMessage::Disconnect(peer_id));
     }
 
     /// Handle the connection of a new peer.
@@ -107,6 +125,7 @@ impl<T: BeaconChainTypes> SimpleSync<T> {
         request_id: RequestId,
         hello: HelloMessage,
     ) {
+        // ignore hello responses if we are shutting down
         trace!(self.log, "HelloRequest"; "peer" => format!("{:?}", peer_id));
 
         // Say hello back.
@@ -149,7 +168,7 @@ impl<T: BeaconChainTypes> SimpleSync<T> {
         } else if remote.finalized_epoch <= local.finalized_epoch
             && remote.finalized_root != Hash256::zero()
             && local.finalized_root != Hash256::zero()
-            && (self.root_at_slot(start_slot(remote.finalized_epoch))
+            && (self.chain.root_at_slot(start_slot(remote.finalized_epoch))
                 != Some(remote.finalized_root))
         {
             // The remotes finalized epoch is less than or greater than ours, but the block root is
@@ -189,18 +208,16 @@ impl<T: BeaconChainTypes> SimpleSync<T> {
             .exists::<BeaconBlock<T::EthSpec>>(&remote.head_root)
             .unwrap_or_else(|_| false)
         {
+            trace!(
+                self.log, "Peer with known chain found";
+                "peer" => format!("{:?}", peer_id),
+                "remote_head_slot" => remote.head_slot,
+                "remote_latest_finalized_epoch" => remote.finalized_epoch,
+            );
+
             // If the node's best-block is already known to us and they are close to our current
             // head, treat them as a fully sync'd peer.
-            if self.chain.best_slot().sub(remote.head_slot).as_u64() < FULL_PEER_TOLERANCE {
-                self.manager.add_full_peer(peer_id);
-                self.process_sync();
-            } else {
-                debug!(
-                    self.log,
-                    "Out of sync peer connected";
-                    "peer" => format!("{:?}", peer_id),
-                );
-            }
+            self.send_to_sync(SyncMessage::AddPeer(peer_id, remote));
         } else {
             // The remote node has an equal or great finalized epoch and we don't know it's head.
             //
@@ -212,87 +229,10 @@ impl<T: BeaconChainTypes> SimpleSync<T> {
                 "local_finalized_epoch" => local.finalized_epoch,
                 "remote_latest_finalized_epoch" => remote.finalized_epoch,
             );
-
-            self.manager.add_peer(peer_id, remote);
-            self.process_sync();
+            self.send_to_sync(SyncMessage::AddPeer(peer_id, remote));
         }
     }
 
-    fn process_sync(&mut self) {
-        loop {
-            match self.manager.poll() {
-                ImportManagerOutcome::Hello(peer_id) => {
-                    trace!(
-                        self.log,
-                        "RPC Request";
-                        "method" => "HELLO",
-                        "peer" => format!("{:?}", peer_id)
-                    );
-                    self.network.send_rpc_request(
-                        None,
-                        peer_id,
-                        RPCRequest::Hello(hello_message(&self.chain)),
-                    );
-                }
-                ImportManagerOutcome::RequestBlocks {
-                    peer_id,
-                    request_id,
-                    request,
-                } => {
-                    trace!(
-                        self.log,
-                        "RPC Request";
-                        "method" => "BeaconBlocks",
-                        "id" => request_id,
-                        "count" => request.count,
-                        "peer" => format!("{:?}", peer_id)
-                    );
-                    self.network.send_rpc_request(
-                        Some(request_id),
-                        peer_id.clone(),
-                        RPCRequest::BeaconBlocks(request),
-                    );
-                }
-                ImportManagerOutcome::RecentRequest(peer_id, req) => {
-                    trace!(
-                        self.log,
-                        "RPC Request";
-                        "method" => "RecentBeaconBlocks",
-                        "count" => req.block_roots.len(),
-                        "peer" => format!("{:?}", peer_id)
-                    );
-                    self.network.send_rpc_request(
-                        None,
-                        peer_id.clone(),
-                        RPCRequest::RecentBeaconBlocks(req),
-                    );
-                }
-                ImportManagerOutcome::DownvotePeer(peer_id) => {
-                    trace!(
-                        self.log,
-                        "Peer downvoted";
-                        "peer" => format!("{:?}", peer_id)
-                    );
-                    // TODO: Implement reputation
-                    self.network
-                        .disconnect(peer_id.clone(), GoodbyeReason::Fault);
-                }
-                ImportManagerOutcome::Idle => {
-                    // nothing to do
-                    return;
-                }
-            }
-        }
-    }
-
-    //TODO: Move to beacon chain
-    fn root_at_slot(&self, target_slot: Slot) -> Option<Hash256> {
-        self.chain
-            .rev_iter_block_roots()
-            .find(|(_root, slot)| *slot == target_slot)
-            .map(|(root, _slot)| root)
-    }
-
     /// Handle a `RecentBeaconBlocks` request from the peer.
     pub fn on_recent_beacon_blocks_request(
         &mut self,
@@ -321,7 +261,7 @@ impl<T: BeaconChainTypes> SimpleSync<T> {
 
         debug!(
             self.log,
-            "BlockBodiesRequest";
+            "RecentBeaconBlocksRequest";
             "peer" => format!("{:?}", peer_id),
             "requested" => request.block_roots.len(),
             "returned" => blocks.len(),
@@ -380,18 +320,16 @@ impl<T: BeaconChainTypes> SimpleSync<T> {
         blocks.reverse();
         blocks.dedup_by_key(|brs| brs.slot);
 
-        if blocks.len() as u64 != req.count {
-            debug!(
-                self.log,
-                "BeaconBlocksRequest response";
-                "peer" => format!("{:?}", peer_id),
-                "msg" => "Failed to return all requested hashes",
-                "start_slot" => req.start_slot,
-                "current_slot" => format!("{:?}", self.chain.slot()),
-                "requested" => req.count,
-                "returned" => blocks.len(),
-            );
-        }
+        debug!(
+            self.log,
+            "BeaconBlocksRequest response";
+            "peer" => format!("{:?}", peer_id),
+            "msg" => "Failed to return all requested hashes",
+            "start_slot" => req.start_slot,
+            "current_slot" => self.chain.slot().unwrap_or_else(|_| Slot::from(0_u64)).as_u64(),
+            "requested" => req.count,
+            "returned" => blocks.len(),
+        );
 
         self.network.send_rpc_response(
             peer_id,
@@ -414,10 +352,11 @@ impl<T: BeaconChainTypes> SimpleSync<T> {
             "count" => beacon_blocks.len(),
         );
 
-        self.manager
-            .beacon_blocks_response(peer_id, request_id, beacon_blocks);
-
-        self.process_sync();
+        self.send_to_sync(SyncMessage::BeaconBlocksResponse {
+            peer_id,
+            request_id,
+            beacon_blocks,
+        });
     }
 
     /// Handle a `RecentBeaconBlocks` response from the peer.
@@ -429,15 +368,16 @@ impl<T: BeaconChainTypes> SimpleSync<T> {
     ) {
         debug!(
             self.log,
-            "BeaconBlocksResponse";
+            "RecentBeaconBlocksResponse";
             "peer" => format!("{:?}", peer_id),
             "count" => beacon_blocks.len(),
         );
 
-        self.manager
-            .recent_blocks_response(peer_id, request_id, beacon_blocks);
-
-        self.process_sync();
+        self.send_to_sync(SyncMessage::RecentBeaconBlocksResponse {
+            peer_id,
+            request_id,
+            beacon_blocks,
+        });
     }
 
     /// Process a gossip message declaring a new block.
@@ -455,9 +395,9 @@ impl<T: BeaconChainTypes> SimpleSync<T> {
                 }
                 BlockProcessingOutcome::ParentUnknown { parent: _ } => {
                     // Inform the sync manager to find parents for this block
-                    trace!(self.log, "Unknown parent gossip";
+                    trace!(self.log, "Block with unknown parent received";
                             "peer_id" => format!("{:?}",peer_id));
-                    self.manager.add_unknown_block(block.clone(), peer_id);
+                    self.send_to_sync(SyncMessage::UnknownBlock(peer_id, block.clone()));
                     SHOULD_FORWARD_GOSSIP_BLOCK
                 }
                 BlockProcessingOutcome::FutureSlot {
@@ -468,7 +408,7 @@ impl<T: BeaconChainTypes> SimpleSync<T> {
                     SHOULD_FORWARD_GOSSIP_BLOCK
                 }
                 BlockProcessingOutcome::BlockIsAlreadyKnown => SHOULD_FORWARD_GOSSIP_BLOCK,
-                _ => SHOULD_NOT_FORWARD_GOSSIP_BLOCK,
+                _ => SHOULD_NOT_FORWARD_GOSSIP_BLOCK, //TODO: Decide if we want to forward these
             }
         } else {
             SHOULD_NOT_FORWARD_GOSSIP_BLOCK
@@ -491,15 +431,10 @@ impl<T: BeaconChainTypes> SimpleSync<T> {
             }
         }
     }
-
-    /// Generates our current state in the form of a HELLO RPC message.
-    pub fn generate_hello(&self) -> HelloMessage {
-        hello_message(&self.chain)
-    }
 }
 
 /// Build a `HelloMessage` representing the state of the given `beacon_chain`.
-fn hello_message<T: BeaconChainTypes>(beacon_chain: &BeaconChain<T>) -> HelloMessage {
+pub(crate) fn hello_message<T: BeaconChainTypes>(beacon_chain: &BeaconChain<T>) -> HelloMessage {
     let state = &beacon_chain.head().beacon_state;
 
     HelloMessage {
@@ -527,7 +462,7 @@ impl NetworkContext {
     pub fn disconnect(&mut self, peer_id: PeerId, reason: GoodbyeReason) {
         warn!(
             &self.log,
-            "Disconnecting peer";
+            "Disconnecting peer (RPC)";
             "reason" => format!("{:?}", reason),
             "peer_id" => format!("{:?}", peer_id),
         );
@@ -560,12 +495,8 @@ impl NetworkContext {
     }
 
     fn send_rpc_event(&mut self, peer_id: PeerId, rpc_event: RPCEvent) {
-        self.send(peer_id, OutgoingMessage::RPC(rpc_event))
-    }
-
-    fn send(&mut self, peer_id: PeerId, outgoing_message: OutgoingMessage) {
         self.network_send
-            .try_send(NetworkMessage::Send(peer_id, outgoing_message))
+            .try_send(NetworkMessage::RPC(peer_id, rpc_event))
             .unwrap_or_else(|_| {
                 warn!(
                     self.log,

beacon_node/src/config.rs

@@ -137,6 +137,7 @@ fn process_testnet_subcommand(
                 .and_then(|s| s.parse::<u16>().ok());
 
             builder.import_bootstrap_libp2p_address(server, port)?;
+            builder.import_bootstrap_enr_address(server)?;
             builder.import_bootstrap_eth2_config(server)?;
 
             builder.set_beacon_chain_start_method(BeaconChainStartMethod::HttpBootstrap {
@@ -301,7 +302,7 @@ impl<'a> ConfigBuilder<'a> {
         self.client_config.eth1_backend_method = method;
     }
 
-    /// Import the libp2p address for `server` into the list of bootnodes in `self`.
+    /// Import the libp2p address for `server` into the list of libp2p nodes to connect with.
     ///
     /// If `port` is `Some`, it is used as the port for the `Multiaddr`. If `port` is `None`,
     /// attempts to connect to the `server` via HTTP and retrieve it's libp2p listen port.
@@ -333,6 +334,28 @@ impl<'a> ConfigBuilder<'a> {
         Ok(())
     }
 
+    /// Import the enr address for `server` into the list of initial enrs (boot nodes).
+    pub fn import_bootstrap_enr_address(&mut self, server: &str) -> Result<()> {
+        let bootstrapper = Bootstrapper::connect(server.to_string(), &self.log)?;
+
+        if let Ok(enr) = bootstrapper.enr() {
+            info!(
+                self.log,
+                "Loaded bootstrapper libp2p address";
+                "enr" => format!("{:?}", enr)
+            );
+
+            self.client_config.network.boot_nodes.push(enr);
+        } else {
+            warn!(
+                self.log,
+                "Unable to estimate a bootstrapper enr address, this node may not find any peers."
+            );
+        };
+
+        Ok(())
+    }
+
     /// Set the config data_dir to be an random directory.
     ///
     /// Useful for easily spinning up ephemeral testnets.

beacon_node/src/main.rs

@@ -33,14 +33,14 @@ fn main() {
         .arg(
             Arg::with_name("logfile")
                 .long("logfile")
-                .value_name("logfile")
+                .value_name("FILE")
                 .help("File path where output will be written.")
                 .takes_value(true),
         )
         .arg(
             Arg::with_name("network-dir")
                 .long("network-dir")
-                .value_name("NETWORK-DIR")
+                .value_name("DIR")
                 .help("Data directory for network keys.")
                 .takes_value(true)
                 .global(true)
@@ -83,7 +83,7 @@ fn main() {
             Arg::with_name("boot-nodes")
                 .long("boot-nodes")
                 .allow_hyphen_values(true)
-                .value_name("BOOTNODES")
+                .value_name("ENR-LIST")
                 .help("One or more comma-delimited base64-encoded ENR's to bootstrap the p2p network.")
                 .takes_value(true),
         )
@@ -128,13 +128,14 @@ fn main() {
         .arg(
             Arg::with_name("rpc-address")
                 .long("rpc-address")
-                .value_name("Address")
+                .value_name("ADDRESS")
                 .help("Listen address for RPC endpoint.")
                 .takes_value(true),
         )
         .arg(
             Arg::with_name("rpc-port")
                 .long("rpc-port")
+                .value_name("PORT")
                 .help("Listen port for RPC endpoint.")
                 .conflicts_with("port-bump")
                 .takes_value(true),
@@ -149,14 +150,14 @@ fn main() {
         .arg(
             Arg::with_name("api-address")
                 .long("api-address")
-                .value_name("APIADDRESS")
+                .value_name("ADDRESS")
                 .help("Set the listen address for the RESTful HTTP API server.")
                 .takes_value(true),
         )
         .arg(
             Arg::with_name("api-port")
                 .long("api-port")
-                .value_name("APIPORT")
+                .value_name("PORT")
                 .help("Set the listen TCP port for the RESTful HTTP API server.")
                 .conflicts_with("port-bump")
                 .takes_value(true),
@@ -196,13 +197,6 @@ fn main() {
                 .possible_values(&["info", "debug", "trace", "warn", "error", "crit"])
                 .default_value("trace"),
         )
-        .arg(
-            Arg::with_name("verbosity")
-                .short("v")
-                .multiple(true)
-                .help("Sets the verbosity level")
-                .takes_value(true),
-        )
         /*
          * The "testnet" sub-command.
          *

book/src/interop-cheat-sheet.md

@@ -9,6 +9,7 @@ interop testing.
 	- [Avoid port clashes when starting multiple nodes](#port-bump)
 	- [Specify a custom slot time](#slot-time)
 - Using the beacon node HTTP API:
+	- [Pretty-print the genesis state and state root](#http-state)
 	- [Curl a node's ENR](#http-enr)
 	- [Curl a node's connected peers](#http-peer-ids)
 	- [Curl a node's local peer id](#http-peer-id)
@@ -82,6 +83,15 @@ $ ./beacon_node testnet -t 500 recent 8
 Examples assume there is a Lighthouse node exposing a HTTP API on
 `localhost:5052`. Responses are JSON.
 
+<a name="http-state"></a>
+### Pretty-print the genesis state and state root
+
+Returns the genesis state and state root in your terminal, in YAML.
+
+```
+$ curl --header "Content-Type: application/yaml" "localhost:5052/beacon/state?slot=0"
+```
+
 <a name="http-enr"></a>
 ### Get the node's ENR
 

book/src/interop-scenarios.md

@@ -25,8 +25,11 @@ cheat-sheet](./interop-cheat-sheet.md).
 To start a brand-new beacon node (with no history) use:
 
 ```
-$ ./beacon_node testnet -f quick 8 1567222226
+$ ./beacon_node testnet -f quick 8 <GENESIS_TIME>
 ```
+
+Where `GENESIS_TIME` is in [unix time](https://duckduckgo.com/?q=unix+time&t=ffab&ia=answer).
+
 > Notes:
 >
 > - This method conforms the ["Quick-start

book/src/interop-tips.md

@@ -1 +0,0 @@
-# Interop Tips & Tricks

book/src/setup.md

@@ -9,11 +9,8 @@ See the [Quick instructions](#quick-instructions) for a summary or the
 
 1. Install Rust + Cargo with [rustup](https://rustup.rs/).
 1. Install build dependencies using your package manager.
-    - `$ <package-manager> clang protobuf libssl-dev cmake git-lfs`
-	- Ensure [git-lfs](https://git-lfs.github.com/) is installed with `git lfs
-		install`.
-1. Clone the [sigp/lighthouse](https://github.com/sigp/lighthouse), ensuring to
-   **initialize submodules**.
+    - `$ <package-manager> clang protobuf libssl-dev cmake`
+1. Clone the [sigp/lighthouse](https://github.com/sigp/lighthouse).
 1. In the root of the repo, run the tests with `cargo test --all --release`.
 1. Then, build the binaries with `cargo build --all --release`.
 1. Lighthouse is now fully built and tested.
@@ -37,13 +34,8 @@ steps:
 	  - `protobuf`: required for protobuf serialization (gRPC)
       - `libssl-dev`: also gRPC
 	  - `cmake`: required for building protobuf
-	  - `git-lfs`: The Git extension for [Large File
-	    Support](https://git-lfs.github.com/) (required for Ethereum Foundation
-	    test vectors).
-   1. Clone the repository with submodules: `git clone --recursive
-	  https://github.com/sigp/lighthouse`.  If you're already cloned the repo,
-	  ensure testing submodules are present: `$ git submodule init; git
-	  submodule update`
+   1. Clone the repository with submodules: `git clone
+	  https://github.com/sigp/lighthouse`.
    1. Change directory to the root of the repository.
    1. Run the test suite with `cargo test --all --release`. The build and test
 	  process can take several minutes. If you experience any failures on
@@ -63,3 +55,27 @@ Perl](http://strawberryperl.com/), or alternatively use a choco install command
 Additionally, the dependency `protoc-grpcio v0.3.1` is reported to have issues
 compiling in Windows. You can specify a known working version by editing
 version in `protos/Cargo.toml`  section to `protoc-grpcio = "<=0.3.0"`.
+
+## eth2.0-spec-tests
+
+The
+[ethereum/eth2.0-spec-tests](https://github.com/ethereum/eth2.0-spec-tests/)
+repository contains a large set of tests that verify Lighthouse behaviour
+against the Ethereum Foundation specifications.
+
+The `tests/ef_tests` crate runs these tests and it has some interesting
+behaviours:
+
+- If the `tests/ef_tests/eth2.0-spec-tests` directory is not present, all tests
+	indicate a `pass` when they did not actually run.
+- If that directory _is_ present, the tests are executed faithfully, failing if
+	a discrepancy is found.
+
+The `tests/ef_tests/eth2.0-spec-tests` directory is not present by default. To
+obtain it, use the Makefile in the root of the repository:
+
+```
+make ef_tests
+```
+
+_Note: this will download 100+ MB of test files from the [ethereum/eth2.0-spec-tests](https://github.com/ethereum/eth2.0-spec-tests/)._

eth2/state_processing/src/common/get_attesting_indices.rs

@@ -17,11 +17,9 @@ pub fn get_attesting_indices<T: EthSpec>(
         target_relative_epoch,
     )?;
 
-    /* TODO(freeze): re-enable this?
-    if bitlist.len() > committee.committee.len() {
+    if bitlist.len() != committee.committee.len() {
         return Err(BeaconStateError::InvalidBitfield);
     }
-    */
 
     Ok(committee
         .committee

eth2/state_processing/src/common/get_compact_committees_root.rs

@@ -3,7 +3,7 @@ use types::*;
 
 /// Return the compact committee root at `relative_epoch`.
 ///
-/// Spec v0.8.0
+/// Spec v0.8.3
 pub fn get_compact_committees_root<T: EthSpec>(
     state: &BeaconState<T>,
     relative_epoch: RelativeEpoch,
@@ -11,28 +11,13 @@ pub fn get_compact_committees_root<T: EthSpec>(
 ) -> Result<Hash256, BeaconStateError> {
     let mut committees =
         FixedVector::<_, T::ShardCount>::from_elem(CompactCommittee::<T>::default());
-    // FIXME: this is a spec bug, whereby the start shard for the epoch after the next epoch
-    // is mistakenly used. The start shard from the cache SHOULD work.
-    // Waiting on a release to fix https://github.com/ethereum/eth2.0-specs/issues/1315
-    let start_shard = if relative_epoch == RelativeEpoch::Next {
-        state.next_epoch_start_shard(spec)?
-    } else {
-        state.get_epoch_start_shard(relative_epoch)?
-    };
+    let start_shard = state.get_epoch_start_shard(relative_epoch)?;
 
     for committee_number in 0..state.get_committee_count(relative_epoch)? {
         let shard = (start_shard + committee_number) % T::ShardCount::to_u64();
-        // FIXME: this is a partial workaround for the above, but it only works in the case
-        // where there's a committee for every shard in every epoch. It works for the minimal
-        // tests but not the mainnet ones.
-        let fake_shard = if relative_epoch == RelativeEpoch::Next {
-            (shard + 1) % T::ShardCount::to_u64()
-        } else {
-            shard
-        };
 
         for &index in state
-            .get_crosslink_committee_for_shard(fake_shard, relative_epoch)?
+            .get_crosslink_committee_for_shard(shard, relative_epoch)?
             .committee
         {
             let validator = state

eth2/state_processing/src/common/get_indexed_attestation.rs

@@ -11,6 +11,8 @@ pub fn get_indexed_attestation<T: EthSpec>(
     state: &BeaconState<T>,
     attestation: &Attestation<T>,
 ) -> Result<IndexedAttestation<T>> {
+    // Note: we rely on both calls to `get_attesting_indices` to check the bitfield lengths
+    // against the committee length
     let attesting_indices =
         get_attesting_indices(state, &attestation.data, &attestation.aggregation_bits)?;
 

eth2/state_processing/src/per_epoch_processing.rs

@@ -1,8 +1,5 @@
 use crate::common::get_compact_committees_root;
-use apply_rewards::process_rewards_and_penalties;
 use errors::EpochProcessingError as Error;
-use process_slashings::process_slashings;
-use registry_updates::process_registry_updates;
 use std::collections::HashMap;
 use tree_hash::TreeHash;
 use types::*;
@@ -17,6 +14,10 @@ pub mod tests;
 pub mod validator_statuses;
 pub mod winning_root;
 
+pub use apply_rewards::process_rewards_and_penalties;
+pub use process_slashings::process_slashings;
+pub use registry_updates::process_registry_updates;
+
 /// Maps a shard to a winning root.
 ///
 /// It is generated during crosslink processing and later used to reward/penalize validators.
@@ -218,45 +219,29 @@ pub fn process_final_updates<T: EthSpec>(
         }
     }
 
-    // Update start shard.
-    state.start_shard = state.next_epoch_start_shard(spec)?;
-
-    // This is a hack to allow us to update index roots and slashed balances for the next epoch.
-    //
-    // The indentation here is to make it obvious where the weird stuff happens.
-    {
-        state.slot += 1;
-
-        // Set active index root
-        let index_epoch = next_epoch + spec.activation_exit_delay;
-        let indices_list = VariableList::<usize, T::ValidatorRegistryLimit>::from(
-            state.get_active_validator_indices(index_epoch),
-        );
-        state.set_active_index_root(
-            index_epoch,
-            Hash256::from_slice(&indices_list.tree_hash_root()),
-            spec,
-        )?;
-
-        // Reset slashings
-        state.set_slashings(next_epoch, 0)?;
-
-        // Set randao mix
-        state.set_randao_mix(next_epoch, *state.get_randao_mix(current_epoch)?)?;
-
-        state.slot -= 1;
-    }
+    // Set active index root
+    let index_epoch = next_epoch + spec.activation_exit_delay;
+    let indices_list = VariableList::<usize, T::ValidatorRegistryLimit>::from(
+        state.get_active_validator_indices(index_epoch),
+    );
+    state.set_active_index_root(
+        index_epoch,
+        Hash256::from_slice(&indices_list.tree_hash_root()),
+        spec,
+    )?;
 
     // Set committees root
-    // Note: we do this out-of-order w.r.t. to the spec, because we don't want the slot to be
-    // incremented. It's safe because the updates to slashings and the RANDAO mix (above) don't
-    // affect this.
     state.set_compact_committee_root(
         next_epoch,
         get_compact_committees_root(state, RelativeEpoch::Next, spec)?,
-        spec,
     )?;
 
+    // Reset slashings
+    state.set_slashings(next_epoch, 0)?;
+
+    // Set randao mix
+    state.set_randao_mix(next_epoch, *state.get_randao_mix(current_epoch)?)?;
+
     // Set historical root accumulator
     if next_epoch.as_u64() % (T::SlotsPerHistoricalRoot::to_u64() / T::slots_per_epoch()) == 0 {
         let historical_batch = state.historical_batch();
@@ -265,6 +250,9 @@ pub fn process_final_updates<T: EthSpec>(
             .push(Hash256::from_slice(&historical_batch.tree_hash_root()))?;
     }
 
+    // Update start shard.
+    state.start_shard = state.get_epoch_start_shard(RelativeEpoch::Next)?;
+
     // Rotate current/previous epoch attestations
     state.previous_epoch_attestations =
         std::mem::replace(&mut state.current_epoch_attestations, VariableList::empty());

eth2/types/src/attestation_data.rs

@@ -4,25 +4,13 @@ use crate::{Checkpoint, Crosslink, Hash256};
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
-use tree_hash::TreeHash;
-use tree_hash_derive::{SignedRoot, TreeHash};
+use tree_hash_derive::TreeHash;
 
 /// The data upon which an attestation is based.
 ///
 /// Spec v0.8.0
 #[derive(
-    Debug,
-    Clone,
-    PartialEq,
-    Eq,
-    Serialize,
-    Deserialize,
-    Hash,
-    Encode,
-    Decode,
-    TreeHash,
-    TestRandom,
-    SignedRoot,
+    Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash, Encode, Decode, TreeHash, TestRandom,
 )]
 pub struct AttestationData {
     // LMD GHOST vote

eth2/types/src/beacon_state.rs

@@ -60,6 +60,22 @@ pub enum Error {
     SszTypesError(ssz_types::Error),
 }
 
+/// Control whether an epoch-indexed field can be indexed at the next epoch or not.
+#[derive(Debug, PartialEq, Clone, Copy)]
+enum AllowNextEpoch {
+    True,
+    False,
+}
+
+impl AllowNextEpoch {
+    fn upper_bound_of(self, current_epoch: Epoch) -> Epoch {
+        match self {
+            AllowNextEpoch::True => current_epoch + 1,
+            AllowNextEpoch::False => current_epoch,
+        }
+    }
+}
+
 /// The state of the `BeaconChain` at some slot.
 ///
 /// Spec v0.8.0
@@ -108,12 +124,12 @@ where
     pub start_shard: u64,
     pub randao_mixes: FixedVector<Hash256, T::EpochsPerHistoricalVector>,
     #[compare_fields(as_slice)]
-    active_index_roots: FixedVector<Hash256, T::EpochsPerHistoricalVector>,
+    pub active_index_roots: FixedVector<Hash256, T::EpochsPerHistoricalVector>,
     #[compare_fields(as_slice)]
-    compact_committees_roots: FixedVector<Hash256, T::EpochsPerHistoricalVector>,
+    pub compact_committees_roots: FixedVector<Hash256, T::EpochsPerHistoricalVector>,
 
     // Slashings
-    slashings: FixedVector<u64, T::EpochsPerSlashingsVector>,
+    pub slashings: FixedVector<u64, T::EpochsPerSlashingsVector>,
 
     // Attestations
     pub previous_epoch_attestations: VariableList<PendingAttestation<T>, T::MaxPendingAttestations>,
@@ -282,14 +298,6 @@ impl<T: EthSpec> BeaconState<T> {
         Ok(cache.epoch_start_shard())
     }
 
-    pub fn next_epoch_start_shard(&self, spec: &ChainSpec) -> Result<u64, Error> {
-        let cache = self.cache(RelativeEpoch::Current)?;
-        let active_validator_count = cache.active_validator_count();
-        let shard_delta = T::get_shard_delta(active_validator_count, spec.target_committee_size);
-
-        Ok((self.start_shard + shard_delta) % T::ShardCount::to_u64())
-    }
-
     /// Get the slot of an attestation.
     ///
     /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized.
@@ -463,12 +471,16 @@ impl<T: EthSpec> BeaconState<T> {
 
     /// Safely obtains the index for `randao_mixes`
     ///
-    /// Spec v0.8.0
-    fn get_randao_mix_index(&self, epoch: Epoch) -> Result<usize, Error> {
+    /// Spec v0.8.1
+    fn get_randao_mix_index(
+        &self,
+        epoch: Epoch,
+        allow_next_epoch: AllowNextEpoch,
+    ) -> Result<usize, Error> {
         let current_epoch = self.current_epoch();
         let len = T::EpochsPerHistoricalVector::to_u64();
 
-        if epoch + len > current_epoch && epoch <= current_epoch {
+        if current_epoch < epoch + len && epoch <= allow_next_epoch.upper_bound_of(current_epoch) {
             Ok(epoch.as_usize() % len as usize)
         } else {
             Err(Error::EpochOutOfBounds)
@@ -496,7 +508,7 @@ impl<T: EthSpec> BeaconState<T> {
     ///
     /// Spec v0.8.1
     pub fn get_randao_mix(&self, epoch: Epoch) -> Result<&Hash256, Error> {
-        let i = self.get_randao_mix_index(epoch)?;
+        let i = self.get_randao_mix_index(epoch, AllowNextEpoch::False)?;
         Ok(&self.randao_mixes[i])
     }
 
@@ -504,21 +516,29 @@ impl<T: EthSpec> BeaconState<T> {
     ///
     /// Spec v0.8.1
     pub fn set_randao_mix(&mut self, epoch: Epoch, mix: Hash256) -> Result<(), Error> {
-        let i = self.get_randao_mix_index(epoch)?;
+        let i = self.get_randao_mix_index(epoch, AllowNextEpoch::True)?;
         self.randao_mixes[i] = mix;
         Ok(())
     }
 
     /// Safely obtains the index for `active_index_roots`, given some `epoch`.
     ///
+    /// If `allow_next_epoch` is `True`, then we allow an _extra_ one epoch of lookahead.
+    ///
     /// Spec v0.8.1
-    fn get_active_index_root_index(&self, epoch: Epoch, spec: &ChainSpec) -> Result<usize, Error> {
+    fn get_active_index_root_index(
+        &self,
+        epoch: Epoch,
+        spec: &ChainSpec,
+        allow_next_epoch: AllowNextEpoch,
+    ) -> Result<usize, Error> {
         let current_epoch = self.current_epoch();
 
         let lookahead = spec.activation_exit_delay;
         let lookback = self.active_index_roots.len() as u64 - lookahead;
+        let epoch_upper_bound = allow_next_epoch.upper_bound_of(current_epoch) + lookahead;
 
-        if epoch + lookback > current_epoch && current_epoch + lookahead >= epoch {
+        if current_epoch < epoch + lookback && epoch <= epoch_upper_bound {
             Ok(epoch.as_usize() % self.active_index_roots.len())
         } else {
             Err(Error::EpochOutOfBounds)
@@ -529,7 +549,7 @@ impl<T: EthSpec> BeaconState<T> {
     ///
     /// Spec v0.8.1
     pub fn get_active_index_root(&self, epoch: Epoch, spec: &ChainSpec) -> Result<Hash256, Error> {
-        let i = self.get_active_index_root_index(epoch, spec)?;
+        let i = self.get_active_index_root_index(epoch, spec, AllowNextEpoch::False)?;
         Ok(self.active_index_roots[i])
     }
 
@@ -542,7 +562,7 @@ impl<T: EthSpec> BeaconState<T> {
         index_root: Hash256,
         spec: &ChainSpec,
     ) -> Result<(), Error> {
-        let i = self.get_active_index_root_index(epoch, spec)?;
+        let i = self.get_active_index_root_index(epoch, spec, AllowNextEpoch::True)?;
         self.active_index_roots[i] = index_root;
         Ok(())
     }
@@ -556,19 +576,17 @@ impl<T: EthSpec> BeaconState<T> {
 
     /// Safely obtains the index for `compact_committees_roots`, given some `epoch`.
     ///
-    /// Spec v0.8.0
+    /// Spec v0.8.1
     fn get_compact_committee_root_index(
         &self,
         epoch: Epoch,
-        spec: &ChainSpec,
+        allow_next_epoch: AllowNextEpoch,
     ) -> Result<usize, Error> {
         let current_epoch = self.current_epoch();
+        let len = T::EpochsPerHistoricalVector::to_u64();
 
-        let lookahead = spec.activation_exit_delay;
-        let lookback = self.compact_committees_roots.len() as u64 - lookahead;
-
-        if epoch + lookback > current_epoch && current_epoch + lookahead >= epoch {
-            Ok(epoch.as_usize() % self.compact_committees_roots.len())
+        if current_epoch < epoch + len && epoch <= allow_next_epoch.upper_bound_of(current_epoch) {
+            Ok(epoch.as_usize() % len as usize)
         } else {
             Err(Error::EpochOutOfBounds)
         }
@@ -576,26 +594,21 @@ impl<T: EthSpec> BeaconState<T> {
 
     /// Return the `compact_committee_root` at a recent `epoch`.
     ///
-    /// Spec v0.8.0
-    pub fn get_compact_committee_root(
-        &self,
-        epoch: Epoch,
-        spec: &ChainSpec,
-    ) -> Result<Hash256, Error> {
-        let i = self.get_compact_committee_root_index(epoch, spec)?;
+    /// Spec v0.8.1
+    pub fn get_compact_committee_root(&self, epoch: Epoch) -> Result<Hash256, Error> {
+        let i = self.get_compact_committee_root_index(epoch, AllowNextEpoch::False)?;
         Ok(self.compact_committees_roots[i])
     }
 
     /// Set the `compact_committee_root` at a recent `epoch`.
     ///
-    /// Spec v0.8.0
+    /// Spec v0.8.1
     pub fn set_compact_committee_root(
         &mut self,
         epoch: Epoch,
         index_root: Hash256,
-        spec: &ChainSpec,
     ) -> Result<(), Error> {
-        let i = self.get_compact_committee_root_index(epoch, spec)?;
+        let i = self.get_compact_committee_root_index(epoch, AllowNextEpoch::True)?;
         self.compact_committees_roots[i] = index_root;
         Ok(())
     }
@@ -646,14 +659,19 @@ impl<T: EthSpec> BeaconState<T> {
 
     /// Safely obtain the index for `slashings`, given some `epoch`.
     ///
-    /// Spec v0.8.0
-    fn get_slashings_index(&self, epoch: Epoch) -> Result<usize, Error> {
+    /// Spec v0.8.1
+    fn get_slashings_index(
+        &self,
+        epoch: Epoch,
+        allow_next_epoch: AllowNextEpoch,
+    ) -> Result<usize, Error> {
         // We allow the slashings vector to be accessed at any cached epoch at or before
-        // the current epoch.
-        if epoch <= self.current_epoch()
-            && epoch + T::EpochsPerSlashingsVector::to_u64() >= self.current_epoch() + 1
+        // the current epoch, or the next epoch if `AllowNextEpoch::True` is passed.
+        let current_epoch = self.current_epoch();
+        if current_epoch < epoch + T::EpochsPerSlashingsVector::to_u64()
+            && epoch <= allow_next_epoch.upper_bound_of(current_epoch)
         {
-            Ok((epoch.as_u64() % T::EpochsPerSlashingsVector::to_u64()) as usize)
+            Ok(epoch.as_usize() % T::EpochsPerSlashingsVector::to_usize())
         } else {
             Err(Error::EpochOutOfBounds)
         }
@@ -668,17 +686,17 @@ impl<T: EthSpec> BeaconState<T> {
 
     /// Get the total slashed balances for some epoch.
     ///
-    /// Spec v0.8.0
+    /// Spec v0.8.1
     pub fn get_slashings(&self, epoch: Epoch) -> Result<u64, Error> {
-        let i = self.get_slashings_index(epoch)?;
+        let i = self.get_slashings_index(epoch, AllowNextEpoch::False)?;
         Ok(self.slashings[i])
     }
 
     /// Set the total slashed balances for some epoch.
     ///
-    /// Spec v0.8.0
+    /// Spec v0.8.1
     pub fn set_slashings(&mut self, epoch: Epoch, value: u64) -> Result<(), Error> {
-        let i = self.get_slashings_index(epoch)?;
+        let i = self.get_slashings_index(epoch, AllowNextEpoch::True)?;
         self.slashings[i] = value;
         Ok(())
     }

eth2/types/src/beacon_state/tests.rs

@@ -90,11 +90,11 @@ fn test_active_index<T: EthSpec>(state_slot: Slot) {
 
     // Test the start and end of the range.
     assert_eq!(
-        state.get_active_index_root_index(*range.start(), &spec),
+        state.get_active_index_root_index(*range.start(), &spec, AllowNextEpoch::False),
         Ok(modulo(*range.start()))
     );
     assert_eq!(
-        state.get_active_index_root_index(*range.end(), &spec),
+        state.get_active_index_root_index(*range.end(), &spec, AllowNextEpoch::False),
         Ok(modulo(*range.end()))
     );
 
@@ -102,12 +102,12 @@ fn test_active_index<T: EthSpec>(state_slot: Slot) {
     if state.current_epoch() > 0 {
         // Test is invalid on epoch zero, cannot subtract from zero.
         assert_eq!(
-            state.get_active_index_root_index(*range.start() - 1, &spec),
+            state.get_active_index_root_index(*range.start() - 1, &spec, AllowNextEpoch::False),
             Err(Error::EpochOutOfBounds)
         );
     }
     assert_eq!(
-        state.get_active_index_root_index(*range.end() + 1, &spec),
+        state.get_active_index_root_index(*range.end() + 1, &spec, AllowNextEpoch::False),
         Err(Error::EpochOutOfBounds)
     );
 }

eth2/types/src/checkpoint.rs

@@ -3,8 +3,7 @@ use crate::{Epoch, Hash256};
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
-use tree_hash::TreeHash;
-use tree_hash_derive::{SignedRoot, TreeHash};
+use tree_hash_derive::TreeHash;
 
 /// Casper FFG checkpoint, used in attestations.
 ///
@@ -22,7 +21,6 @@ use tree_hash_derive::{SignedRoot, TreeHash};
     Decode,
     TreeHash,
     TestRandom,
-    SignedRoot,
 )]
 pub struct Checkpoint {
     pub epoch: Epoch,

eth2/types/src/lib.rs

@@ -86,5 +86,8 @@ pub type AttesterMap = HashMap<(u64, u64), Vec<usize>>;
 /// Maps a slot to a block proposer.
 pub type ProposerMap = HashMap<u64, usize>;
 
-pub use bls::{AggregatePublicKey, AggregateSignature, Keypair, PublicKey, SecretKey, Signature};
+pub use bls::{
+    AggregatePublicKey, AggregateSignature, Keypair, PublicKey, PublicKeyBytes, SecretKey,
+    Signature, SignatureBytes,
+};
 pub use ssz_types::{typenum, typenum::Unsigned, BitList, BitVector, FixedVector, VariableList};

eth2/types/src/slot_epoch_macros.rs

@@ -201,6 +201,10 @@ macro_rules! impl_ssz {
                 <u64 as Encode>::ssz_fixed_len()
             }
 
+            fn ssz_bytes_len(&self) -> usize {
+                0_u64.ssz_bytes_len()
+            }
+
             fn ssz_append(&self, buf: &mut Vec<u8>) {
                 self.0.ssz_append(buf)
             }

eth2/utils/bls/Cargo.toml

@@ -5,7 +5,8 @@ authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 
 [dependencies]
-milagro_bls = { git = "https://github.com/sigp/milagro_bls", tag = "v0.10.0" }
+# FIXME: update sigp repo
+milagro_bls = { git = "https://github.com/michaelsproul/milagro_bls", branch = "little-endian-v0.10" }
 eth2_hashing = { path = "../eth2_hashing" }
 hex = "0.3"
 rand = "^0.5"

eth2/utils/bls/src/macros.rs

@@ -9,6 +9,10 @@ macro_rules! impl_ssz {
                 $byte_size
             }
 
+            fn ssz_bytes_len(&self) -> usize {
+                $byte_size
+            }
+
             fn ssz_append(&self, buf: &mut Vec<u8>) {
                 buf.append(&mut self.as_bytes())
             }

eth2/utils/eth2_interop_keypairs/Cargo.toml

@@ -10,7 +10,7 @@ edition = "2018"
 lazy_static = "1.4"
 num-bigint = "0.2"
 eth2_hashing = "0.1"
-milagro_bls = { git = "https://github.com/sigp/milagro_bls", tag = "v0.10.0" }
+milagro_bls = { git = "https://github.com/michaelsproul/milagro_bls", branch = "little-endian-v0.10" }
 
 [dev-dependencies]
 base64 = "0.10"

eth2/utils/slot_clock/src/lib.rs

@@ -5,7 +5,7 @@ mod metrics;
 mod system_time_slot_clock;
 mod testing_slot_clock;
 
-use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
+use std::time::{Duration, Instant, SystemTime, SystemTimeError, UNIX_EPOCH};
 
 pub use crate::system_time_slot_clock::SystemTimeSlotClock;
 pub use crate::testing_slot_clock::TestingSlotClock;
@@ -17,18 +17,21 @@ pub trait SlotClock: Send + Sync + Sized {
         genesis_slot: Slot,
         genesis_seconds: u64,
         slot_duration: Duration,
-    ) -> Option<Self> {
-        let duration_between_now_and_unix_epoch =
-            SystemTime::now().duration_since(UNIX_EPOCH).ok()?;
+    ) -> Result<Self, SystemTimeError> {
+        let duration_between_now_and_unix_epoch = SystemTime::now().duration_since(UNIX_EPOCH)?;
         let duration_between_unix_epoch_and_genesis = Duration::from_secs(genesis_seconds);
 
-        if duration_between_now_and_unix_epoch < duration_between_unix_epoch_and_genesis {
-            None
+        let genesis_instant = if duration_between_now_and_unix_epoch
+            < duration_between_unix_epoch_and_genesis
+        {
+            Instant::now()
+                + (duration_between_unix_epoch_and_genesis - duration_between_now_and_unix_epoch)
         } else {
-            let genesis_instant = Instant::now()
-                - (duration_between_now_and_unix_epoch - duration_between_unix_epoch_and_genesis);
-            Some(Self::new(genesis_slot, genesis_instant, slot_duration))
-        }
+            Instant::now()
+                - (duration_between_now_and_unix_epoch - duration_between_unix_epoch_and_genesis)
+        };
+
+        Ok(Self::new(genesis_slot, genesis_instant, slot_duration))
     }
 
     fn new(genesis_slot: Slot, genesis: Instant, slot_duration: Duration) -> Self;

eth2/utils/slot_clock/src/system_time_slot_clock.rs

@@ -42,7 +42,7 @@ impl SlotClock for SystemTimeSlotClock {
     fn duration_to_next_slot(&self) -> Option<Duration> {
         let now = Instant::now();
         if now < self.genesis {
-            None
+            Some(self.genesis - now)
         } else {
             let duration_since_genesis = now - self.genesis;
             let millis_since_genesis = duration_since_genesis.as_millis();

eth2/utils/ssz/examples/struct_definition.rs

@@ -12,6 +12,13 @@ impl Encode for Foo {
         <u16 as Encode>::is_ssz_fixed_len() && <Vec<u16> as Encode>::is_ssz_fixed_len()
     }
 
+    fn ssz_bytes_len(&self) -> usize {
+        <u16 as Encode>::ssz_fixed_len()
+            + ssz::BYTES_PER_LENGTH_OFFSET
+            + <u16 as Encode>::ssz_fixed_len()
+            + self.b.ssz_bytes_len()
+    }
+
     fn ssz_append(&self, buf: &mut Vec<u8>) {
         let offset = <u16 as Encode>::ssz_fixed_len()
             + <Vec<u16> as Encode>::ssz_fixed_len()

eth2/utils/ssz/src/encode.rs

@@ -27,6 +27,8 @@ pub trait Encode {
         BYTES_PER_LENGTH_OFFSET
     }
 
+    fn ssz_bytes_len(&self) -> usize;
+
     /// Returns the full-form encoding of this object.
     ///
     /// The default implementation of this method should suffice for most cases.

eth2/utils/ssz/src/encode/impls.rs

@@ -13,6 +13,10 @@ macro_rules! impl_encodable_for_uint {
                 $bit_size / 8
             }
 
+            fn ssz_bytes_len(&self) -> usize {
+                $bit_size / 8
+            }
+
             fn ssz_append(&self, buf: &mut Vec<u8>) {
                 buf.extend_from_slice(&self.to_le_bytes());
             }
@@ -58,6 +62,23 @@ macro_rules! impl_encode_for_tuples {
                     }
                 }
 
+                fn ssz_bytes_len(&self) -> usize {
+                    if <Self as Encode>::is_ssz_fixed_len() {
+                        <Self as Encode>::ssz_fixed_len()
+                    } else {
+                        let mut len = 0;
+                        $(
+                            len += if <$T as Encode>::is_ssz_fixed_len() {
+                                <$T as Encode>::ssz_fixed_len()
+                            } else {
+                                BYTES_PER_LENGTH_OFFSET +
+                                self.$idx.ssz_bytes_len()
+                            };
+                        )*
+                        len
+                    }
+                }
+
                 fn ssz_append(&self, buf: &mut Vec<u8>) {
                     let offset = $(
                             <$T as Encode>::ssz_fixed_len() +
@@ -185,6 +206,19 @@ impl<T: Encode> Encode for Option<T> {
         false
     }
 
+    fn ssz_bytes_len(&self) -> usize {
+        if let Some(some) = self {
+            let len = if <T as Encode>::is_ssz_fixed_len() {
+                <T as Encode>::ssz_fixed_len()
+            } else {
+                some.ssz_bytes_len()
+            };
+            len + BYTES_PER_LENGTH_OFFSET
+        } else {
+            BYTES_PER_LENGTH_OFFSET
+        }
+    }
+
     fn ssz_append(&self, buf: &mut Vec<u8>) {
         match self {
             None => buf.append(&mut encode_union_index(0)),
@@ -201,6 +235,16 @@ impl<T: Encode> Encode for Vec<T> {
         false
     }
 
+    fn ssz_bytes_len(&self) -> usize {
+        if <T as Encode>::is_ssz_fixed_len() {
+            <T as Encode>::ssz_fixed_len() * self.len()
+        } else {
+            let mut len = self.into_iter().map(|item| item.ssz_bytes_len()).sum();
+            len += BYTES_PER_LENGTH_OFFSET * self.len();
+            len
+        }
+    }
+
     fn ssz_append(&self, buf: &mut Vec<u8>) {
         if T::is_ssz_fixed_len() {
             buf.reserve(T::ssz_fixed_len() * self.len());
@@ -229,6 +273,10 @@ impl Encode for bool {
         1
     }
 
+    fn ssz_bytes_len(&self) -> usize {
+        1
+    }
+
     fn ssz_append(&self, buf: &mut Vec<u8>) {
         buf.extend_from_slice(&(*self as u8).to_le_bytes());
     }
@@ -243,6 +291,10 @@ impl Encode for NonZeroUsize {
         <usize as Encode>::ssz_fixed_len()
     }
 
+    fn ssz_bytes_len(&self) -> usize {
+        std::mem::size_of::<usize>()
+    }
+
     fn ssz_append(&self, buf: &mut Vec<u8>) {
         self.get().ssz_append(buf)
     }
@@ -257,6 +309,10 @@ impl Encode for H256 {
         32
     }
 
+    fn ssz_bytes_len(&self) -> usize {
+        32
+    }
+
     fn ssz_append(&self, buf: &mut Vec<u8>) {
         buf.extend_from_slice(self.as_bytes());
     }
@@ -271,6 +327,10 @@ impl Encode for U256 {
         32
     }
 
+    fn ssz_bytes_len(&self) -> usize {
+        32
+    }
+
     fn ssz_append(&self, buf: &mut Vec<u8>) {
         let n = <Self as Encode>::ssz_fixed_len();
         let s = buf.len();
@@ -289,6 +349,10 @@ impl Encode for U128 {
         16
     }
 
+    fn ssz_bytes_len(&self) -> usize {
+        16
+    }
+
     fn ssz_append(&self, buf: &mut Vec<u8>) {
         let n = <Self as Encode>::ssz_fixed_len();
         let s = buf.len();
@@ -309,6 +373,10 @@ macro_rules! impl_encodable_for_u8_array {
                 $len
             }
 
+            fn ssz_bytes_len(&self) -> usize {
+                $len
+            }
+
             fn ssz_append(&self, buf: &mut Vec<u8>) {
                 buf.extend_from_slice(&self[..]);
             }

eth2/utils/ssz/src/lib.rs

@@ -36,7 +36,6 @@
 
 mod decode;
 mod encode;
-mod macros;
 
 pub use decode::{
     impls::decode_list_of_variable_length_items, Decode, DecodeError, SszDecoder, SszDecoderBuilder,

eth2/utils/ssz/src/macros.rs

@@ -1,96 +1 @@
-/// Implements `Encode` for `$impl_type` using an implementation of `From<$impl_type> for
-/// $from_type`.
-///
-/// In effect, this allows for easy implementation of `Encode` for some type that implements a
-/// `From` conversion into another type that already has `Encode` implemented.
-#[macro_export]
-macro_rules! impl_encode_via_from {
-    ($impl_type: ty, $from_type: ty) => {
-        impl ssz::Encode for $impl_type {
-            fn is_ssz_fixed_len() -> bool {
-                <$from_type as ssz::Encode>::is_ssz_fixed_len()
-            }
 
-            fn ssz_fixed_len() -> usize {
-                <$from_type as ssz::Encode>::ssz_fixed_len()
-            }
-
-            fn ssz_append(&self, buf: &mut Vec<u8>) {
-                let conv: $from_type = self.clone().into();
-
-                conv.ssz_append(buf)
-            }
-        }
-    };
-}
-
-/// Implements `Decode` for `$impl_type` using an implementation of `From<$impl_type> for
-/// $from_type`.
-///
-/// In effect, this allows for easy implementation of `Decode` for some type that implements a
-/// `From` conversion into another type that already has `Decode` implemented.
-#[macro_export]
-macro_rules! impl_decode_via_from {
-    ($impl_type: ty, $from_type: tt) => {
-        impl ssz::Decode for $impl_type {
-            fn is_ssz_fixed_len() -> bool {
-                <$from_type as ssz::Decode>::is_ssz_fixed_len()
-            }
-
-            fn ssz_fixed_len() -> usize {
-                <$from_type as ssz::Decode>::ssz_fixed_len()
-            }
-
-            fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, ssz::DecodeError> {
-                $from_type::from_ssz_bytes(bytes).and_then(|dec| Ok(dec.into()))
-            }
-        }
-    };
-}
-
-#[cfg(test)]
-mod tests {
-    use self::ssz::{Decode, Encode};
-    use crate as ssz;
-
-    #[derive(PartialEq, Debug, Clone, Copy)]
-    struct Wrapper(u64);
-
-    impl From<u64> for Wrapper {
-        fn from(x: u64) -> Wrapper {
-            Wrapper(x)
-        }
-    }
-
-    impl From<Wrapper> for u64 {
-        fn from(x: Wrapper) -> u64 {
-            x.0
-        }
-    }
-
-    impl_encode_via_from!(Wrapper, u64);
-    impl_decode_via_from!(Wrapper, u64);
-
-    #[test]
-    fn impl_encode_via_from() {
-        let check_encode = |a: u64, b: Wrapper| assert_eq!(a.as_ssz_bytes(), b.as_ssz_bytes());
-
-        check_encode(0, Wrapper(0));
-        check_encode(1, Wrapper(1));
-        check_encode(42, Wrapper(42));
-    }
-
-    #[test]
-    fn impl_decode_via_from() {
-        let check_decode = |bytes: Vec<u8>| {
-            let a = u64::from_ssz_bytes(&bytes).unwrap();
-            let b = Wrapper::from_ssz_bytes(&bytes).unwrap();
-
-            assert_eq!(a, b.into())
-        };
-
-        check_decode(vec![0, 0, 0, 0, 0, 0, 0, 0]);
-        check_decode(vec![1, 0, 0, 0, 0, 0, 0, 0]);
-        check_decode(vec![1, 0, 0, 0, 2, 0, 0, 0]);
-    }
-}

eth2/utils/ssz/tests/tests.rs

@@ -8,6 +8,7 @@ mod round_trip {
     fn round_trip<T: Encode + Decode + std::fmt::Debug + PartialEq>(items: Vec<T>) {
         for item in items {
             let encoded = &item.as_ssz_bytes();
+            assert_eq!(item.ssz_bytes_len(), encoded.len());
             assert_eq!(T::from_ssz_bytes(&encoded), Ok(item));
         }
     }

eth2/utils/ssz_derive/src/lib.rs

@@ -81,9 +81,12 @@ pub fn ssz_encode_derive(input: TokenStream) -> TokenStream {
     };
 
     let field_idents = get_serializable_named_field_idents(&struct_data);
+    let field_idents_a = get_serializable_named_field_idents(&struct_data);
     let field_types_a = get_serializable_field_types(&struct_data);
     let field_types_b = field_types_a.clone();
-    let field_types_c = field_types_a.clone();
+    let field_types_d = field_types_a.clone();
+    let field_types_e = field_types_a.clone();
+    let field_types_f = field_types_a.clone();
 
     let output = quote! {
         impl #impl_generics ssz::Encode for #name #ty_generics #where_clause {
@@ -105,9 +108,27 @@ pub fn ssz_encode_derive(input: TokenStream) -> TokenStream {
                 }
             }
 
+            fn ssz_bytes_len(&self) -> usize {
+                if <Self as ssz::Encode>::is_ssz_fixed_len() {
+                    <Self as ssz::Encode>::ssz_fixed_len()
+                } else {
+                    let mut len = 0;
+                    #(
+                        if <#field_types_d as ssz::Encode>::is_ssz_fixed_len() {
+                            len += <#field_types_e as ssz::Encode>::ssz_fixed_len();
+                        } else {
+                            len += ssz::BYTES_PER_LENGTH_OFFSET;
+                            len += self.#field_idents_a.ssz_bytes_len();
+                        }
+                    )*
+
+                    len
+                }
+            }
+
             fn ssz_append(&self, buf: &mut Vec<u8>) {
                 let offset = #(
-                        <#field_types_c as ssz::Encode>::ssz_fixed_len() +
+                        <#field_types_f as ssz::Encode>::ssz_fixed_len() +
                     )*
                         0;
 

eth2/utils/ssz_types/src/bitfield.rs

@@ -476,6 +476,12 @@ impl<N: Unsigned + Clone> Encode for Bitfield<Variable<N>> {
         false
     }
 
+    fn ssz_bytes_len(&self) -> usize {
+        // We could likely do better than turning this into bytes and reading the length, however
+        // it is kept this way for simplicity.
+        self.clone().into_bytes().len()
+    }
+
     fn ssz_append(&self, buf: &mut Vec<u8>) {
         buf.append(&mut self.clone().into_bytes())
     }
@@ -498,6 +504,10 @@ impl<N: Unsigned + Clone> Encode for Bitfield<Fixed<N>> {
         true
     }
 
+    fn ssz_bytes_len(&self) -> usize {
+        self.as_slice().len()
+    }
+
     fn ssz_fixed_len() -> usize {
         bytes_for_bit_len(N::to_usize())
     }
@@ -616,6 +626,7 @@ mod bitvector {
     pub type BitVector4 = BitVector<typenum::U4>;
     pub type BitVector8 = BitVector<typenum::U8>;
     pub type BitVector16 = BitVector<typenum::U16>;
+    pub type BitVector64 = BitVector<typenum::U64>;
 
     #[test]
     fn ssz_encode() {
@@ -706,6 +717,18 @@ mod bitvector {
     fn assert_round_trip<T: Encode + Decode + PartialEq + std::fmt::Debug>(t: T) {
         assert_eq!(T::from_ssz_bytes(&t.as_ssz_bytes()).unwrap(), t);
     }
+
+    #[test]
+    fn ssz_bytes_len() {
+        for i in 0..64 {
+            let mut bitfield = BitVector64::new();
+            for j in 0..i {
+                bitfield.set(j, true).expect("should set bit in bounds");
+            }
+            let bytes = bitfield.as_ssz_bytes();
+            assert_eq!(bitfield.ssz_bytes_len(), bytes.len(), "i = {}", i);
+        }
+    }
 }
 
 #[cfg(test)]
@@ -1152,4 +1175,16 @@ mod bitlist {
             vec![false, false, true, false, false, false, false, false, true]
         );
     }
+
+    #[test]
+    fn ssz_bytes_len() {
+        for i in 1..64 {
+            let mut bitfield = BitList1024::with_capacity(i).unwrap();
+            for j in 0..i {
+                bitfield.set(j, true).expect("should set bit in bounds");
+            }
+            let bytes = bitfield.as_ssz_bytes();
+            assert_eq!(bitfield.ssz_bytes_len(), bytes.len(), "i = {}", i);
+        }
+    }
 }

eth2/utils/ssz_types/src/fixed_vector.rs

@@ -172,7 +172,7 @@ where
     T: ssz::Encode,
 {
     fn is_ssz_fixed_len() -> bool {
-        true
+        T::is_ssz_fixed_len()
     }
 
     fn ssz_fixed_len() -> usize {
@@ -183,6 +183,10 @@ where
         }
     }
 
+    fn ssz_bytes_len(&self) -> usize {
+        self.vec.ssz_bytes_len()
+    }
+
     fn ssz_append(&self, buf: &mut Vec<u8>) {
         if T::is_ssz_fixed_len() {
             buf.reserve(T::ssz_fixed_len() * self.len());
@@ -220,13 +224,26 @@ where
 
     fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, ssz::DecodeError> {
         if bytes.is_empty() {
-            Ok(FixedVector::from(vec![]))
+            Err(ssz::DecodeError::InvalidByteLength {
+                len: 0,
+                expected: 1,
+            })
         } else if T::is_ssz_fixed_len() {
             bytes
                 .chunks(T::ssz_fixed_len())
                 .map(|chunk| T::from_ssz_bytes(chunk))
                 .collect::<Result<Vec<T>, _>>()
-                .and_then(|vec| Ok(vec.into()))
+                .and_then(|vec| {
+                    if vec.len() == N::to_usize() {
+                        Ok(vec.into())
+                    } else {
+                        Err(ssz::DecodeError::BytesInvalid(format!(
+                            "wrong number of vec elements, got: {}, expected: {}",
+                            vec.len(),
+                            N::to_usize()
+                        )))
+                    }
+                })
         } else {
             ssz::decode_list_of_variable_length_items(bytes).and_then(|vec| Ok(vec.into()))
         }
@@ -305,6 +322,7 @@ mod test {
 
     fn ssz_round_trip<T: Encode + Decode + std::fmt::Debug + PartialEq>(item: T) {
         let encoded = &item.as_ssz_bytes();
+        assert_eq!(item.ssz_bytes_len(), encoded.len());
         assert_eq!(T::from_ssz_bytes(&encoded), Ok(item));
     }
 

eth2/utils/ssz_types/src/variable_list.rs

@@ -208,6 +208,10 @@ where
         <Vec<T>>::ssz_fixed_len()
     }
 
+    fn ssz_bytes_len(&self) -> usize {
+        self.vec.ssz_bytes_len()
+    }
+
     fn ssz_append(&self, buf: &mut Vec<u8>) {
         self.vec.ssz_append(buf)
     }
@@ -304,6 +308,7 @@ mod test {
 
     fn round_trip<T: Encode + Decode + std::fmt::Debug + PartialEq>(item: T) {
         let encoded = &item.as_ssz_bytes();
+        assert_eq!(item.ssz_bytes_len(), encoded.len());
         assert_eq!(T::from_ssz_bytes(&encoded), Ok(item));
     }
 

eth2/utils/tree_hash/src/impls.rs

@@ -1,5 +1,5 @@
 use super::*;
-use ethereum_types::H256;
+use ethereum_types::{H256, U128, U256};
 
 macro_rules! impl_for_bitsize {
     ($type: ident, $bit_size: expr) => {
@@ -73,6 +73,46 @@ macro_rules! impl_for_u8_array {
 impl_for_u8_array!(4);
 impl_for_u8_array!(32);
 
+impl TreeHash for U128 {
+    fn tree_hash_type() -> TreeHashType {
+        TreeHashType::Basic
+    }
+
+    fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+        let mut result = vec![0; 16];
+        self.to_little_endian(&mut result);
+        result
+    }
+
+    fn tree_hash_packing_factor() -> usize {
+        2
+    }
+
+    fn tree_hash_root(&self) -> Vec<u8> {
+        merkle_root(&self.tree_hash_packed_encoding(), 0)
+    }
+}
+
+impl TreeHash for U256 {
+    fn tree_hash_type() -> TreeHashType {
+        TreeHashType::Basic
+    }
+
+    fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+        let mut result = vec![0; 32];
+        self.to_little_endian(&mut result);
+        result
+    }
+
+    fn tree_hash_packing_factor() -> usize {
+        1
+    }
+
+    fn tree_hash_root(&self) -> Vec<u8> {
+        merkle_root(&self.tree_hash_packed_encoding(), 0)
+    }
+}
+
 impl TreeHash for H256 {
     fn tree_hash_type() -> TreeHashType {
         TreeHashType::Vector

tests/ef_tests/.gitignore

@@ -0,0 +1 @@
+/eth2.0-spec-tests

tests/ef_tests/Cargo.toml

@@ -18,7 +18,9 @@ serde_derive = "1.0"
 serde_repr = "0.1"
 serde_yaml = "0.8"
 eth2_ssz = "0.1"
+eth2_ssz_derive = "0.1"
 tree_hash = "0.1"
+tree_hash_derive = "0.2"
 state_processing = { path = "../../eth2/state_processing" }
 swap_or_not_shuffle = { path = "../../eth2/utils/swap_or_not_shuffle" }
 types = { path = "../../eth2/types" }

tests/ef_tests/eth2.0-spec-tests

@@ -1 +0,0 @@
-Subproject commit aaa1673f508103e11304833e0456e4149f880065

tests/ef_tests/src/bls_setting.rs

@@ -2,7 +2,6 @@ use self::BlsSetting::*;
 use crate::error::Error;
 use serde_repr::Deserialize_repr;
 
-// TODO: use this in every test case
 #[derive(Deserialize_repr, Debug, Clone, Copy)]
 #[repr(u8)]
 pub enum BlsSetting {

tests/ef_tests/src/case_result.rs

@@ -1,6 +1,7 @@
 use super::*;
 use compare_fields::{CompareFields, Comparison, FieldComparison};
 use std::fmt::Debug;
+use std::path::{Path, PathBuf};
 use types::BeaconState;
 
 pub const MAX_VALUE_STRING_LEN: usize = 500;
@@ -9,14 +10,21 @@ pub const MAX_VALUE_STRING_LEN: usize = 500;
 pub struct CaseResult {
     pub case_index: usize,
     pub desc: String,
+    pub path: PathBuf,
     pub result: Result<(), Error>,
 }
 
 impl CaseResult {
-    pub fn new(case_index: usize, case: &impl Case, result: Result<(), Error>) -> Self {
+    pub fn new(
+        case_index: usize,
+        path: &Path,
+        case: &impl Case,
+        result: Result<(), Error>,
+    ) -> Self {
         CaseResult {
             case_index,
             desc: case.description(),
+            path: path.into(),
             result,
         }
     }

tests/ef_tests/src/cases.rs

@@ -1,5 +1,7 @@
 use super::*;
+use rayon::prelude::*;
 use std::fmt::Debug;
+use std::path::{Path, PathBuf};
 
 mod bls_aggregate_pubkeys;
 mod bls_aggregate_sigs;
@@ -7,20 +9,11 @@ mod bls_g2_compressed;
 mod bls_g2_uncompressed;
 mod bls_priv_to_pub;
 mod bls_sign_msg;
-mod epoch_processing_crosslinks;
-mod epoch_processing_final_updates;
-mod epoch_processing_justification_and_finalization;
-mod epoch_processing_registry_updates;
-mod epoch_processing_slashings;
+mod common;
+mod epoch_processing;
 mod genesis_initialization;
 mod genesis_validity;
-mod operations_attestation;
-mod operations_attester_slashing;
-mod operations_block_header;
-mod operations_deposit;
-mod operations_exit;
-mod operations_proposer_slashing;
-mod operations_transfer;
+mod operations;
 mod sanity_blocks;
 mod sanity_slots;
 mod shuffling;
@@ -33,27 +26,23 @@ pub use bls_g2_compressed::*;
 pub use bls_g2_uncompressed::*;
 pub use bls_priv_to_pub::*;
 pub use bls_sign_msg::*;
-pub use epoch_processing_crosslinks::*;
-pub use epoch_processing_final_updates::*;
-pub use epoch_processing_justification_and_finalization::*;
-pub use epoch_processing_registry_updates::*;
-pub use epoch_processing_slashings::*;
+pub use common::SszStaticType;
+pub use epoch_processing::*;
 pub use genesis_initialization::*;
 pub use genesis_validity::*;
-pub use operations_attestation::*;
-pub use operations_attester_slashing::*;
-pub use operations_block_header::*;
-pub use operations_deposit::*;
-pub use operations_exit::*;
-pub use operations_proposer_slashing::*;
-pub use operations_transfer::*;
+pub use operations::*;
 pub use sanity_blocks::*;
 pub use sanity_slots::*;
 pub use shuffling::*;
 pub use ssz_generic::*;
 pub use ssz_static::*;
 
-pub trait Case: Debug {
+pub trait LoadCase: Sized {
+    /// Load the test case from a test case directory.
+    fn load_from_dir(_path: &Path) -> Result<Self, Error>;
+}
+
+pub trait Case: Debug + Sync {
     /// An optional field for implementing a custom description.
     ///
     /// Defaults to "no description".
@@ -70,51 +59,15 @@ pub trait Case: Debug {
 
 #[derive(Debug)]
 pub struct Cases<T> {
-    pub test_cases: Vec<T>,
+    pub test_cases: Vec<(PathBuf, T)>,
 }
 
-impl<T> EfTest for Cases<T>
-where
-    T: Case + Debug,
-{
-    fn test_results(&self) -> Vec<CaseResult> {
+impl<T: Case> Cases<T> {
+    pub fn test_results(&self) -> Vec<CaseResult> {
         self.test_cases
-            .iter()
+            .into_par_iter()
             .enumerate()
-            .map(|(i, tc)| CaseResult::new(i, tc, tc.result(i)))
+            .map(|(i, (ref path, ref tc))| CaseResult::new(i, path, tc, tc.result(i)))
             .collect()
     }
 }
-
-impl<T: YamlDecode> YamlDecode for Cases<T> {
-    /// Decodes a YAML list of test cases
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        let mut p = 0;
-        let mut elems: Vec<&str> = yaml
-            .match_indices("\n- ")
-            // Skip the `\n` used for matching a new line
-            .map(|(i, _)| i + 1)
-            .map(|i| {
-                let yaml_element = &yaml[p..i];
-                p = i;
-
-                yaml_element
-            })
-            .collect();
-
-        elems.push(&yaml[p..]);
-
-        let test_cases = elems
-            .iter()
-            .map(|s| {
-                // Remove the `- ` prefix.
-                let s = &s[2..];
-                // Remove a single level of indenting.
-                s.replace("\n  ", "\n")
-            })
-            .map(|s| T::yaml_decode(&s.to_string()).unwrap())
-            .collect();
-
-        Ok(Self { test_cases })
-    }
-}

tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs

@@ -1,5 +1,6 @@
 use super::*;
 use crate::case_result::compare_result;
+use crate::cases::common::BlsCase;
 use bls::{AggregatePublicKey, PublicKey};
 use serde_derive::Deserialize;
 
@@ -9,11 +10,7 @@ pub struct BlsAggregatePubkeys {
     pub output: String,
 }
 
-impl YamlDecode for BlsAggregatePubkeys {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(yaml).unwrap())
-    }
-}
+impl BlsCase for BlsAggregatePubkeys {}
 
 impl Case for BlsAggregatePubkeys {
     fn result(&self, _case_index: usize) -> Result<(), Error> {

tests/ef_tests/src/cases/bls_aggregate_sigs.rs

@@ -1,5 +1,6 @@
 use super::*;
 use crate::case_result::compare_result;
+use crate::cases::common::BlsCase;
 use bls::{AggregateSignature, Signature};
 use serde_derive::Deserialize;
 
@@ -9,11 +10,7 @@ pub struct BlsAggregateSigs {
     pub output: String,
 }
 
-impl YamlDecode for BlsAggregateSigs {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(yaml).unwrap())
-    }
-}
+impl BlsCase for BlsAggregateSigs {}
 
 impl Case for BlsAggregateSigs {
     fn result(&self, _case_index: usize) -> Result<(), Error> {

tests/ef_tests/src/cases/bls_g2_compressed.rs

@@ -1,5 +1,6 @@
 use super::*;
 use crate::case_result::compare_result;
+use crate::cases::common::BlsCase;
 use bls::{compress_g2, hash_on_g2};
 use serde_derive::Deserialize;
 
@@ -15,11 +16,7 @@ pub struct BlsG2Compressed {
     pub output: Vec<String>,
 }
 
-impl YamlDecode for BlsG2Compressed {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(yaml).unwrap())
-    }
-}
+impl BlsCase for BlsG2Compressed {}
 
 impl Case for BlsG2Compressed {
     fn result(&self, _case_index: usize) -> Result<(), Error> {
@@ -45,14 +42,9 @@ impl Case for BlsG2Compressed {
     }
 }
 
-// Converts a vector to u64 (from big endian)
+// Converts a vector to u64 (from little endian)
 fn bytes_to_u64(array: &[u8]) -> u64 {
-    let mut result: u64 = 0;
-    for (i, value) in array.iter().rev().enumerate() {
-        if i == 8 {
-            break;
-        }
-        result += u64::pow(2, i as u32 * 8) * u64::from(*value);
-    }
-    result
+    let mut bytes = [0u8; 8];
+    bytes.copy_from_slice(array);
+    u64::from_le_bytes(bytes)
 }

tests/ef_tests/src/cases/bls_g2_uncompressed.rs

@@ -1,5 +1,6 @@
 use super::*;
 use crate::case_result::compare_result;
+use crate::cases::common::BlsCase;
 use bls::hash_on_g2;
 use serde_derive::Deserialize;
 
@@ -9,18 +10,14 @@ pub struct BlsG2UncompressedInput {
     pub domain: String,
 }
 
+impl BlsCase for BlsG2UncompressedInput {}
+
 #[derive(Debug, Clone, Deserialize)]
 pub struct BlsG2Uncompressed {
     pub input: BlsG2UncompressedInput,
     pub output: Vec<Vec<String>>,
 }
 
-impl YamlDecode for BlsG2Uncompressed {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(yaml).unwrap())
-    }
-}
-
 impl Case for BlsG2Uncompressed {
     fn result(&self, _case_index: usize) -> Result<(), Error> {
         // Convert message and domain to required types

tests/ef_tests/src/cases/bls_priv_to_pub.rs

@@ -1,5 +1,6 @@
 use super::*;
 use crate::case_result::compare_result;
+use crate::cases::common::BlsCase;
 use bls::{PublicKey, SecretKey};
 use serde_derive::Deserialize;
 
@@ -9,11 +10,7 @@ pub struct BlsPrivToPub {
     pub output: String,
 }
 
-impl YamlDecode for BlsPrivToPub {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(yaml).unwrap())
-    }
-}
+impl BlsCase for BlsPrivToPub {}
 
 impl Case for BlsPrivToPub {
     fn result(&self, _case_index: usize) -> Result<(), Error> {

tests/ef_tests/src/cases/bls_sign_msg.rs

@@ -1,5 +1,6 @@
 use super::*;
 use crate::case_result::compare_result;
+use crate::cases::common::BlsCase;
 use bls::{SecretKey, Signature};
 use serde_derive::Deserialize;
 
@@ -16,11 +17,7 @@ pub struct BlsSign {
     pub output: String,
 }
 
-impl YamlDecode for BlsSign {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(yaml).unwrap())
-    }
-}
+impl BlsCase for BlsSign {}
 
 impl Case for BlsSign {
     fn result(&self, _case_index: usize) -> Result<(), Error> {
@@ -45,16 +42,11 @@ impl Case for BlsSign {
     }
 }
 
-// Converts a vector to u64 (from big endian)
+// Converts a vector to u64 (from little endian)
 fn bytes_to_u64(array: &[u8]) -> u64 {
-    let mut result: u64 = 0;
-    for (i, value) in array.iter().rev().enumerate() {
-        if i == 8 {
-            break;
-        }
-        result += u64::pow(2, i as u32 * 8) * u64::from(*value);
-    }
-    result
+    let mut bytes = [0u8; 8];
+    bytes.copy_from_slice(array);
+    u64::from_le_bytes(bytes)
 }
 
 // Increase the size of an array to 48 bytes

tests/ef_tests/src/cases/common.rs

@@ -0,0 +1,72 @@
+use crate::cases::LoadCase;
+use crate::decode::yaml_decode_file;
+use crate::error::Error;
+use serde_derive::Deserialize;
+use ssz::{Decode, Encode};
+use ssz_derive::{Decode, Encode};
+use std::convert::TryFrom;
+use std::fmt::Debug;
+use std::path::Path;
+use tree_hash::TreeHash;
+
+/// Trait for all BLS cases to eliminate some boilerplate.
+pub trait BlsCase: serde::de::DeserializeOwned {}
+
+impl<T: BlsCase> LoadCase for T {
+    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+        yaml_decode_file(&path.join("data.yaml"))
+    }
+}
+
+/// Macro to wrap U128 and U256 so they deserialize correctly.
+macro_rules! uint_wrapper {
+    ($wrapper_name:ident, $wrapped_type:ty) => {
+        #[derive(Debug, Clone, Copy, Default, PartialEq, Decode, Encode, Deserialize)]
+        #[serde(try_from = "String")]
+        pub struct $wrapper_name {
+            pub x: $wrapped_type,
+        }
+
+        impl TryFrom<String> for $wrapper_name {
+            type Error = String;
+
+            fn try_from(s: String) -> Result<Self, Self::Error> {
+                <$wrapped_type>::from_dec_str(&s)
+                    .map(|x| Self { x })
+                    .map_err(|e| format!("{:?}", e))
+            }
+        }
+
+        impl tree_hash::TreeHash for $wrapper_name {
+            fn tree_hash_type() -> tree_hash::TreeHashType {
+                <$wrapped_type>::tree_hash_type()
+            }
+
+            fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+                self.x.tree_hash_packed_encoding()
+            }
+
+            fn tree_hash_packing_factor() -> usize {
+                <$wrapped_type>::tree_hash_packing_factor()
+            }
+
+            fn tree_hash_root(&self) -> Vec<u8> {
+                self.x.tree_hash_root()
+            }
+        }
+    };
+}
+
+uint_wrapper!(TestU128, ethereum_types::U128);
+uint_wrapper!(TestU256, ethereum_types::U256);
+
+/// Trait alias for all deez bounds
+pub trait SszStaticType:
+    serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + Sync
+{
+}
+
+impl<T> SszStaticType for T where
+    T: serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + Sync
+{
+}

tests/ef_tests/src/cases/epoch_processing.rs

@@ -0,0 +1,143 @@
+use super::*;
+use crate::bls_setting::BlsSetting;
+use crate::case_result::compare_beacon_state_results_without_caches;
+use crate::decode::{ssz_decode_file, yaml_decode_file};
+use crate::type_name;
+use crate::type_name::TypeName;
+use serde_derive::Deserialize;
+use state_processing::per_epoch_processing::{
+    errors::EpochProcessingError, process_crosslinks, process_final_updates,
+    process_justification_and_finalization, process_registry_updates, process_slashings,
+    validator_statuses::ValidatorStatuses,
+};
+use std::marker::PhantomData;
+use std::path::{Path, PathBuf};
+use types::{BeaconState, ChainSpec, EthSpec};
+
+#[derive(Debug, Clone, Default, Deserialize)]
+pub struct Metadata {
+    pub description: Option<String>,
+    pub bls_setting: Option<BlsSetting>,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(bound = "E: EthSpec")]
+pub struct EpochProcessing<E: EthSpec, T: EpochTransition<E>> {
+    pub path: PathBuf,
+    pub metadata: Metadata,
+    pub pre: BeaconState<E>,
+    pub post: Option<BeaconState<E>>,
+    #[serde(skip_deserializing)]
+    _phantom: PhantomData<T>,
+}
+
+pub trait EpochTransition<E: EthSpec>: TypeName + Debug + Sync {
+    fn run(state: &mut BeaconState<E>, spec: &ChainSpec) -> Result<(), EpochProcessingError>;
+}
+
+#[derive(Debug)]
+pub struct JustificationAndFinalization;
+#[derive(Debug)]
+pub struct Crosslinks;
+#[derive(Debug)]
+pub struct RegistryUpdates;
+#[derive(Debug)]
+pub struct Slashings;
+#[derive(Debug)]
+pub struct FinalUpdates;
+
+type_name!(
+    JustificationAndFinalization,
+    "justification_and_finalization"
+);
+type_name!(Crosslinks, "crosslinks");
+type_name!(RegistryUpdates, "registry_updates");
+type_name!(Slashings, "slashings");
+type_name!(FinalUpdates, "final_updates");
+
+impl<E: EthSpec> EpochTransition<E> for JustificationAndFinalization {
+    fn run(state: &mut BeaconState<E>, spec: &ChainSpec) -> Result<(), EpochProcessingError> {
+        let mut validator_statuses = ValidatorStatuses::new(state, spec)?;
+        validator_statuses.process_attestations(state, spec)?;
+        process_justification_and_finalization(state, &validator_statuses.total_balances)
+    }
+}
+
+impl<E: EthSpec> EpochTransition<E> for Crosslinks {
+    fn run(state: &mut BeaconState<E>, spec: &ChainSpec) -> Result<(), EpochProcessingError> {
+        process_crosslinks(state, spec)?;
+        Ok(())
+    }
+}
+
+impl<E: EthSpec> EpochTransition<E> for RegistryUpdates {
+    fn run(state: &mut BeaconState<E>, spec: &ChainSpec) -> Result<(), EpochProcessingError> {
+        process_registry_updates(state, spec)
+    }
+}
+
+impl<E: EthSpec> EpochTransition<E> for Slashings {
+    fn run(state: &mut BeaconState<E>, spec: &ChainSpec) -> Result<(), EpochProcessingError> {
+        let mut validator_statuses = ValidatorStatuses::new(&state, spec)?;
+        validator_statuses.process_attestations(&state, spec)?;
+        process_slashings(state, validator_statuses.total_balances.current_epoch, spec)?;
+        Ok(())
+    }
+}
+
+impl<E: EthSpec> EpochTransition<E> for FinalUpdates {
+    fn run(state: &mut BeaconState<E>, spec: &ChainSpec) -> Result<(), EpochProcessingError> {
+        process_final_updates(state, spec)
+    }
+}
+
+impl<E: EthSpec, T: EpochTransition<E>> LoadCase for EpochProcessing<E, T> {
+    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+        let metadata_path = path.join("meta.yaml");
+        let metadata: Metadata = if metadata_path.is_file() {
+            yaml_decode_file(&metadata_path)?
+        } else {
+            Metadata::default()
+        };
+        let pre = ssz_decode_file(&path.join("pre.ssz"))?;
+        let post_file = path.join("post.ssz");
+        let post = if post_file.is_file() {
+            Some(ssz_decode_file(&post_file)?)
+        } else {
+            None
+        };
+
+        Ok(Self {
+            path: path.into(),
+            metadata,
+            pre,
+            post,
+            _phantom: PhantomData,
+        })
+    }
+}
+
+impl<E: EthSpec, T: EpochTransition<E>> Case for EpochProcessing<E, T> {
+    fn description(&self) -> String {
+        self.metadata
+            .description
+            .clone()
+            .unwrap_or_else(String::new)
+    }
+
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        let mut state = self.pre.clone();
+        let mut expected = self.post.clone();
+
+        let spec = &E::default_spec();
+
+        let mut result = (|| {
+            // Processing requires the epoch cache.
+            state.build_all_caches(spec)?;
+
+            T::run(&mut state, spec).map(|_| state)
+        })();
+
+        compare_beacon_state_results_without_caches(&mut result, &mut expected)
+    }
+}

tests/ef_tests/src/cases/epoch_processing_crosslinks.rs

@@ -1,37 +0,0 @@
-use super::*;
-use crate::case_result::compare_beacon_state_results_without_caches;
-use serde_derive::Deserialize;
-use state_processing::per_epoch_processing::process_crosslinks;
-use types::{BeaconState, EthSpec};
-
-#[derive(Debug, Clone, Deserialize)]
-#[serde(bound = "E: EthSpec")]
-pub struct EpochProcessingCrosslinks<E: EthSpec> {
-    pub description: String,
-    pub pre: BeaconState<E>,
-    pub post: Option<BeaconState<E>>,
-}
-
-impl<E: EthSpec> YamlDecode for EpochProcessingCrosslinks<E> {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(yaml).unwrap())
-    }
-}
-
-impl<E: EthSpec> Case for EpochProcessingCrosslinks<E> {
-    fn description(&self) -> String {
-        self.description.clone()
-    }
-
-    fn result(&self, _case_index: usize) -> Result<(), Error> {
-        let mut state = self.pre.clone();
-        let mut expected = self.post.clone();
-
-        // Processing requires the epoch cache.
-        state.build_all_caches(&E::default_spec()).unwrap();
-
-        let mut result = process_crosslinks(&mut state, &E::default_spec()).map(|_| state);
-
-        compare_beacon_state_results_without_caches(&mut result, &mut expected)
-    }
-}

tests/ef_tests/src/cases/epoch_processing_final_updates.rs

@@ -1,41 +0,0 @@
-use super::*;
-use crate::case_result::compare_beacon_state_results_without_caches;
-use serde_derive::Deserialize;
-use state_processing::per_epoch_processing::process_final_updates;
-use types::{BeaconState, EthSpec};
-
-#[derive(Debug, Clone, Deserialize)]
-#[serde(bound = "E: EthSpec")]
-pub struct EpochProcessingFinalUpdates<E: EthSpec> {
-    pub description: String,
-    pub pre: BeaconState<E>,
-    pub post: Option<BeaconState<E>>,
-}
-
-impl<E: EthSpec> YamlDecode for EpochProcessingFinalUpdates<E> {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(yaml).unwrap())
-    }
-}
-
-impl<E: EthSpec> Case for EpochProcessingFinalUpdates<E> {
-    fn description(&self) -> String {
-        self.description.clone()
-    }
-
-    fn result(&self, _case_index: usize) -> Result<(), Error> {
-        let mut state = self.pre.clone();
-        let mut expected = self.post.clone();
-
-        let spec = &E::default_spec();
-
-        let mut result = (|| {
-            // Processing requires the epoch cache.
-            state.build_all_caches(spec)?;
-
-            process_final_updates(&mut state, spec).map(|_| state)
-        })();
-
-        compare_beacon_state_results_without_caches(&mut result, &mut expected)
-    }
-}

tests/ef_tests/src/cases/epoch_processing_justification_and_finalization.rs

@@ -1,46 +0,0 @@
-use super::*;
-use crate::case_result::compare_beacon_state_results_without_caches;
-use serde_derive::Deserialize;
-use state_processing::per_epoch_processing::{
-    process_justification_and_finalization, validator_statuses::ValidatorStatuses,
-};
-use types::{BeaconState, EthSpec};
-
-#[derive(Debug, Clone, Deserialize)]
-#[serde(bound = "E: EthSpec")]
-pub struct EpochProcessingJustificationAndFinalization<E: EthSpec> {
-    pub description: String,
-    pub pre: BeaconState<E>,
-    pub post: Option<BeaconState<E>>,
-}
-
-impl<E: EthSpec> YamlDecode for EpochProcessingJustificationAndFinalization<E> {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(yaml).unwrap())
-    }
-}
-
-impl<E: EthSpec> Case for EpochProcessingJustificationAndFinalization<E> {
-    fn description(&self) -> String {
-        self.description.clone()
-    }
-
-    fn result(&self, _case_index: usize) -> Result<(), Error> {
-        let mut state = self.pre.clone();
-        let mut expected = self.post.clone();
-
-        let spec = &E::default_spec();
-
-        // Processing requires the epoch cache.
-        state.build_all_caches(spec).unwrap();
-
-        let mut result = (|| {
-            let mut validator_statuses = ValidatorStatuses::new(&state, spec)?;
-            validator_statuses.process_attestations(&state, spec)?;
-            process_justification_and_finalization(&mut state, &validator_statuses.total_balances)
-                .map(|_| state)
-        })();
-
-        compare_beacon_state_results_without_caches(&mut result, &mut expected)
-    }
-}

tests/ef_tests/src/cases/epoch_processing_registry_updates.rs

@@ -1,38 +0,0 @@
-use super::*;
-use crate::case_result::compare_beacon_state_results_without_caches;
-use serde_derive::Deserialize;
-use state_processing::per_epoch_processing::registry_updates::process_registry_updates;
-use types::{BeaconState, EthSpec};
-
-#[derive(Debug, Clone, Deserialize)]
-#[serde(bound = "E: EthSpec")]
-pub struct EpochProcessingRegistryUpdates<E: EthSpec> {
-    pub description: String,
-    pub pre: BeaconState<E>,
-    pub post: Option<BeaconState<E>>,
-}
-
-impl<E: EthSpec> YamlDecode for EpochProcessingRegistryUpdates<E> {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(yaml).unwrap())
-    }
-}
-
-impl<E: EthSpec> Case for EpochProcessingRegistryUpdates<E> {
-    fn description(&self) -> String {
-        self.description.clone()
-    }
-
-    fn result(&self, _case_index: usize) -> Result<(), Error> {
-        let mut state = self.pre.clone();
-        let mut expected = self.post.clone();
-        let spec = &E::default_spec();
-
-        // Processing requires the epoch cache.
-        state.build_all_caches(spec).unwrap();
-
-        let mut result = process_registry_updates(&mut state, spec).map(|_| state);
-
-        compare_beacon_state_results_without_caches(&mut result, &mut expected)
-    }
-}

tests/ef_tests/src/cases/epoch_processing_slashings.rs

@@ -1,50 +0,0 @@
-use super::*;
-use crate::case_result::compare_beacon_state_results_without_caches;
-use serde_derive::Deserialize;
-use state_processing::per_epoch_processing::{
-    process_slashings::process_slashings, validator_statuses::ValidatorStatuses,
-};
-use types::{BeaconState, EthSpec};
-
-#[derive(Debug, Clone, Deserialize)]
-#[serde(bound = "E: EthSpec")]
-pub struct EpochProcessingSlashings<E: EthSpec> {
-    pub description: String,
-    pub pre: BeaconState<E>,
-    pub post: Option<BeaconState<E>>,
-}
-
-impl<E: EthSpec> YamlDecode for EpochProcessingSlashings<E> {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(yaml).unwrap())
-    }
-}
-
-impl<E: EthSpec> Case for EpochProcessingSlashings<E> {
-    fn description(&self) -> String {
-        self.description.clone()
-    }
-
-    fn result(&self, _case_index: usize) -> Result<(), Error> {
-        let mut state = self.pre.clone();
-        let mut expected = self.post.clone();
-
-        let spec = &E::default_spec();
-
-        let mut result = (|| {
-            // Processing requires the epoch cache.
-            state.build_all_caches(spec)?;
-
-            let mut validator_statuses = ValidatorStatuses::new(&state, spec)?;
-            validator_statuses.process_attestations(&state, spec)?;
-            process_slashings(
-                &mut state,
-                validator_statuses.total_balances.current_epoch,
-                spec,
-            )
-            .map(|_| state)
-        })();
-
-        compare_beacon_state_results_without_caches(&mut result, &mut expected)
-    }
-}

tests/ef_tests/src/cases/genesis_initialization.rs

@@ -1,34 +1,51 @@
 use super::*;
-use crate::bls_setting::BlsSetting;
 use crate::case_result::compare_beacon_state_results_without_caches;
+use crate::decode::{ssz_decode_file, yaml_decode_file};
 use serde_derive::Deserialize;
 use state_processing::initialize_beacon_state_from_eth1;
+use std::path::PathBuf;
 use types::{BeaconState, Deposit, EthSpec, Hash256};
 
+#[derive(Debug, Clone, Deserialize)]
+struct Metadata {
+    deposits_count: usize,
+}
+
 #[derive(Debug, Clone, Deserialize)]
 #[serde(bound = "E: EthSpec")]
 pub struct GenesisInitialization<E: EthSpec> {
-    pub description: String,
-    pub bls_setting: Option<BlsSetting>,
+    pub path: PathBuf,
     pub eth1_block_hash: Hash256,
     pub eth1_timestamp: u64,
     pub deposits: Vec<Deposit>,
     pub state: Option<BeaconState<E>>,
 }
 
-impl<E: EthSpec> YamlDecode for GenesisInitialization<E> {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(yaml).unwrap())
+impl<E: EthSpec> LoadCase for GenesisInitialization<E> {
+    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+        let eth1_block_hash = ssz_decode_file(&path.join("eth1_block_hash.ssz"))?;
+        let eth1_timestamp = yaml_decode_file(&path.join("eth1_timestamp.yaml"))?;
+        let meta: Metadata = yaml_decode_file(&path.join("meta.yaml"))?;
+        let deposits: Vec<Deposit> = (0..meta.deposits_count)
+            .map(|i| {
+                let filename = format!("deposits_{}.ssz", i);
+                ssz_decode_file(&path.join(filename))
+            })
+            .collect::<Result<_, _>>()?;
+        let state = ssz_decode_file(&path.join("state.ssz"))?;
+
+        Ok(Self {
+            path: path.into(),
+            eth1_block_hash,
+            eth1_timestamp,
+            deposits,
+            state: Some(state),
+        })
     }
 }
 
 impl<E: EthSpec> Case for GenesisInitialization<E> {
-    fn description(&self) -> String {
-        self.description.clone()
-    }
-
     fn result(&self, _case_index: usize) -> Result<(), Error> {
-        self.bls_setting.unwrap_or_default().check()?;
         let spec = &E::default_spec();
 
         let mut result = initialize_beacon_state_from_eth1(

tests/ef_tests/src/cases/genesis_validity.rs

@@ -1,31 +1,28 @@
 use super::*;
-use crate::bls_setting::BlsSetting;
+use crate::decode::{ssz_decode_file, yaml_decode_file};
 use serde_derive::Deserialize;
 use state_processing::is_valid_genesis_state;
+use std::path::Path;
 use types::{BeaconState, EthSpec};
 
 #[derive(Debug, Clone, Deserialize)]
 #[serde(bound = "E: EthSpec")]
 pub struct GenesisValidity<E: EthSpec> {
-    pub description: String,
-    pub bls_setting: Option<BlsSetting>,
     pub genesis: BeaconState<E>,
     pub is_valid: bool,
 }
 
-impl<E: EthSpec> YamlDecode for GenesisValidity<E> {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(yaml).unwrap())
+impl<E: EthSpec> LoadCase for GenesisValidity<E> {
+    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+        let genesis = ssz_decode_file(&path.join("genesis.ssz"))?;
+        let is_valid = yaml_decode_file(&path.join("is_valid.yaml"))?;
+
+        Ok(Self { genesis, is_valid })
     }
 }
 
 impl<E: EthSpec> Case for GenesisValidity<E> {
-    fn description(&self) -> String {
-        self.description.clone()
-    }
-
     fn result(&self, _case_index: usize) -> Result<(), Error> {
-        self.bls_setting.unwrap_or_default().check()?;
         let spec = &E::default_spec();
 
         let is_valid = is_valid_genesis_state(&self.genesis, spec);

tests/ef_tests/src/cases/operations.rs

@@ -0,0 +1,194 @@
+use super::*;
+use crate::bls_setting::BlsSetting;
+use crate::case_result::compare_beacon_state_results_without_caches;
+use crate::decode::{ssz_decode_file, yaml_decode_file};
+use crate::type_name::TypeName;
+use serde_derive::Deserialize;
+use ssz::Decode;
+use state_processing::per_block_processing::{
+    errors::BlockProcessingError, process_attestations, process_attester_slashings,
+    process_block_header, process_deposits, process_exits, process_proposer_slashings,
+    process_transfers, VerifySignatures,
+};
+use std::fmt::Debug;
+use std::path::Path;
+use types::{
+    Attestation, AttesterSlashing, BeaconBlock, BeaconState, ChainSpec, Deposit, EthSpec,
+    ProposerSlashing, Transfer, VoluntaryExit,
+};
+
+#[derive(Debug, Clone, Default, Deserialize)]
+struct Metadata {
+    description: Option<String>,
+    bls_setting: Option<BlsSetting>,
+}
+
+#[derive(Debug, Clone)]
+pub struct Operations<E: EthSpec, O: Operation<E>> {
+    metadata: Metadata,
+    pub pre: BeaconState<E>,
+    pub operation: O,
+    pub post: Option<BeaconState<E>>,
+}
+
+pub trait Operation<E: EthSpec>: Decode + TypeName + Debug + Sync {
+    fn handler_name() -> String {
+        Self::name().to_lowercase()
+    }
+
+    fn filename() -> String {
+        format!("{}.ssz", Self::handler_name())
+    }
+
+    fn apply_to(
+        &self,
+        state: &mut BeaconState<E>,
+        spec: &ChainSpec,
+    ) -> Result<(), BlockProcessingError>;
+}
+
+impl<E: EthSpec> Operation<E> for Attestation<E> {
+    fn apply_to(
+        &self,
+        state: &mut BeaconState<E>,
+        spec: &ChainSpec,
+    ) -> Result<(), BlockProcessingError> {
+        process_attestations(state, &[self.clone()], VerifySignatures::True, spec)
+    }
+}
+
+impl<E: EthSpec> Operation<E> for AttesterSlashing<E> {
+    fn handler_name() -> String {
+        "attester_slashing".into()
+    }
+
+    fn apply_to(
+        &self,
+        state: &mut BeaconState<E>,
+        spec: &ChainSpec,
+    ) -> Result<(), BlockProcessingError> {
+        process_attester_slashings(state, &[self.clone()], VerifySignatures::True, spec)
+    }
+}
+
+impl<E: EthSpec> Operation<E> for Deposit {
+    fn apply_to(
+        &self,
+        state: &mut BeaconState<E>,
+        spec: &ChainSpec,
+    ) -> Result<(), BlockProcessingError> {
+        process_deposits(state, &[self.clone()], spec)
+    }
+}
+
+impl<E: EthSpec> Operation<E> for ProposerSlashing {
+    fn handler_name() -> String {
+        "proposer_slashing".into()
+    }
+
+    fn apply_to(
+        &self,
+        state: &mut BeaconState<E>,
+        spec: &ChainSpec,
+    ) -> Result<(), BlockProcessingError> {
+        process_proposer_slashings(state, &[self.clone()], VerifySignatures::True, spec)
+    }
+}
+
+impl<E: EthSpec> Operation<E> for Transfer {
+    fn apply_to(
+        &self,
+        state: &mut BeaconState<E>,
+        spec: &ChainSpec,
+    ) -> Result<(), BlockProcessingError> {
+        process_transfers(state, &[self.clone()], VerifySignatures::True, spec)
+    }
+}
+
+impl<E: EthSpec> Operation<E> for VoluntaryExit {
+    fn handler_name() -> String {
+        "voluntary_exit".into()
+    }
+
+    fn apply_to(
+        &self,
+        state: &mut BeaconState<E>,
+        spec: &ChainSpec,
+    ) -> Result<(), BlockProcessingError> {
+        process_exits(state, &[self.clone()], VerifySignatures::True, spec)
+    }
+}
+
+impl<E: EthSpec> Operation<E> for BeaconBlock<E> {
+    fn handler_name() -> String {
+        "block_header".into()
+    }
+
+    fn filename() -> String {
+        "block.ssz".into()
+    }
+
+    fn apply_to(
+        &self,
+        state: &mut BeaconState<E>,
+        spec: &ChainSpec,
+    ) -> Result<(), BlockProcessingError> {
+        Ok(process_block_header(
+            state,
+            self,
+            None,
+            VerifySignatures::True,
+            spec,
+        )?)
+    }
+}
+
+impl<E: EthSpec, O: Operation<E>> LoadCase for Operations<E, O> {
+    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+        let metadata_path = path.join("meta.yaml");
+        let metadata: Metadata = if metadata_path.is_file() {
+            yaml_decode_file(&metadata_path)?
+        } else {
+            Metadata::default()
+        };
+        let pre = ssz_decode_file(&path.join("pre.ssz"))?;
+        let operation = ssz_decode_file(&path.join(O::filename()))?;
+        let post_filename = path.join("post.ssz");
+        let post = if post_filename.is_file() {
+            Some(ssz_decode_file(&post_filename)?)
+        } else {
+            None
+        };
+
+        Ok(Self {
+            metadata,
+            pre,
+            operation,
+            post,
+        })
+    }
+}
+
+impl<E: EthSpec, O: Operation<E>> Case for Operations<E, O> {
+    fn description(&self) -> String {
+        self.metadata
+            .description
+            .clone()
+            .unwrap_or_else(String::new)
+    }
+
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        self.metadata.bls_setting.unwrap_or_default().check()?;
+
+        let spec = &E::default_spec();
+        let mut state = self.pre.clone();
+        let mut expected = self.post.clone();
+
+        // Processing requires the epoch cache.
+        state.build_all_caches(spec).unwrap();
+
+        let mut result = self.operation.apply_to(&mut state, spec).map(|()| state);
+
+        compare_beacon_state_results_without_caches(&mut result, &mut expected)
+    }
+}

tests/ef_tests/src/cases/operations_attestation.rs

@@ -1,47 +0,0 @@
-use super::*;
-use crate::bls_setting::BlsSetting;
-use crate::case_result::compare_beacon_state_results_without_caches;
-use serde_derive::Deserialize;
-use state_processing::per_block_processing::{process_attestations, VerifySignatures};
-use types::{Attestation, BeaconState, EthSpec};
-
-#[derive(Debug, Clone, Deserialize)]
-#[serde(bound = "E: EthSpec")]
-pub struct OperationsAttestation<E: EthSpec> {
-    pub description: String,
-    pub bls_setting: Option<BlsSetting>,
-    pub pre: BeaconState<E>,
-    pub attestation: Attestation<E>,
-    pub post: Option<BeaconState<E>>,
-}
-
-impl<E: EthSpec> YamlDecode for OperationsAttestation<E> {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(&yaml).unwrap())
-    }
-}
-
-impl<E: EthSpec> Case for OperationsAttestation<E> {
-    fn description(&self) -> String {
-        self.description.clone()
-    }
-
-    fn result(&self, _case_index: usize) -> Result<(), Error> {
-        let spec = &E::default_spec();
-
-        self.bls_setting.unwrap_or_default().check()?;
-
-        let mut state = self.pre.clone();
-        let attestation = self.attestation.clone();
-        let mut expected = self.post.clone();
-
-        // Processing requires the epoch cache.
-        state.build_all_caches(spec).unwrap();
-
-        let result = process_attestations(&mut state, &[attestation], VerifySignatures::True, spec);
-
-        let mut result = result.and_then(|_| Ok(state));
-
-        compare_beacon_state_results_without_caches(&mut result, &mut expected)
-    }
-}

tests/ef_tests/src/cases/operations_attester_slashing.rs

@@ -1,52 +0,0 @@
-use super::*;
-use crate::bls_setting::BlsSetting;
-use crate::case_result::compare_beacon_state_results_without_caches;
-use serde_derive::Deserialize;
-use state_processing::per_block_processing::{process_attester_slashings, VerifySignatures};
-use types::{AttesterSlashing, BeaconState, EthSpec};
-
-#[derive(Debug, Clone, Deserialize)]
-pub struct OperationsAttesterSlashing<E: EthSpec> {
-    pub description: String,
-    pub bls_setting: Option<BlsSetting>,
-    #[serde(bound = "E: EthSpec")]
-    pub pre: BeaconState<E>,
-    #[serde(bound = "E: EthSpec")]
-    pub attester_slashing: AttesterSlashing<E>,
-    #[serde(bound = "E: EthSpec")]
-    pub post: Option<BeaconState<E>>,
-}
-
-impl<E: EthSpec> YamlDecode for OperationsAttesterSlashing<E> {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(yaml).unwrap())
-    }
-}
-
-impl<E: EthSpec> Case for OperationsAttesterSlashing<E> {
-    fn description(&self) -> String {
-        self.description.clone()
-    }
-
-    fn result(&self, _case_index: usize) -> Result<(), Error> {
-        self.bls_setting.unwrap_or_default().check()?;
-
-        let mut state = self.pre.clone();
-        let attester_slashing = self.attester_slashing.clone();
-        let mut expected = self.post.clone();
-
-        // Processing requires the epoch cache.
-        state.build_all_caches(&E::default_spec()).unwrap();
-
-        let result = process_attester_slashings(
-            &mut state,
-            &[attester_slashing],
-            VerifySignatures::True,
-            &E::default_spec(),
-        );
-
-        let mut result = result.and_then(|_| Ok(state));
-
-        compare_beacon_state_results_without_caches(&mut result, &mut expected)
-    }
-}

tests/ef_tests/src/cases/operations_block_header.rs

@@ -1,46 +0,0 @@
-use super::*;
-use crate::bls_setting::BlsSetting;
-use crate::case_result::compare_beacon_state_results_without_caches;
-use serde_derive::Deserialize;
-use state_processing::per_block_processing::{process_block_header, VerifySignatures};
-use types::{BeaconBlock, BeaconState, EthSpec};
-
-#[derive(Debug, Clone, Deserialize)]
-#[serde(bound = "E: EthSpec")]
-pub struct OperationsBlockHeader<E: EthSpec> {
-    pub description: String,
-    pub bls_setting: Option<BlsSetting>,
-    pub pre: BeaconState<E>,
-    pub block: BeaconBlock<E>,
-    pub post: Option<BeaconState<E>>,
-}
-
-impl<E: EthSpec> YamlDecode for OperationsBlockHeader<E> {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(yaml).unwrap())
-    }
-}
-
-impl<E: EthSpec> Case for OperationsBlockHeader<E> {
-    fn description(&self) -> String {
-        self.description.clone()
-    }
-
-    fn result(&self, _case_index: usize) -> Result<(), Error> {
-        let spec = &E::default_spec();
-
-        self.bls_setting.unwrap_or_default().check()?;
-
-        let mut state = self.pre.clone();
-        let mut expected = self.post.clone();
-
-        // Processing requires the epoch cache.
-        state.build_all_caches(spec).unwrap();
-
-        let mut result =
-            process_block_header(&mut state, &self.block, None, VerifySignatures::True, spec)
-                .map(|_| state);
-
-        compare_beacon_state_results_without_caches(&mut result, &mut expected)
-    }
-}

tests/ef_tests/src/cases/operations_deposit.rs

@@ -1,42 +0,0 @@
-use super::*;
-use crate::bls_setting::BlsSetting;
-use crate::case_result::compare_beacon_state_results_without_caches;
-use serde_derive::Deserialize;
-use state_processing::per_block_processing::process_deposits;
-use types::{BeaconState, Deposit, EthSpec};
-
-#[derive(Debug, Clone, Deserialize)]
-#[serde(bound = "E: EthSpec")]
-pub struct OperationsDeposit<E: EthSpec> {
-    pub description: String,
-    pub bls_setting: Option<BlsSetting>,
-    pub pre: BeaconState<E>,
-    pub deposit: Deposit,
-    pub post: Option<BeaconState<E>>,
-}
-
-impl<E: EthSpec> YamlDecode for OperationsDeposit<E> {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(yaml).unwrap())
-    }
-}
-
-impl<E: EthSpec> Case for OperationsDeposit<E> {
-    fn description(&self) -> String {
-        self.description.clone()
-    }
-
-    fn result(&self, _case_index: usize) -> Result<(), Error> {
-        self.bls_setting.unwrap_or_default().check()?;
-
-        let mut state = self.pre.clone();
-        let deposit = self.deposit.clone();
-        let mut expected = self.post.clone();
-
-        let result = process_deposits(&mut state, &[deposit], &E::default_spec());
-
-        let mut result = result.and_then(|_| Ok(state));
-
-        compare_beacon_state_results_without_caches(&mut result, &mut expected)
-    }
-}

tests/ef_tests/src/cases/operations_exit.rs

@@ -1,50 +0,0 @@
-use super::*;
-use crate::bls_setting::BlsSetting;
-use crate::case_result::compare_beacon_state_results_without_caches;
-use serde_derive::Deserialize;
-use state_processing::per_block_processing::{process_exits, VerifySignatures};
-use types::{BeaconState, EthSpec, VoluntaryExit};
-
-#[derive(Debug, Clone, Deserialize)]
-#[serde(bound = "E: EthSpec")]
-pub struct OperationsExit<E: EthSpec> {
-    pub description: String,
-    pub bls_setting: Option<BlsSetting>,
-    pub pre: BeaconState<E>,
-    pub voluntary_exit: VoluntaryExit,
-    pub post: Option<BeaconState<E>>,
-}
-
-impl<E: EthSpec> YamlDecode for OperationsExit<E> {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(yaml).unwrap())
-    }
-}
-
-impl<E: EthSpec> Case for OperationsExit<E> {
-    fn description(&self) -> String {
-        self.description.clone()
-    }
-
-    fn result(&self, _case_index: usize) -> Result<(), Error> {
-        self.bls_setting.unwrap_or_default().check()?;
-
-        let mut state = self.pre.clone();
-        let exit = self.voluntary_exit.clone();
-        let mut expected = self.post.clone();
-
-        // Exit processing requires the epoch cache.
-        state.build_all_caches(&E::default_spec()).unwrap();
-
-        let result = process_exits(
-            &mut state,
-            &[exit],
-            VerifySignatures::True,
-            &E::default_spec(),
-        );
-
-        let mut result = result.and_then(|_| Ok(state));
-
-        compare_beacon_state_results_without_caches(&mut result, &mut expected)
-    }
-}

tests/ef_tests/src/cases/operations_proposer_slashing.rs

@@ -1,50 +0,0 @@
-use super::*;
-use crate::bls_setting::BlsSetting;
-use crate::case_result::compare_beacon_state_results_without_caches;
-use serde_derive::Deserialize;
-use state_processing::per_block_processing::{process_proposer_slashings, VerifySignatures};
-use types::{BeaconState, EthSpec, ProposerSlashing};
-
-#[derive(Debug, Clone, Deserialize)]
-#[serde(bound = "E: EthSpec")]
-pub struct OperationsProposerSlashing<E: EthSpec> {
-    pub description: String,
-    pub bls_setting: Option<BlsSetting>,
-    pub pre: BeaconState<E>,
-    pub proposer_slashing: ProposerSlashing,
-    pub post: Option<BeaconState<E>>,
-}
-
-impl<E: EthSpec> YamlDecode for OperationsProposerSlashing<E> {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(yaml).unwrap())
-    }
-}
-
-impl<E: EthSpec> Case for OperationsProposerSlashing<E> {
-    fn description(&self) -> String {
-        self.description.clone()
-    }
-
-    fn result(&self, _case_index: usize) -> Result<(), Error> {
-        self.bls_setting.unwrap_or_default().check()?;
-
-        let mut state = self.pre.clone();
-        let proposer_slashing = self.proposer_slashing.clone();
-        let mut expected = self.post.clone();
-
-        // Processing requires the epoch cache.
-        state.build_all_caches(&E::default_spec()).unwrap();
-
-        let result = process_proposer_slashings(
-            &mut state,
-            &[proposer_slashing],
-            VerifySignatures::True,
-            &E::default_spec(),
-        );
-
-        let mut result = result.and_then(|_| Ok(state));
-
-        compare_beacon_state_results_without_caches(&mut result, &mut expected)
-    }
-}

tests/ef_tests/src/cases/operations_transfer.rs

@@ -1,47 +0,0 @@
-use super::*;
-use crate::bls_setting::BlsSetting;
-use crate::case_result::compare_beacon_state_results_without_caches;
-use serde_derive::Deserialize;
-use state_processing::per_block_processing::{process_transfers, VerifySignatures};
-use types::{BeaconState, EthSpec, Transfer};
-
-#[derive(Debug, Clone, Deserialize)]
-#[serde(bound = "E: EthSpec")]
-pub struct OperationsTransfer<E: EthSpec> {
-    pub description: String,
-    pub bls_setting: Option<BlsSetting>,
-    pub pre: BeaconState<E>,
-    pub transfer: Transfer,
-    pub post: Option<BeaconState<E>>,
-}
-
-impl<E: EthSpec> YamlDecode for OperationsTransfer<E> {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(yaml).unwrap())
-    }
-}
-
-impl<E: EthSpec> Case for OperationsTransfer<E> {
-    fn description(&self) -> String {
-        self.description.clone()
-    }
-
-    fn result(&self, _case_index: usize) -> Result<(), Error> {
-        self.bls_setting.unwrap_or_default().check()?;
-
-        let mut state = self.pre.clone();
-        let transfer = self.transfer.clone();
-        let mut expected = self.post.clone();
-
-        // Transfer processing requires the epoch cache.
-        state.build_all_caches(&E::default_spec()).unwrap();
-
-        let spec = E::default_spec();
-
-        let result = process_transfers(&mut state, &[transfer], VerifySignatures::True, &spec);
-
-        let mut result = result.and_then(|_| Ok(state));
-
-        compare_beacon_state_results_without_caches(&mut result, &mut expected)
-    }
-}

tests/ef_tests/src/cases/sanity_blocks.rs

@@ -1,35 +1,65 @@
 use super::*;
 use crate::bls_setting::BlsSetting;
 use crate::case_result::compare_beacon_state_results_without_caches;
+use crate::decode::{ssz_decode_file, yaml_decode_file};
 use serde_derive::Deserialize;
 use state_processing::{
     per_block_processing, per_slot_processing, BlockProcessingError, BlockSignatureStrategy,
 };
 use types::{BeaconBlock, BeaconState, EthSpec, RelativeEpoch};
 
+#[derive(Debug, Clone, Deserialize)]
+pub struct Metadata {
+    pub description: Option<String>,
+    pub bls_setting: Option<BlsSetting>,
+    pub blocks_count: usize,
+}
+
 #[derive(Debug, Clone, Deserialize)]
 #[serde(bound = "E: EthSpec")]
 pub struct SanityBlocks<E: EthSpec> {
-    pub description: String,
-    pub bls_setting: Option<BlsSetting>,
+    pub metadata: Metadata,
     pub pre: BeaconState<E>,
     pub blocks: Vec<BeaconBlock<E>>,
     pub post: Option<BeaconState<E>>,
 }
 
-impl<E: EthSpec> YamlDecode for SanityBlocks<E> {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(yaml).unwrap())
+impl<E: EthSpec> LoadCase for SanityBlocks<E> {
+    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+        let metadata: Metadata = yaml_decode_file(&path.join("meta.yaml"))?;
+        let pre = ssz_decode_file(&path.join("pre.ssz"))?;
+        let blocks: Vec<BeaconBlock<E>> = (0..metadata.blocks_count)
+            .map(|i| {
+                let filename = format!("blocks_{}.ssz", i);
+                ssz_decode_file(&path.join(filename))
+            })
+            .collect::<Result<_, _>>()?;
+        let post_file = path.join("post.ssz");
+        let post = if post_file.is_file() {
+            Some(ssz_decode_file(&post_file)?)
+        } else {
+            None
+        };
+
+        Ok(Self {
+            metadata,
+            pre,
+            blocks,
+            post,
+        })
     }
 }
 
 impl<E: EthSpec> Case for SanityBlocks<E> {
     fn description(&self) -> String {
-        self.description.clone()
+        self.metadata
+            .description
+            .clone()
+            .unwrap_or_else(String::new)
     }
 
     fn result(&self, _case_index: usize) -> Result<(), Error> {
-        self.bls_setting.unwrap_or_default().check()?;
+        self.metadata.bls_setting.unwrap_or_default().check()?;
 
         let mut state = self.pre.clone();
         let mut expected = self.post.clone();

tests/ef_tests/src/cases/sanity_slots.rs

@@ -1,30 +1,63 @@
 use super::*;
+use crate::bls_setting::BlsSetting;
 use crate::case_result::compare_beacon_state_results_without_caches;
+use crate::decode::{ssz_decode_file, yaml_decode_file};
 use serde_derive::Deserialize;
 use state_processing::per_slot_processing;
 use types::{BeaconState, EthSpec};
 
+#[derive(Debug, Clone, Default, Deserialize)]
+pub struct Metadata {
+    pub description: Option<String>,
+    pub bls_setting: Option<BlsSetting>,
+}
+
 #[derive(Debug, Clone, Deserialize)]
 #[serde(bound = "E: EthSpec")]
 pub struct SanitySlots<E: EthSpec> {
-    pub description: String,
+    pub metadata: Metadata,
     pub pre: BeaconState<E>,
-    pub slots: usize,
+    pub slots: u64,
     pub post: Option<BeaconState<E>>,
 }
 
-impl<E: EthSpec> YamlDecode for SanitySlots<E> {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(yaml).unwrap())
+impl<E: EthSpec> LoadCase for SanitySlots<E> {
+    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+        let metadata_path = path.join("meta.yaml");
+        let metadata: Metadata = if metadata_path.is_file() {
+            yaml_decode_file(&metadata_path)?
+        } else {
+            Metadata::default()
+        };
+        let pre = ssz_decode_file(&path.join("pre.ssz"))?;
+        let slots: u64 = yaml_decode_file(&path.join("slots.yaml"))?;
+        let post_file = path.join("post.ssz");
+        let post = if post_file.is_file() {
+            Some(ssz_decode_file(&post_file)?)
+        } else {
+            None
+        };
+
+        Ok(Self {
+            metadata,
+            pre,
+            slots,
+            post,
+        })
     }
 }
 
 impl<E: EthSpec> Case for SanitySlots<E> {
     fn description(&self) -> String {
-        self.description.clone()
+        self.metadata
+            .description
+            .clone()
+            .unwrap_or_else(String::new)
     }
 
     fn result(&self, _case_index: usize) -> Result<(), Error> {
+        self.metadata.bls_setting.unwrap_or_default().check()?;
+
         let mut state = self.pre.clone();
         let mut expected = self.post.clone();
         let spec = &E::default_spec();

tests/ef_tests/src/cases/shuffling.rs

@@ -1,5 +1,6 @@
 use super::*;
 use crate::case_result::compare_result;
+use crate::decode::yaml_decode_file;
 use serde_derive::Deserialize;
 use std::marker::PhantomData;
 use swap_or_not_shuffle::{get_permutated_index, shuffle_list};
@@ -8,21 +9,21 @@ use swap_or_not_shuffle::{get_permutated_index, shuffle_list};
 pub struct Shuffling<T> {
     pub seed: String,
     pub count: usize,
-    pub shuffled: Vec<usize>,
+    pub mapping: Vec<usize>,
     #[serde(skip)]
     _phantom: PhantomData<T>,
 }
 
-impl<T> YamlDecode for Shuffling<T> {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(yaml).unwrap())
+impl<T: EthSpec> LoadCase for Shuffling<T> {
+    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+        yaml_decode_file(&path.join("mapping.yaml"))
     }
 }
 
 impl<T: EthSpec> Case for Shuffling<T> {
     fn result(&self, _case_index: usize) -> Result<(), Error> {
         if self.count == 0 {
-            compare_result::<_, Error>(&Ok(vec![]), &Some(self.shuffled.clone()))?;
+            compare_result::<_, Error>(&Ok(vec![]), &Some(self.mapping.clone()))?;
         } else {
             let spec = T::default_spec();
             let seed = hex::decode(&self.seed[2..])
@@ -34,12 +35,12 @@ impl<T: EthSpec> Case for Shuffling<T> {
                     get_permutated_index(i, self.count, &seed, spec.shuffle_round_count).unwrap()
                 })
                 .collect();
-            compare_result::<_, Error>(&Ok(shuffling), &Some(self.shuffled.clone()))?;
+            compare_result::<_, Error>(&Ok(shuffling), &Some(self.mapping.clone()))?;
 
             // Test "shuffle_list"
             let input: Vec<usize> = (0..self.count).collect();
             let shuffling = shuffle_list(input, spec.shuffle_round_count, &seed, false).unwrap();
-            compare_result::<_, Error>(&Ok(shuffling), &Some(self.shuffled.clone()))?;
+            compare_result::<_, Error>(&Ok(shuffling), &Some(self.mapping.clone()))?;
         }
 
         Ok(())

tests/ef_tests/src/cases/ssz_generic.rs

@@ -1,68 +1,302 @@
+#![allow(non_snake_case)]
+
 use super::*;
-use crate::case_result::compare_result;
-use ethereum_types::{U128, U256};
+use crate::cases::common::{SszStaticType, TestU128, TestU256};
+use crate::cases::ssz_static::{check_serialization, check_tree_hash};
+use crate::decode::yaml_decode_file;
+use serde::{de::Error as SerdeError, Deserializer};
 use serde_derive::Deserialize;
-use ssz::Decode;
-use std::fmt::Debug;
+use ssz_derive::{Decode, Encode};
+use std::fs;
+use std::path::{Path, PathBuf};
+use tree_hash_derive::TreeHash;
+use types::typenum::*;
+use types::{BitList, BitVector, FixedVector, VariableList};
 
 #[derive(Debug, Clone, Deserialize)]
-pub struct SszGeneric {
-    #[serde(alias = "type")]
-    pub type_name: String,
-    pub valid: bool,
-    pub value: Option<String>,
-    pub ssz: Option<String>,
+struct Metadata {
+    root: String,
+    signing_root: Option<String>,
 }
 
-impl YamlDecode for SszGeneric {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        Ok(serde_yaml::from_str(yaml).unwrap())
+#[derive(Debug, Clone)]
+pub struct SszGeneric {
+    path: PathBuf,
+    handler_name: String,
+    case_name: String,
+}
+
+impl LoadCase for SszGeneric {
+    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+        let components = path
+            .components()
+            .map(|c| c.as_os_str().to_string_lossy().into_owned())
+            .rev()
+            .collect::<Vec<_>>();
+        // Test case name is last
+        let case_name = components[0].clone();
+        // Handler name is third last, before suite name and case name
+        let handler_name = components[2].clone();
+        Ok(Self {
+            path: path.into(),
+            handler_name,
+            case_name,
+        })
+    }
+}
+
+macro_rules! type_dispatch {
+    ($function:ident,
+     ($($arg:expr),*),
+     $base_ty:tt,
+     <$($param_ty:ty),*>,
+     [ $value:expr => primitive_type ] $($rest:tt)*) => {
+        match $value {
+            "bool" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* bool>, $($rest)*),
+            "uint8" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u8>, $($rest)*),
+            "uint16" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u16>, $($rest)*),
+            "uint32" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u32>, $($rest)*),
+            "uint64" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u64>, $($rest)*),
+            "uint128" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* TestU128>, $($rest)*),
+            "uint256" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* TestU256>, $($rest)*),
+            _ => Err(Error::FailedToParseTest(format!("unsupported: {}", $value))),
+        }
+    };
+    ($function:ident,
+     ($($arg:expr),*),
+     $base_ty:tt,
+     <$($param_ty:ty),*>,
+     [ $value:expr => typenum ] $($rest:tt)*) => {
+        match $value {
+            // DO YOU LIKE NUMBERS?
+            "0" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U0>, $($rest)*),
+            "1" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U1>, $($rest)*),
+            "2" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U2>, $($rest)*),
+            "3" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U3>, $($rest)*),
+            "4" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U4>, $($rest)*),
+            "5" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U5>, $($rest)*),
+            "6" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U6>, $($rest)*),
+            "7" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U7>, $($rest)*),
+            "8" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U8>, $($rest)*),
+            "9" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U9>, $($rest)*),
+            "16" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U16>, $($rest)*),
+            "31" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U31>, $($rest)*),
+            "32" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U32>, $($rest)*),
+            "64" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U64>, $($rest)*),
+            "128" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U128>, $($rest)*),
+            "256" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U256>, $($rest)*),
+            "512" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U512>, $($rest)*),
+            "513" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U513>, $($rest)*),
+            "1024" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U1024>, $($rest)*),
+            "2048" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U2048>, $($rest)*),
+            "4096" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U4096>, $($rest)*),
+            "8192" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U8192>, $($rest)*),
+            _ => Err(Error::FailedToParseTest(format!("unsupported: {}", $value))),
+        }
+    };
+    ($function:ident,
+     ($($arg:expr),*),
+     $base_ty:tt,
+     <$($param_ty:ty),*>,
+     [ $value:expr => test_container ] $($rest:tt)*) => {
+        match $value {
+            "SingleFieldTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* SingleFieldTestStruct>, $($rest)*),
+            "SmallTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* SmallTestStruct>, $($rest)*),
+            "FixedTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* FixedTestStruct>, $($rest)*),
+            "VarTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* VarTestStruct>, $($rest)*),
+            "ComplexTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ComplexTestStruct>, $($rest)*),
+            "BitsStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* BitsStruct>, $($rest)*),
+            _ => Err(Error::FailedToParseTest(format!("unsupported: {}", $value))),
+        }
+    };
+    // No base type: apply type params to function
+    ($function:ident, ($($arg:expr),*), _, <$($param_ty:ty),*>,) => {
+        $function::<$($param_ty),*>($($arg),*)
+    };
+    ($function:ident, ($($arg:expr),*), $base_type_name:ident, <$($param_ty:ty),*>,) => {
+        $function::<$base_type_name<$($param_ty),*>>($($arg),*)
     }
 }
 
 impl Case for SszGeneric {
     fn result(&self, _case_index: usize) -> Result<(), Error> {
-        if let Some(ssz) = &self.ssz {
-            match self.type_name.as_ref() {
-                "uint8" => ssz_generic_test::<u8>(self.valid, ssz, &self.value),
-                "uint16" => ssz_generic_test::<u16>(self.valid, ssz, &self.value),
-                "uint32" => ssz_generic_test::<u32>(self.valid, ssz, &self.value),
-                "uint64" => ssz_generic_test::<u64>(self.valid, ssz, &self.value),
-                "uint128" => ssz_generic_test::<U128>(self.valid, ssz, &self.value),
-                "uint256" => ssz_generic_test::<U256>(self.valid, ssz, &self.value),
-                _ => Err(Error::FailedToParseTest(format!(
-                    "Unknown type: {}",
-                    self.type_name
-                ))),
+        let parts = self.case_name.split('_').collect::<Vec<_>>();
+
+        match self.handler_name.as_str() {
+            "basic_vector" => {
+                let elem_ty = parts[1];
+                let length = parts[2];
+
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path),
+                    FixedVector,
+                    <>,
+                    [elem_ty => primitive_type]
+                    [length => typenum]
+                )?;
             }
-        } else {
-            // Skip tests that do not have an ssz field.
-            //
-            // See: https://github.com/ethereum/eth2.0-specs/issues/1079
-            Ok(())
+            "bitlist" => {
+                let mut limit = parts[1];
+
+                // Test format is inconsistent, pretend the limit is 32 (arbitrary)
+                // https://github.com/ethereum/eth2.0-spec-tests
+                if limit == "no" {
+                    limit = "32";
+                }
+
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path),
+                    BitList,
+                    <>,
+                    [limit => typenum]
+                )?;
+            }
+            "bitvector" => {
+                let length = parts[1];
+
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path),
+                    BitVector,
+                    <>,
+                    [length => typenum]
+                )?;
+            }
+            "boolean" => {
+                ssz_generic_test::<bool>(&self.path)?;
+            }
+            "uints" => {
+                let type_name = "uint".to_owned() + parts[1];
+
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path),
+                    _,
+                    <>,
+                    [type_name.as_str() => primitive_type]
+                )?;
+            }
+            "containers" => {
+                let type_name = parts[0];
+
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path),
+                    _,
+                    <>,
+                    [type_name => test_container]
+                )?;
+            }
+            _ => panic!("unsupported handler: {}", self.handler_name),
         }
+        Ok(())
     }
 }
 
-/// Execute a `ssz_generic` test case.
-fn ssz_generic_test<T>(should_be_ok: bool, ssz: &str, value: &Option<String>) -> Result<(), Error>
-where
-    T: Decode + YamlDecode + Debug + PartialEq<T>,
-{
-    let ssz = hex::decode(&ssz[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
-
-    // We do not cater for the scenario where the test is valid but we are not passed any SSZ.
-    if should_be_ok && value.is_none() {
-        panic!("Unexpected test input. Cannot pass without value.")
-    }
-
-    let expected = if let Some(string) = value {
-        Some(T::yaml_decode(string)?)
+fn ssz_generic_test<T: SszStaticType>(path: &Path) -> Result<(), Error> {
+    let meta_path = path.join("meta.yaml");
+    let meta: Option<Metadata> = if meta_path.is_file() {
+        Some(yaml_decode_file(&meta_path)?)
     } else {
         None
     };
 
-    let decoded = T::from_ssz_bytes(&ssz);
+    let serialized = fs::read(&path.join("serialized.ssz")).expect("serialized.ssz exists");
 
-    compare_result(&decoded, &expected)
+    let value_path = path.join("value.yaml");
+    let value: Option<T> = if value_path.is_file() {
+        Some(yaml_decode_file(&value_path)?)
+    } else {
+        None
+    };
+
+    // Valid
+    // TODO: signing root (annoying because of traits)
+    if let Some(value) = value {
+        check_serialization(&value, &serialized)?;
+
+        if let Some(ref meta) = meta {
+            check_tree_hash(&meta.root, value.tree_hash_root())?;
+        }
+    }
+    // Invalid
+    else {
+        if let Ok(decoded) = T::from_ssz_bytes(&serialized) {
+            return Err(Error::DidntFail(format!(
+                "Decoded invalid bytes into: {:?}",
+                decoded
+            )));
+        }
+    }
+
+    Ok(())
+}
+
+// Containers for SSZ generic tests
+#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+struct SingleFieldTestStruct {
+    A: u8,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+struct SmallTestStruct {
+    A: u16,
+    B: u16,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+struct FixedTestStruct {
+    A: u8,
+    B: u64,
+    C: u32,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+struct VarTestStruct {
+    A: u16,
+    B: VariableList<u16, U1024>,
+    C: u8,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+struct ComplexTestStruct {
+    A: u16,
+    B: VariableList<u16, U128>,
+    C: u8,
+    #[serde(deserialize_with = "byte_list_from_hex_str")]
+    D: VariableList<u8, U256>,
+    E: VarTestStruct,
+    F: FixedVector<FixedTestStruct, U4>,
+    G: FixedVector<VarTestStruct, U2>,
+}
+
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+struct BitsStruct {
+    A: BitList<U5>,
+    B: BitVector<U2>,
+    C: BitVector<U1>,
+    D: BitList<U6>,
+    E: BitVector<U8>,
+}
+
+fn byte_list_from_hex_str<'de, D, N: Unsigned>(
+    deserializer: D,
+) -> Result<VariableList<u8, N>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let s: String = serde::de::Deserialize::deserialize(deserializer)?;
+    let decoded: Vec<u8> = hex::decode(&s.as_str()[2..]).map_err(D::Error::custom)?;
+
+    if decoded.len() > N::to_usize() {
+        return Err(D::Error::custom(format!(
+            "Too many values for list, got: {}, limit: {}",
+            decoded.len(),
+            N::to_usize()
+        )));
+    } else {
+        Ok(decoded.into())
+    }
 }

tests/ef_tests/src/cases/ssz_static.rs

@@ -1,127 +1,101 @@
 use super::*;
 use crate::case_result::compare_result;
+use crate::cases::common::SszStaticType;
+use crate::decode::yaml_decode_file;
 use serde_derive::Deserialize;
-use ssz::{Decode, Encode};
-use std::fmt::Debug;
-use std::marker::PhantomData;
-use tree_hash::TreeHash;
-use types::{
-    test_utils::TestRandom, Attestation, AttestationData, AttestationDataAndCustodyBit,
-    AttesterSlashing, BeaconBlock, BeaconBlockBody, BeaconBlockHeader, BeaconState, Checkpoint,
-    CompactCommittee, Crosslink, Deposit, DepositData, Eth1Data, EthSpec, Fork, Hash256,
-    HistoricalBatch, IndexedAttestation, PendingAttestation, ProposerSlashing, Transfer, Validator,
-    VoluntaryExit,
-};
-
-// Enum variant names are used by Serde when deserializing the test YAML
-#[allow(clippy::large_enum_variant)]
-#[derive(Debug, Clone, Deserialize)]
-pub enum SszStatic<E>
-where
-    E: EthSpec,
-{
-    Fork(SszStaticInner<Fork, E>),
-    Crosslink(SszStaticInner<Crosslink, E>),
-    Checkpoint(SszStaticInner<Checkpoint, E>),
-    CompactCommittee(SszStaticInner<CompactCommittee<E>, E>),
-    Eth1Data(SszStaticInner<Eth1Data, E>),
-    AttestationData(SszStaticInner<AttestationData, E>),
-    AttestationDataAndCustodyBit(SszStaticInner<AttestationDataAndCustodyBit, E>),
-    IndexedAttestation(SszStaticInner<IndexedAttestation<E>, E>),
-    DepositData(SszStaticInner<DepositData, E>),
-    BeaconBlockHeader(SszStaticInner<BeaconBlockHeader, E>),
-    Validator(SszStaticInner<Validator, E>),
-    PendingAttestation(SszStaticInner<PendingAttestation<E>, E>),
-    HistoricalBatch(SszStaticInner<HistoricalBatch<E>, E>),
-    ProposerSlashing(SszStaticInner<ProposerSlashing, E>),
-    AttesterSlashing(SszStaticInner<AttesterSlashing<E>, E>),
-    Attestation(SszStaticInner<Attestation<E>, E>),
-    Deposit(SszStaticInner<Deposit, E>),
-    VoluntaryExit(SszStaticInner<VoluntaryExit, E>),
-    Transfer(SszStaticInner<Transfer, E>),
-    BeaconBlockBody(SszStaticInner<BeaconBlockBody<E>, E>),
-    BeaconBlock(SszStaticInner<BeaconBlock<E>, E>),
-    BeaconState(SszStaticInner<BeaconState<E>, E>),
-}
+use std::fs;
+use tree_hash::SignedRoot;
+use types::Hash256;
 
 #[derive(Debug, Clone, Deserialize)]
-pub struct SszStaticInner<T, E>
-where
-    E: EthSpec,
-{
-    pub value: T,
-    pub serialized: String,
-    pub root: String,
-    #[serde(skip, default)]
-    _phantom: PhantomData<E>,
+struct SszStaticRoots {
+    root: String,
+    signing_root: Option<String>,
 }
 
-impl<E: EthSpec + serde::de::DeserializeOwned> YamlDecode for SszStatic<E> {
-    fn yaml_decode(yaml: &str) -> Result<Self, Error> {
-        serde_yaml::from_str(yaml).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))
+#[derive(Debug, Clone)]
+pub struct SszStatic<T> {
+    roots: SszStaticRoots,
+    serialized: Vec<u8>,
+    value: T,
+}
+
+#[derive(Debug, Clone)]
+pub struct SszStaticSR<T> {
+    roots: SszStaticRoots,
+    serialized: Vec<u8>,
+    value: T,
+}
+
+fn load_from_dir<T: SszStaticType>(path: &Path) -> Result<(SszStaticRoots, Vec<u8>, T), Error> {
+    let roots = yaml_decode_file(&path.join("roots.yaml"))?;
+    let serialized = fs::read(&path.join("serialized.ssz")).expect("serialized.ssz exists");
+    let value = yaml_decode_file(&path.join("value.yaml"))?;
+
+    Ok((roots, serialized, value))
+}
+
+impl<T: SszStaticType> LoadCase for SszStatic<T> {
+    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+        load_from_dir(path).map(|(roots, serialized, value)| Self {
+            roots,
+            serialized,
+            value,
+        })
     }
 }
 
-impl<E: EthSpec> Case for SszStatic<E> {
-    fn result(&self, _case_index: usize) -> Result<(), Error> {
-        use self::SszStatic::*;
-
-        match *self {
-            Fork(ref val) => ssz_static_test(val),
-            Crosslink(ref val) => ssz_static_test(val),
-            Checkpoint(ref val) => ssz_static_test(val),
-            CompactCommittee(ref val) => ssz_static_test(val),
-            Eth1Data(ref val) => ssz_static_test(val),
-            AttestationData(ref val) => ssz_static_test(val),
-            AttestationDataAndCustodyBit(ref val) => ssz_static_test(val),
-            IndexedAttestation(ref val) => ssz_static_test(val),
-            DepositData(ref val) => ssz_static_test(val),
-            BeaconBlockHeader(ref val) => ssz_static_test(val),
-            Validator(ref val) => ssz_static_test(val),
-            PendingAttestation(ref val) => ssz_static_test(val),
-            HistoricalBatch(ref val) => ssz_static_test(val),
-            ProposerSlashing(ref val) => ssz_static_test(val),
-            AttesterSlashing(ref val) => ssz_static_test(val),
-            Attestation(ref val) => ssz_static_test(val),
-            Deposit(ref val) => ssz_static_test(val),
-            VoluntaryExit(ref val) => ssz_static_test(val),
-            Transfer(ref val) => ssz_static_test(val),
-            BeaconBlockBody(ref val) => ssz_static_test(val),
-            BeaconBlock(ref val) => ssz_static_test(val),
-            BeaconState(ref val) => ssz_static_test(val),
-        }
+impl<T: SszStaticType + SignedRoot> LoadCase for SszStaticSR<T> {
+    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+        load_from_dir(path).map(|(roots, serialized, value)| Self {
+            roots,
+            serialized,
+            value,
+        })
     }
 }
 
-fn ssz_static_test<T, E: EthSpec>(tc: &SszStaticInner<T, E>) -> Result<(), Error>
-where
-    T: Clone
-        + Decode
-        + Debug
-        + Encode
-        + PartialEq<T>
-        + serde::de::DeserializeOwned
-        + TreeHash
-        + TestRandom,
-{
-    // Verify we can decode SSZ in the same way we can decode YAML.
-    let ssz = hex::decode(&tc.serialized[2..])
-        .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
-    let expected = tc.value.clone();
-    let decode_result = T::from_ssz_bytes(&ssz);
-    compare_result(&decode_result, &Some(expected))?;
+pub fn check_serialization<T: SszStaticType>(value: &T, serialized: &[u8]) -> Result<(), Error> {
+    // Check serialization
+    let serialized_result = value.as_ssz_bytes();
+    compare_result::<usize, Error>(&Ok(value.ssz_bytes_len()), &Some(serialized.len()))?;
+    compare_result::<Vec<u8>, Error>(&Ok(serialized_result), &Some(serialized.to_vec()))?;
 
-    // Verify we can encode the result back into original ssz bytes
-    let decoded = decode_result.unwrap();
-    let encoded_result = decoded.as_ssz_bytes();
-    compare_result::<Vec<u8>, Error>(&Ok(encoded_result), &Some(ssz))?;
-
-    // Verify the TreeHash root of the decoded struct matches the test.
-    let expected_root =
-        &hex::decode(&tc.root[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
-    let expected_root = Hash256::from_slice(&expected_root);
-    let tree_hash_root = Hash256::from_slice(&decoded.tree_hash_root());
-    compare_result::<Hash256, Error>(&Ok(tree_hash_root), &Some(expected_root))?;
+    // Check deserialization
+    let deserialized_result = T::from_ssz_bytes(serialized);
+    compare_result(&deserialized_result, &Some(value.clone()))?;
 
     Ok(())
 }
+
+pub fn check_tree_hash(expected_str: &str, actual_root: Vec<u8>) -> Result<(), Error> {
+    let expected_root = hex::decode(&expected_str[2..])
+        .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+    let expected_root = Hash256::from_slice(&expected_root);
+    let tree_hash_root = Hash256::from_slice(&actual_root);
+    compare_result::<Hash256, Error>(&Ok(tree_hash_root), &Some(expected_root))
+}
+
+impl<T: SszStaticType> Case for SszStatic<T> {
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        check_serialization(&self.value, &self.serialized)?;
+        check_tree_hash(&self.roots.root, self.value.tree_hash_root())?;
+        Ok(())
+    }
+}
+
+impl<T: SszStaticType + SignedRoot> Case for SszStaticSR<T> {
+    fn result(&self, _case_index: usize) -> Result<(), Error> {
+        check_serialization(&self.value, &self.serialized)?;
+        check_tree_hash(&self.roots.root, self.value.tree_hash_root())?;
+        check_tree_hash(
+            &self
+                .roots
+                .signing_root
+                .as_ref()
+                .expect("signed root exists"),
+            self.value.signed_root(),
+        )?;
+        Ok(())
+    }
+}

tests/ef_tests/src/decode.rs

@@ -0,0 +1,31 @@
+use super::*;
+use std::fs;
+use std::path::Path;
+
+pub fn yaml_decode<T: serde::de::DeserializeOwned>(string: &str) -> Result<T, Error> {
+    serde_yaml::from_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))
+}
+
+pub fn yaml_decode_file<T: serde::de::DeserializeOwned>(path: &Path) -> Result<T, Error> {
+    fs::read_to_string(path)
+        .map_err(|e| {
+            Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e))
+        })
+        .and_then(|s| yaml_decode(&s))
+}
+
+pub fn ssz_decode_file<T: ssz::Decode>(path: &Path) -> Result<T, Error> {
+    fs::read(path)
+        .map_err(|e| {
+            Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e))
+        })
+        .and_then(|s| {
+            T::from_ssz_bytes(&s).map_err(|e| {
+                Error::FailedToParseTest(format!(
+                    "Unable to parse SSZ at {}: {:?}",
+                    path.display(),
+                    e
+                ))
+            })
+        })
+}

tests/ef_tests/src/doc.rs

@@ -1,253 +0,0 @@
-use crate::case_result::CaseResult;
-use crate::cases::*;
-use crate::doc_header::DocHeader;
-use crate::error::Error;
-use crate::yaml_decode::{yaml_split_header_and_cases, YamlDecode};
-use crate::EfTest;
-use serde_derive::Deserialize;
-use std::{fs::File, io::prelude::*, path::PathBuf};
-use types::{MainnetEthSpec, MinimalEthSpec};
-
-#[derive(Debug, Deserialize)]
-pub struct Doc {
-    pub header_yaml: String,
-    pub cases_yaml: String,
-    pub path: PathBuf,
-}
-
-impl Doc {
-    fn from_path(path: PathBuf) -> Self {
-        let mut file = File::open(path.clone()).unwrap();
-
-        let mut yaml = String::new();
-        file.read_to_string(&mut yaml).unwrap();
-
-        let (header_yaml, cases_yaml) = yaml_split_header_and_cases(yaml.clone());
-
-        Self {
-            header_yaml,
-            cases_yaml,
-            path,
-        }
-    }
-
-    pub fn test_results(&self) -> Vec<CaseResult> {
-        let header: DocHeader = serde_yaml::from_str(&self.header_yaml.as_str()).unwrap();
-
-        match (
-            header.runner.as_ref(),
-            header.handler.as_ref(),
-            header.config.as_ref(),
-        ) {
-            ("ssz", "uint", _) => run_test::<SszGeneric>(self),
-            ("ssz", "static", "minimal") => run_test::<SszStatic<MinimalEthSpec>>(self),
-            ("ssz", "static", "mainnet") => run_test::<SszStatic<MainnetEthSpec>>(self),
-            ("sanity", "slots", "minimal") => run_test::<SanitySlots<MinimalEthSpec>>(self),
-            // FIXME: skipped due to compact committees issue
-            ("sanity", "slots", "mainnet") => vec![], // run_test::<SanitySlots<MainnetEthSpec>>(self),
-            ("sanity", "blocks", "minimal") => run_test::<SanityBlocks<MinimalEthSpec>>(self),
-            // FIXME: skipped due to compact committees issue
-            ("sanity", "blocks", "mainnet") => vec![], // run_test::<SanityBlocks<MainnetEthSpec>>(self),
-            ("shuffling", "core", "minimal") => run_test::<Shuffling<MinimalEthSpec>>(self),
-            ("shuffling", "core", "mainnet") => run_test::<Shuffling<MainnetEthSpec>>(self),
-            ("bls", "aggregate_pubkeys", "mainnet") => run_test::<BlsAggregatePubkeys>(self),
-            ("bls", "aggregate_sigs", "mainnet") => run_test::<BlsAggregateSigs>(self),
-            ("bls", "msg_hash_compressed", "mainnet") => run_test::<BlsG2Compressed>(self),
-            // Note this test fails due to a difference in our internal representations. It does
-            // not effect verification or external representation.
-            //
-            // It is skipped.
-            ("bls", "msg_hash_uncompressed", "mainnet") => vec![],
-            ("bls", "priv_to_pub", "mainnet") => run_test::<BlsPrivToPub>(self),
-            ("bls", "sign_msg", "mainnet") => run_test::<BlsSign>(self),
-            ("operations", "deposit", "mainnet") => {
-                run_test::<OperationsDeposit<MainnetEthSpec>>(self)
-            }
-            ("operations", "deposit", "minimal") => {
-                run_test::<OperationsDeposit<MinimalEthSpec>>(self)
-            }
-            ("operations", "transfer", "mainnet") => {
-                run_test::<OperationsTransfer<MainnetEthSpec>>(self)
-            }
-            ("operations", "transfer", "minimal") => {
-                run_test::<OperationsTransfer<MinimalEthSpec>>(self)
-            }
-            ("operations", "voluntary_exit", "mainnet") => {
-                run_test::<OperationsExit<MainnetEthSpec>>(self)
-            }
-            ("operations", "voluntary_exit", "minimal") => {
-                run_test::<OperationsExit<MinimalEthSpec>>(self)
-            }
-            ("operations", "proposer_slashing", "mainnet") => {
-                run_test::<OperationsProposerSlashing<MainnetEthSpec>>(self)
-            }
-            ("operations", "proposer_slashing", "minimal") => {
-                run_test::<OperationsProposerSlashing<MinimalEthSpec>>(self)
-            }
-            ("operations", "attester_slashing", "mainnet") => {
-                run_test::<OperationsAttesterSlashing<MainnetEthSpec>>(self)
-            }
-            ("operations", "attester_slashing", "minimal") => {
-                run_test::<OperationsAttesterSlashing<MinimalEthSpec>>(self)
-            }
-            ("operations", "attestation", "mainnet") => {
-                run_test::<OperationsAttestation<MainnetEthSpec>>(self)
-            }
-            ("operations", "attestation", "minimal") => {
-                run_test::<OperationsAttestation<MinimalEthSpec>>(self)
-            }
-            ("operations", "block_header", "mainnet") => {
-                run_test::<OperationsBlockHeader<MainnetEthSpec>>(self)
-            }
-            ("operations", "block_header", "minimal") => {
-                run_test::<OperationsBlockHeader<MinimalEthSpec>>(self)
-            }
-            ("epoch_processing", "crosslinks", "minimal") => {
-                run_test::<EpochProcessingCrosslinks<MinimalEthSpec>>(self)
-            }
-            ("epoch_processing", "crosslinks", "mainnet") => {
-                run_test::<EpochProcessingCrosslinks<MainnetEthSpec>>(self)
-            }
-            ("epoch_processing", "registry_updates", "minimal") => {
-                run_test::<EpochProcessingRegistryUpdates<MinimalEthSpec>>(self)
-            }
-            ("epoch_processing", "registry_updates", "mainnet") => {
-                run_test::<EpochProcessingRegistryUpdates<MainnetEthSpec>>(self)
-            }
-            ("epoch_processing", "justification_and_finalization", "minimal") => {
-                run_test::<EpochProcessingJustificationAndFinalization<MinimalEthSpec>>(self)
-            }
-            ("epoch_processing", "justification_and_finalization", "mainnet") => {
-                run_test::<EpochProcessingJustificationAndFinalization<MainnetEthSpec>>(self)
-            }
-            ("epoch_processing", "slashings", "minimal") => {
-                run_test::<EpochProcessingSlashings<MinimalEthSpec>>(self)
-            }
-            ("epoch_processing", "slashings", "mainnet") => {
-                run_test::<EpochProcessingSlashings<MainnetEthSpec>>(self)
-            }
-            ("epoch_processing", "final_updates", "minimal") => {
-                run_test::<EpochProcessingFinalUpdates<MinimalEthSpec>>(self)
-            }
-            ("epoch_processing", "final_updates", "mainnet") => {
-                vec![]
-                // FIXME: skipped due to compact committees issue
-                // run_test::<EpochProcessingFinalUpdates<MainnetEthSpec>>(self)
-            }
-            ("genesis", "initialization", "minimal") => {
-                run_test::<GenesisInitialization<MinimalEthSpec>>(self)
-            }
-            ("genesis", "initialization", "mainnet") => {
-                run_test::<GenesisInitialization<MainnetEthSpec>>(self)
-            }
-            ("genesis", "validity", "minimal") => run_test::<GenesisValidity<MinimalEthSpec>>(self),
-            ("genesis", "validity", "mainnet") => run_test::<GenesisValidity<MainnetEthSpec>>(self),
-            (runner, handler, config) => panic!(
-                "No implementation for runner: \"{}\", handler: \"{}\", config: \"{}\"",
-                runner, handler, config
-            ),
-        }
-    }
-
-    pub fn assert_tests_pass(path: PathBuf) {
-        let doc = Self::from_path(path);
-        let results = doc.test_results();
-
-        let (failed, skipped_bls, skipped_known_failures) = categorize_results(&results);
-
-        if failed.len() + skipped_known_failures.len() > 0 {
-            print_results(
-                &doc,
-                &failed,
-                &skipped_bls,
-                &skipped_known_failures,
-                &results,
-            );
-            if !failed.is_empty() {
-                panic!("Tests failed (see above)");
-            }
-        } else {
-            println!("Passed {} tests in {:?}", results.len(), doc.path);
-        }
-    }
-}
-
-pub fn run_test<T>(doc: &Doc) -> Vec<CaseResult>
-where
-    Cases<T>: EfTest + YamlDecode,
-{
-    // Pass only the "test_cases" YAML string to `yaml_decode`.
-    let test_cases: Cases<T> = Cases::yaml_decode(&doc.cases_yaml).unwrap();
-
-    test_cases.test_results()
-}
-
-pub fn categorize_results(
-    results: &[CaseResult],
-) -> (Vec<&CaseResult>, Vec<&CaseResult>, Vec<&CaseResult>) {
-    let mut failed = vec![];
-    let mut skipped_bls = vec![];
-    let mut skipped_known_failures = vec![];
-
-    for case in results {
-        match case.result.as_ref().err() {
-            Some(Error::SkippedBls) => skipped_bls.push(case),
-            Some(Error::SkippedKnownFailure) => skipped_known_failures.push(case),
-            Some(_) => failed.push(case),
-            None => (),
-        }
-    }
-
-    (failed, skipped_bls, skipped_known_failures)
-}
-
-pub fn print_results(
-    doc: &Doc,
-    failed: &[&CaseResult],
-    skipped_bls: &[&CaseResult],
-    skipped_known_failures: &[&CaseResult],
-    results: &[CaseResult],
-) {
-    let header: DocHeader = serde_yaml::from_str(&doc.header_yaml).unwrap();
-    println!("--------------------------------------------------");
-    println!(
-        "Test {}",
-        if failed.is_empty() {
-            "Result"
-        } else {
-            "Failure"
-        }
-    );
-    println!("Title: {}", header.title);
-    println!("File: {:?}", doc.path);
-    println!(
-        "{} tests, {} failed, {} skipped (known failure), {} skipped (bls), {} passed. (See below for errors)",
-        results.len(),
-        failed.len(),
-        skipped_known_failures.len(),
-        skipped_bls.len(),
-        results.len() - skipped_bls.len() - skipped_known_failures.len() - failed.len()
-    );
-    println!();
-
-    for case in skipped_known_failures {
-        println!("-------");
-        println!(
-            "case[{}] ({}) skipped because it's a known failure",
-            case.case_index, case.desc,
-        );
-    }
-    for failure in failed {
-        let error = failure.result.clone().unwrap_err();
-
-        println!("-------");
-        println!(
-            "case[{}] ({}) failed with {}:",
-            failure.case_index,
-            failure.desc,
-            error.name()
-        );
-        println!("{}", error.message());
-    }
-    println!();
-}

tests/ef_tests/src/doc_header.rs

@@ -1,12 +0,0 @@
-use serde_derive::Deserialize;
-
-#[derive(Debug, Deserialize)]
-pub struct DocHeader {
-    pub title: String,
-    pub summary: String,
-    pub forks_timeline: String,
-    pub forks: Vec<String>,
-    pub config: String,
-    pub runner: String,
-    pub handler: String,
-}

tests/ef_tests/src/handler.rs

@@ -0,0 +1,297 @@
+use crate::cases::{self, Case, Cases, EpochTransition, LoadCase, Operation};
+use crate::type_name;
+use crate::type_name::TypeName;
+use std::fs;
+use std::marker::PhantomData;
+use std::path::PathBuf;
+use tree_hash::SignedRoot;
+use types::EthSpec;
+
+pub trait Handler {
+    type Case: Case + LoadCase;
+
+    fn config_name() -> &'static str {
+        "general"
+    }
+
+    fn fork_name() -> &'static str {
+        "phase0"
+    }
+
+    fn runner_name() -> &'static str;
+
+    fn handler_name() -> String;
+
+    fn run() {
+        let handler_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+            .join("eth2.0-spec-tests")
+            .join("tests")
+            .join(Self::config_name())
+            .join(Self::fork_name())
+            .join(Self::runner_name())
+            .join(Self::handler_name());
+
+        // If the directory containing the tests does not exist, just let all tests pass.
+        if !handler_path.exists() {
+            return;
+        }
+
+        // Iterate through test suites
+        let test_cases = fs::read_dir(&handler_path)
+            .expect("handler dir exists")
+            .flat_map(|entry| {
+                entry
+                    .ok()
+                    .filter(|e| e.file_type().map(|ty| ty.is_dir()).unwrap_or(false))
+            })
+            .flat_map(|suite| fs::read_dir(suite.path()).expect("suite dir exists"))
+            .flat_map(Result::ok)
+            .map(|test_case_dir| {
+                let path = test_case_dir.path();
+                let case = Self::Case::load_from_dir(&path).expect("test should load");
+                (path, case)
+            })
+            .collect();
+
+        let results = Cases { test_cases }.test_results();
+
+        let name = format!("{}/{}", Self::runner_name(), Self::handler_name());
+        crate::results::assert_tests_pass(&name, &handler_path, &results);
+    }
+}
+
+macro_rules! bls_handler {
+    ($runner_name: ident, $case_name:ident, $handler_name:expr) => {
+        pub struct $runner_name;
+
+        impl Handler for $runner_name {
+            type Case = cases::$case_name;
+
+            fn runner_name() -> &'static str {
+                "bls"
+            }
+
+            fn handler_name() -> String {
+                $handler_name.into()
+            }
+        }
+    };
+}
+
+bls_handler!(
+    BlsAggregatePubkeysHandler,
+    BlsAggregatePubkeys,
+    "aggregate_pubkeys"
+);
+bls_handler!(BlsAggregateSigsHandler, BlsAggregateSigs, "aggregate_sigs");
+bls_handler!(
+    BlsG2CompressedHandler,
+    BlsG2Compressed,
+    "msg_hash_compressed"
+);
+bls_handler!(BlsPrivToPubHandler, BlsPrivToPub, "priv_to_pub");
+bls_handler!(BlsSignMsgHandler, BlsSign, "sign_msg");
+
+/// Handler for SSZ types that do not implement `SignedRoot`.
+pub struct SszStaticHandler<T, E>(PhantomData<(T, E)>);
+
+/// Handler for SSZ types that do implement `SignedRoot`.
+pub struct SszStaticSRHandler<T, E>(PhantomData<(T, E)>);
+
+impl<T, E> Handler for SszStaticHandler<T, E>
+where
+    T: cases::SszStaticType + TypeName,
+    E: TypeName,
+{
+    type Case = cases::SszStatic<T>;
+
+    fn config_name() -> &'static str {
+        E::name()
+    }
+
+    fn runner_name() -> &'static str {
+        "ssz_static"
+    }
+
+    fn handler_name() -> String {
+        T::name().into()
+    }
+}
+
+impl<T, E> Handler for SszStaticSRHandler<T, E>
+where
+    T: cases::SszStaticType + SignedRoot + TypeName,
+    E: TypeName,
+{
+    type Case = cases::SszStaticSR<T>;
+
+    fn config_name() -> &'static str {
+        E::name()
+    }
+
+    fn runner_name() -> &'static str {
+        "ssz_static"
+    }
+
+    fn handler_name() -> String {
+        T::name().into()
+    }
+}
+
+pub struct ShufflingHandler<E>(PhantomData<E>);
+
+impl<E: EthSpec + TypeName> Handler for ShufflingHandler<E> {
+    type Case = cases::Shuffling<E>;
+
+    fn config_name() -> &'static str {
+        E::name()
+    }
+
+    fn runner_name() -> &'static str {
+        "shuffling"
+    }
+
+    fn handler_name() -> String {
+        "core".into()
+    }
+}
+
+pub struct SanityBlocksHandler<E>(PhantomData<E>);
+
+impl<E: EthSpec + TypeName> Handler for SanityBlocksHandler<E> {
+    type Case = cases::SanityBlocks<E>;
+
+    fn config_name() -> &'static str {
+        E::name()
+    }
+
+    fn runner_name() -> &'static str {
+        "sanity"
+    }
+
+    fn handler_name() -> String {
+        "blocks".into()
+    }
+}
+
+pub struct SanitySlotsHandler<E>(PhantomData<E>);
+
+impl<E: EthSpec + TypeName> Handler for SanitySlotsHandler<E> {
+    type Case = cases::SanitySlots<E>;
+
+    fn config_name() -> &'static str {
+        E::name()
+    }
+
+    fn runner_name() -> &'static str {
+        "sanity"
+    }
+
+    fn handler_name() -> String {
+        "slots".into()
+    }
+}
+
+pub struct EpochProcessingHandler<E, T>(PhantomData<(E, T)>);
+
+impl<E: EthSpec + TypeName, T: EpochTransition<E>> Handler for EpochProcessingHandler<E, T> {
+    type Case = cases::EpochProcessing<E, T>;
+
+    fn config_name() -> &'static str {
+        E::name()
+    }
+
+    fn runner_name() -> &'static str {
+        "epoch_processing"
+    }
+
+    fn handler_name() -> String {
+        T::name().into()
+    }
+}
+
+pub struct GenesisValidityHandler<E>(PhantomData<E>);
+
+impl<E: EthSpec + TypeName> Handler for GenesisValidityHandler<E> {
+    type Case = cases::GenesisValidity<E>;
+
+    fn config_name() -> &'static str {
+        E::name()
+    }
+
+    fn runner_name() -> &'static str {
+        "genesis"
+    }
+
+    fn handler_name() -> String {
+        "validity".into()
+    }
+}
+
+pub struct GenesisInitializationHandler<E>(PhantomData<E>);
+
+impl<E: EthSpec + TypeName> Handler for GenesisInitializationHandler<E> {
+    type Case = cases::GenesisInitialization<E>;
+
+    fn config_name() -> &'static str {
+        E::name()
+    }
+
+    fn runner_name() -> &'static str {
+        "genesis"
+    }
+
+    fn handler_name() -> String {
+        "initialization".into()
+    }
+}
+
+pub struct OperationsHandler<E, O>(PhantomData<(E, O)>);
+
+impl<E: EthSpec + TypeName, O: Operation<E>> Handler for OperationsHandler<E, O> {
+    type Case = cases::Operations<E, O>;
+
+    fn config_name() -> &'static str {
+        E::name()
+    }
+
+    fn runner_name() -> &'static str {
+        "operations"
+    }
+
+    fn handler_name() -> String {
+        O::handler_name()
+    }
+}
+
+pub struct SszGenericHandler<H>(PhantomData<H>);
+
+impl<H: TypeName> Handler for SszGenericHandler<H> {
+    type Case = cases::SszGeneric;
+
+    fn config_name() -> &'static str {
+        "general"
+    }
+
+    fn runner_name() -> &'static str {
+        "ssz_generic"
+    }
+
+    fn handler_name() -> String {
+        H::name().into()
+    }
+}
+
+// Supported SSZ generic handlers
+pub struct BasicVector;
+type_name!(BasicVector, "basic_vector");
+pub struct Bitlist;
+type_name!(Bitlist, "bitlist");
+pub struct Bitvector;
+type_name!(Bitvector, "bitvector");
+pub struct Boolean;
+type_name!(Boolean, "boolean");
+pub struct Uints;
+type_name!(Uints, "uints");
+pub struct Containers;
+type_name!(Containers, "containers");

tests/ef_tests/src/lib.rs

@@ -2,21 +2,17 @@ use types::EthSpec;
 
 pub use case_result::CaseResult;
 pub use cases::Case;
-pub use doc::Doc;
+pub use cases::{
+    Crosslinks, FinalUpdates, JustificationAndFinalization, RegistryUpdates, Slashings,
+};
 pub use error::Error;
-pub use yaml_decode::YamlDecode;
+pub use handler::*;
 
 mod bls_setting;
 mod case_result;
 mod cases;
-mod doc;
-mod doc_header;
+mod decode;
 mod error;
-mod yaml_decode;
-
-/// Defined where an object can return the results of some test(s) adhering to the Ethereum
-/// Foundation testing format.
-pub trait EfTest {
-    /// Returns the results of executing one or more tests.
-    fn test_results(&self) -> Vec<CaseResult>;
-}
+mod handler;
+mod results;
+mod type_name;

tests/ef_tests/src/results.rs

@@ -0,0 +1,92 @@
+use crate::case_result::CaseResult;
+use crate::error::Error;
+use std::path::Path;
+
+pub fn assert_tests_pass(handler_name: &str, path: &Path, results: &[CaseResult]) {
+    let (failed, skipped_bls, skipped_known_failures) = categorize_results(results);
+
+    if failed.len() + skipped_known_failures.len() > 0 {
+        print_results(
+            handler_name,
+            &failed,
+            &skipped_bls,
+            &skipped_known_failures,
+            &results,
+        );
+        if !failed.is_empty() {
+            panic!("Tests failed (see above)");
+        }
+    } else {
+        println!("Passed {} tests in {}", results.len(), path.display());
+    }
+}
+
+pub fn categorize_results(
+    results: &[CaseResult],
+) -> (Vec<&CaseResult>, Vec<&CaseResult>, Vec<&CaseResult>) {
+    let mut failed = vec![];
+    let mut skipped_bls = vec![];
+    let mut skipped_known_failures = vec![];
+
+    for case in results {
+        match case.result.as_ref().err() {
+            Some(Error::SkippedBls) => skipped_bls.push(case),
+            Some(Error::SkippedKnownFailure) => skipped_known_failures.push(case),
+            Some(_) => failed.push(case),
+            None => (),
+        }
+    }
+
+    (failed, skipped_bls, skipped_known_failures)
+}
+
+pub fn print_results(
+    handler_name: &str,
+    failed: &[&CaseResult],
+    skipped_bls: &[&CaseResult],
+    skipped_known_failures: &[&CaseResult],
+    results: &[CaseResult],
+) {
+    println!("--------------------------------------------------");
+    println!(
+        "Test {}",
+        if failed.is_empty() {
+            "Result"
+        } else {
+            "Failure"
+        }
+    );
+    println!("Title: {}", handler_name);
+    println!(
+        "{} tests, {} failed, {} skipped (known failure), {} skipped (bls), {} passed. (See below for errors)",
+        results.len(),
+        failed.len(),
+        skipped_known_failures.len(),
+        skipped_bls.len(),
+        results.len() - skipped_bls.len() - skipped_known_failures.len() - failed.len()
+    );
+    println!();
+
+    for case in skipped_known_failures {
+        println!("-------");
+        println!(
+            "case ({}) from {} skipped because it's a known failure",
+            case.desc,
+            case.path.display()
+        );
+    }
+    for failure in failed {
+        let error = failure.result.clone().unwrap_err();
+
+        println!("-------");
+        println!(
+            "case {} ({}) from {} failed with {}:",
+            failure.case_index,
+            failure.desc,
+            failure.path.display(),
+            error.name()
+        );
+        println!("{}", error.message());
+    }
+    println!();
+}

tests/ef_tests/src/type_name.rs

@@ -0,0 +1,60 @@
+//! Mapping from types to canonical string identifiers used in testing.
+use types::*;
+
+pub trait TypeName {
+    fn name() -> &'static str;
+}
+
+#[macro_export]
+macro_rules! type_name {
+    ($typ:ident) => {
+        type_name!($typ, stringify!($typ));
+    };
+    ($typ:ident, $name:expr) => {
+        impl TypeName for $typ {
+            fn name() -> &'static str {
+                $name
+            }
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! type_name_generic {
+    ($typ:ident) => {
+        type_name_generic!($typ, stringify!($typ));
+    };
+    ($typ:ident, $name:expr) => {
+        impl<E: EthSpec> TypeName for $typ<E> {
+            fn name() -> &'static str {
+                $name
+            }
+        }
+    };
+}
+
+type_name!(MinimalEthSpec, "minimal");
+type_name!(MainnetEthSpec, "mainnet");
+
+type_name_generic!(Attestation);
+type_name!(AttestationData);
+type_name!(AttestationDataAndCustodyBit);
+type_name_generic!(AttesterSlashing);
+type_name_generic!(BeaconBlock);
+type_name_generic!(BeaconBlockBody);
+type_name!(BeaconBlockHeader);
+type_name_generic!(BeaconState);
+type_name!(Checkpoint);
+type_name_generic!(CompactCommittee);
+type_name!(Crosslink);
+type_name!(Deposit);
+type_name!(DepositData);
+type_name!(Eth1Data);
+type_name!(Fork);
+type_name_generic!(HistoricalBatch);
+type_name_generic!(IndexedAttestation);
+type_name_generic!(PendingAttestation);
+type_name!(ProposerSlashing);
+type_name!(Transfer);
+type_name!(Validator);
+type_name!(VoluntaryExit);

tests/ef_tests/src/yaml_decode.rs

@@ -1,59 +0,0 @@
-use super::*;
-use ethereum_types::{U128, U256};
-use types::Fork;
-
-mod utils;
-
-pub use utils::*;
-
-pub trait YamlDecode: Sized {
-    /// Decode an object from the test specification YAML.
-    fn yaml_decode(string: &str) -> Result<Self, Error>;
-}
-
-/// Basic types can general be decoded with the `parse` fn if they implement `str::FromStr`.
-macro_rules! impl_via_parse {
-    ($ty: ty) => {
-        impl YamlDecode for $ty {
-            fn yaml_decode(string: &str) -> Result<Self, Error> {
-                string
-                    .parse::<Self>()
-                    .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))
-            }
-        }
-    };
-}
-
-impl_via_parse!(u8);
-impl_via_parse!(u16);
-impl_via_parse!(u32);
-impl_via_parse!(u64);
-
-/// Some `ethereum-types` methods have a `str::FromStr` implementation that expects `0x`-prefixed:
-/// hex, so we use `from_dec_str` instead.
-macro_rules! impl_via_from_dec_str {
-    ($ty: ty) => {
-        impl YamlDecode for $ty {
-            fn yaml_decode(string: &str) -> Result<Self, Error> {
-                Self::from_dec_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))
-            }
-        }
-    };
-}
-
-impl_via_from_dec_str!(U128);
-impl_via_from_dec_str!(U256);
-
-/// Types that already implement `serde::Deserialize` can be decoded using `serde_yaml`.
-macro_rules! impl_via_serde_yaml {
-    ($ty: ty) => {
-        impl YamlDecode for $ty {
-            fn yaml_decode(string: &str) -> Result<Self, Error> {
-                serde_yaml::from_str(string)
-                    .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))
-            }
-        }
-    };
-}
-
-impl_via_serde_yaml!(Fork);

tests/ef_tests/src/yaml_decode/utils.rs

@@ -1,10 +0,0 @@
-pub fn yaml_split_header_and_cases(mut yaml: String) -> (String, String) {
-    let test_cases_start = yaml.find("\ntest_cases:\n").unwrap();
-    // + 1 to skip the \n we used for matching.
-    let mut test_cases = yaml.split_off(test_cases_start + 1);
-
-    let end_of_first_line = test_cases.find('\n').unwrap();
-    let test_cases = test_cases.split_off(end_of_first_line + 1);
-
-    (yaml, test_cases)
-}

tests/ef_tests/tests/tests.rs

@@ -1,225 +1,214 @@
 use ef_tests::*;
-use rayon::prelude::*;
-use std::path::{Path, PathBuf};
-use walkdir::WalkDir;
-
-fn yaml_files_in_test_dir(dir: &Path) -> Vec<PathBuf> {
-    let base_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-        .join("eth2.0-spec-tests")
-        .join("tests")
-        .join(dir);
-
-    assert!(
-        base_path.exists(),
-        format!(
-            "Unable to locate {:?}. Did you init git submodules?",
-            base_path
-        )
-    );
-
-    let mut paths: Vec<PathBuf> = WalkDir::new(base_path)
-        .into_iter()
-        .filter_map(|e| e.ok())
-        .filter_map(|entry| {
-            if entry.file_type().is_file() {
-                match entry.file_name().to_str() {
-                    Some(f) if f.ends_with(".yaml") => Some(entry.path().to_path_buf()),
-                    Some(f) if f.ends_with(".yml") => Some(entry.path().to_path_buf()),
-                    _ => None,
-                }
-            } else {
-                None
-            }
-        })
-        .collect();
-
-    // Reverse the file order. Assuming files come in lexicographical order, executing tests in
-    // reverse means we get the "minimal" tests before the "mainnet" tests. This makes life easier
-    // for debugging.
-    paths.reverse();
-    paths
-}
-
-#[test]
-#[cfg(feature = "fake_crypto")]
-fn ssz_generic() {
-    yaml_files_in_test_dir(&Path::new("ssz_generic"))
-        .into_par_iter()
-        .for_each(|file| {
-            Doc::assert_tests_pass(file);
-        });
-}
-
-#[test]
-#[cfg(feature = "fake_crypto")]
-fn ssz_static() {
-    yaml_files_in_test_dir(&Path::new("ssz_static"))
-        .into_par_iter()
-        .for_each(|file| {
-            Doc::assert_tests_pass(file);
-        });
-}
+use types::*;
 
 #[test]
 fn shuffling() {
-    yaml_files_in_test_dir(&Path::new("shuffling").join("core"))
-        .into_par_iter()
-        .for_each(|file| {
-            Doc::assert_tests_pass(file);
-        });
+    ShufflingHandler::<MinimalEthSpec>::run();
+    ShufflingHandler::<MainnetEthSpec>::run();
 }
 
 #[test]
 fn operations_deposit() {
-    yaml_files_in_test_dir(&Path::new("operations").join("deposit"))
-        .into_par_iter()
-        .for_each(|file| {
-            Doc::assert_tests_pass(file);
-        });
+    OperationsHandler::<MinimalEthSpec, Deposit>::run();
+    OperationsHandler::<MainnetEthSpec, Deposit>::run();
 }
 
 #[test]
 fn operations_transfer() {
-    yaml_files_in_test_dir(&Path::new("operations").join("transfer"))
-        .into_par_iter()
-        .rev()
-        .for_each(|file| {
-            Doc::assert_tests_pass(file);
-        });
+    OperationsHandler::<MinimalEthSpec, Transfer>::run();
+    // Note: there are no transfer tests for mainnet
 }
 
 #[test]
 fn operations_exit() {
-    yaml_files_in_test_dir(&Path::new("operations").join("voluntary_exit"))
-        .into_par_iter()
-        .for_each(|file| {
-            Doc::assert_tests_pass(file);
-        });
+    OperationsHandler::<MinimalEthSpec, VoluntaryExit>::run();
+    OperationsHandler::<MainnetEthSpec, VoluntaryExit>::run();
 }
 
 #[test]
 fn operations_proposer_slashing() {
-    yaml_files_in_test_dir(&Path::new("operations").join("proposer_slashing"))
-        .into_par_iter()
-        .for_each(|file| {
-            Doc::assert_tests_pass(file);
-        });
+    OperationsHandler::<MinimalEthSpec, ProposerSlashing>::run();
+    OperationsHandler::<MainnetEthSpec, ProposerSlashing>::run();
 }
 
 #[test]
 fn operations_attester_slashing() {
-    yaml_files_in_test_dir(&Path::new("operations").join("attester_slashing"))
-        .into_par_iter()
-        .for_each(|file| {
-            Doc::assert_tests_pass(file);
-        });
+    OperationsHandler::<MinimalEthSpec, AttesterSlashing<_>>::run();
+    OperationsHandler::<MainnetEthSpec, AttesterSlashing<_>>::run();
 }
 
 #[test]
 fn operations_attestation() {
-    yaml_files_in_test_dir(&Path::new("operations").join("attestation"))
-        .into_par_iter()
-        .for_each(|file| {
-            Doc::assert_tests_pass(file);
-        });
+    OperationsHandler::<MinimalEthSpec, Attestation<_>>::run();
+    OperationsHandler::<MainnetEthSpec, Attestation<_>>::run();
 }
 
 #[test]
 fn operations_block_header() {
-    yaml_files_in_test_dir(&Path::new("operations").join("block_header"))
-        .into_par_iter()
-        .for_each(|file| {
-            Doc::assert_tests_pass(file);
-        });
+    OperationsHandler::<MinimalEthSpec, BeaconBlock<_>>::run();
+    OperationsHandler::<MainnetEthSpec, BeaconBlock<_>>::run();
 }
 
 #[test]
 fn sanity_blocks() {
-    yaml_files_in_test_dir(&Path::new("sanity").join("blocks"))
-        .into_par_iter()
-        .for_each(|file| {
-            Doc::assert_tests_pass(file);
-        });
+    SanityBlocksHandler::<MinimalEthSpec>::run();
+    SanityBlocksHandler::<MainnetEthSpec>::run();
 }
 
 #[test]
 fn sanity_slots() {
-    yaml_files_in_test_dir(&Path::new("sanity").join("slots"))
-        .into_par_iter()
-        .for_each(|file| {
-            Doc::assert_tests_pass(file);
-        });
+    SanitySlotsHandler::<MinimalEthSpec>::run();
+    SanitySlotsHandler::<MainnetEthSpec>::run();
 }
 
 #[test]
 #[cfg(not(feature = "fake_crypto"))]
-fn bls() {
-    yaml_files_in_test_dir(&Path::new("bls"))
-        .into_par_iter()
-        .for_each(|file| {
-            Doc::assert_tests_pass(file);
-        });
+fn bls_aggregate_pubkeys() {
+    BlsAggregatePubkeysHandler::run();
+}
+
+#[test]
+#[cfg(not(feature = "fake_crypto"))]
+fn bls_aggregate_sigs() {
+    BlsAggregateSigsHandler::run();
+}
+
+#[test]
+#[cfg(not(feature = "fake_crypto"))]
+fn bls_msg_hash_g2_compressed() {
+    BlsG2CompressedHandler::run();
+}
+
+#[test]
+#[cfg(not(feature = "fake_crypto"))]
+fn bls_priv_to_pub() {
+    BlsPrivToPubHandler::run();
+}
+
+#[test]
+#[cfg(not(feature = "fake_crypto"))]
+fn bls_sign_msg() {
+    BlsSignMsgHandler::run();
+}
+
+#[cfg(feature = "fake_crypto")]
+macro_rules! ssz_static_test {
+    // Signed-root
+    ($test_name:ident, $typ:ident$(<$generics:tt>)?, SR) => {
+        ssz_static_test!($test_name, SszStaticSRHandler, $typ$(<$generics>)?);
+    };
+    // Non-signed root
+    ($test_name:ident, $typ:ident$(<$generics:tt>)?) => {
+        ssz_static_test!($test_name, SszStaticHandler, $typ$(<$generics>)?);
+    };
+    // Generic
+    ($test_name:ident, $handler:ident, $typ:ident<_>) => {
+        ssz_static_test!(
+            $test_name, $handler, {
+                ($typ<MinimalEthSpec>, MinimalEthSpec),
+                ($typ<MainnetEthSpec>, MainnetEthSpec)
+            }
+        );
+    };
+    // Non-generic
+    ($test_name:ident, $handler:ident, $typ:ident) => {
+        ssz_static_test!(
+            $test_name, $handler, {
+                ($typ, MinimalEthSpec),
+                ($typ, MainnetEthSpec)
+            }
+        );
+    };
+    // Base case
+    ($test_name:ident, $handler:ident, { $(($typ:ty, $spec:ident)),+ }) => {
+        #[test]
+        fn $test_name() {
+            $(
+                $handler::<$typ, $spec>::run();
+            )+
+        }
+    };
+}
+
+#[cfg(feature = "fake_crypto")]
+mod ssz_static {
+    use ef_tests::{Handler, SszStaticHandler, SszStaticSRHandler};
+    use types::*;
+
+    ssz_static_test!(attestation, Attestation<_>, SR);
+    ssz_static_test!(attestation_data, AttestationData);
+    ssz_static_test!(
+        attestation_data_and_custody_bit,
+        AttestationDataAndCustodyBit
+    );
+    ssz_static_test!(attester_slashing, AttesterSlashing<_>);
+    ssz_static_test!(beacon_block, BeaconBlock<_>, SR);
+    ssz_static_test!(beacon_block_body, BeaconBlockBody<_>);
+    ssz_static_test!(beacon_block_header, BeaconBlockHeader, SR);
+    ssz_static_test!(beacon_state, BeaconState<_>);
+    ssz_static_test!(checkpoint, Checkpoint);
+    ssz_static_test!(compact_committee, CompactCommittee<_>);
+    ssz_static_test!(crosslink, Crosslink);
+    ssz_static_test!(deposit, Deposit);
+    ssz_static_test!(deposit_data, DepositData, SR);
+    ssz_static_test!(eth1_data, Eth1Data);
+    ssz_static_test!(fork, Fork);
+    ssz_static_test!(historical_batch, HistoricalBatch<_>);
+    ssz_static_test!(indexed_attestation, IndexedAttestation<_>, SR);
+    ssz_static_test!(pending_attestation, PendingAttestation<_>);
+    ssz_static_test!(proposer_slashing, ProposerSlashing);
+    ssz_static_test!(transfer, Transfer, SR);
+    ssz_static_test!(validator, Validator);
+    ssz_static_test!(voluntary_exit, VoluntaryExit, SR);
+}
+
+#[test]
+fn ssz_generic() {
+    SszGenericHandler::<BasicVector>::run();
+    SszGenericHandler::<Bitlist>::run();
+    SszGenericHandler::<Bitvector>::run();
+    SszGenericHandler::<Boolean>::run();
+    SszGenericHandler::<Uints>::run();
+    SszGenericHandler::<Containers>::run();
 }
 
 #[test]
 fn epoch_processing_justification_and_finalization() {
-    yaml_files_in_test_dir(&Path::new("epoch_processing").join("justification_and_finalization"))
-        .into_par_iter()
-        .for_each(|file| {
-            Doc::assert_tests_pass(file);
-        });
+    EpochProcessingHandler::<MinimalEthSpec, JustificationAndFinalization>::run();
+    EpochProcessingHandler::<MainnetEthSpec, JustificationAndFinalization>::run();
 }
 
 #[test]
 fn epoch_processing_crosslinks() {
-    yaml_files_in_test_dir(&Path::new("epoch_processing").join("crosslinks"))
-        .into_par_iter()
-        .for_each(|file| {
-            Doc::assert_tests_pass(file);
-        });
+    EpochProcessingHandler::<MinimalEthSpec, Crosslinks>::run();
+    EpochProcessingHandler::<MainnetEthSpec, Crosslinks>::run();
 }
 
 #[test]
 fn epoch_processing_registry_updates() {
-    yaml_files_in_test_dir(&Path::new("epoch_processing").join("registry_updates"))
-        .into_par_iter()
-        .for_each(|file| {
-            Doc::assert_tests_pass(file);
-        });
+    EpochProcessingHandler::<MinimalEthSpec, RegistryUpdates>::run();
+    EpochProcessingHandler::<MainnetEthSpec, RegistryUpdates>::run();
 }
 
 #[test]
 fn epoch_processing_slashings() {
-    yaml_files_in_test_dir(&Path::new("epoch_processing").join("slashings"))
-        .into_par_iter()
-        .for_each(|file| {
-            Doc::assert_tests_pass(file);
-        });
+    EpochProcessingHandler::<MinimalEthSpec, Slashings>::run();
+    EpochProcessingHandler::<MainnetEthSpec, Slashings>::run();
 }
 
 #[test]
 fn epoch_processing_final_updates() {
-    yaml_files_in_test_dir(&Path::new("epoch_processing").join("final_updates"))
-        .into_par_iter()
-        .for_each(|file| {
-            Doc::assert_tests_pass(file);
-        });
+    EpochProcessingHandler::<MainnetEthSpec, FinalUpdates>::run();
+    EpochProcessingHandler::<MainnetEthSpec, FinalUpdates>::run();
 }
 
 #[test]
 fn genesis_initialization() {
-    yaml_files_in_test_dir(&Path::new("genesis").join("initialization"))
-        .into_par_iter()
-        .for_each(|file| {
-            Doc::assert_tests_pass(file);
-        });
+    GenesisInitializationHandler::<MinimalEthSpec>::run();
 }
 
 #[test]
 fn genesis_validity() {
-    yaml_files_in_test_dir(&Path::new("genesis").join("validity"))
-        .into_par_iter()
-        .for_each(|file| {
-            Doc::assert_tests_pass(file);
-        });
+    GenesisValidityHandler::<MinimalEthSpec>::run();
+    // Note: there are no genesis validity tests for mainnet
 }

validator_client/src/service.rs

@@ -27,7 +27,7 @@ use slot_clock::{SlotClock, SystemTimeSlotClock};
 use std::marker::PhantomData;
 use std::sync::Arc;
 use std::sync::RwLock;
-use std::time::{Duration, Instant, SystemTime};
+use std::time::{Duration, Instant};
 use tokio::prelude::*;
 use tokio::runtime::Builder;
 use tokio::timer::Interval;
@@ -46,8 +46,8 @@ pub struct Service<B: BeaconNodeDuties + 'static, S: Signer + 'static, E: EthSpe
     fork: Fork,
     /// The slot clock for this service.
     slot_clock: SystemTimeSlotClock,
-    /// The current slot we are processing.
-    current_slot: Slot,
+    /// The slot that is currently, or was previously processed by the service.
+    current_slot: Option<Slot>,
     slots_per_epoch: u64,
     /// The chain specification for this clients instance.
     spec: Arc<ChainSpec>,
@@ -100,19 +100,6 @@ impl<B: BeaconNodeDuties + 'static, S: Signer + 'static, E: EthSpec> Service<B,
                     continue;
                 }
                 Ok(info) => {
-                    // verify the node's genesis time
-                    if SystemTime::now()
-                        .duration_since(SystemTime::UNIX_EPOCH)
-                        .unwrap()
-                        .as_secs()
-                        < info.genesis_time
-                    {
-                        error!(
-                            log,
-                            "Beacon Node's genesis time is in the future. No work to do.\n Exiting"
-                        );
-                        return Err("Genesis time in the future".into());
-                    }
                     // verify the node's network id
                     if eth2_config.spec.network_id != info.network_id as u8 {
                         error!(
@@ -177,12 +164,8 @@ impl<B: BeaconNodeDuties + 'static, S: Signer + 'static, E: EthSpec> Service<B,
             genesis_time,
             Duration::from_millis(eth2_config.spec.milliseconds_per_slot),
         )
-        .ok_or_else::<error_chain::Error, _>(|| {
-            "Unable to start slot clock. Genesis may not have occurred yet. Exiting.".into()
-        })?;
-
-        let current_slot = slot_clock.now().ok_or_else::<error_chain::Error, _>(|| {
-            "Genesis has not yet occurred. Exiting.".into()
+        .map_err::<error_chain::Error, _>(|e| {
+            format!("Unable to start slot clock: {}.", e).into()
         })?;
 
         /* Generate the duties manager */
@@ -215,7 +198,7 @@ impl<B: BeaconNodeDuties + 'static, S: Signer + 'static, E: EthSpec> Service<B,
         Ok(Service {
             fork,
             slot_clock,
-            current_slot,
+            current_slot: None,
             slots_per_epoch,
             spec,
             duties_manager,
@@ -237,7 +220,7 @@ impl<B: BeaconNodeDuties + 'static, S: Signer + 'static, E: EthSpec> Service<B,
         let mut service = Service::<ValidatorServiceClient, Keypair, E>::initialize_service(
             client_config,
             eth2_config,
-            log,
+            log.clone(),
         )?;
 
         // we have connected to a node and established its parameters. Spin up the core service
@@ -253,7 +236,7 @@ impl<B: BeaconNodeDuties + 'static, S: Signer + 'static, E: EthSpec> Service<B,
             .slot_clock
             .duration_to_next_slot()
             .ok_or_else::<error_chain::Error, _>(|| {
-                "Genesis is not in the past. Exiting.".into()
+                "Unable to determine duration to next slot. Exiting.".into()
             })?;
 
         // set up the validator work interval - start at next slot and proceed every slot
@@ -264,6 +247,19 @@ impl<B: BeaconNodeDuties + 'static, S: Signer + 'static, E: EthSpec> Service<B,
             Interval::new(Instant::now() + duration_to_next_slot, slot_duration)
         };
 
+        if service.slot_clock.now().is_none() {
+            warn!(
+                log,
+                "Starting node prior to genesis";
+            );
+        }
+
+        info!(
+            log,
+            "Waiting for next slot";
+            "seconds_to_wait" => duration_to_next_slot.as_secs()
+        );
+
         /* kick off the core service */
         runtime.block_on(
             interval
@@ -298,27 +294,29 @@ impl<B: BeaconNodeDuties + 'static, S: Signer + 'static, E: EthSpec> Service<B,
 
     /// Updates the known current slot and epoch.
     fn update_current_slot(&mut self) -> error_chain::Result<()> {
-        let current_slot = self
+        let wall_clock_slot = self
             .slot_clock
             .now()
             .ok_or_else::<error_chain::Error, _>(|| {
                 "Genesis is not in the past. Exiting.".into()
             })?;
 
-        let current_epoch = current_slot.epoch(self.slots_per_epoch);
+        let wall_clock_epoch = wall_clock_slot.epoch(self.slots_per_epoch);
 
         // this is a non-fatal error. If the slot clock repeats, the node could
         // have been slow to process the previous slot and is now duplicating tasks.
         // We ignore duplicated but raise a critical error.
-        if current_slot <= self.current_slot {
-            crit!(
-                self.log,
-                "The validator tried to duplicate a slot. Likely missed the previous slot"
-            );
-            return Err("Duplicate slot".into());
+        if let Some(current_slot) = self.current_slot {
+            if wall_clock_slot <= current_slot {
+                crit!(
+                    self.log,
+                    "The validator tried to duplicate a slot. Likely missed the previous slot"
+                );
+                return Err("Duplicate slot".into());
+            }
         }
-        self.current_slot = current_slot;
-        info!(self.log, "Processing"; "slot" => current_slot.as_u64(), "epoch" => current_epoch.as_u64());
+        self.current_slot = Some(wall_clock_slot);
+        info!(self.log, "Processing"; "slot" => wall_clock_slot.as_u64(), "epoch" => wall_clock_epoch.as_u64());
         Ok(())
     }
 
@@ -326,7 +324,10 @@ impl<B: BeaconNodeDuties + 'static, S: Signer + 'static, E: EthSpec> Service<B,
     fn check_for_duties(&mut self) {
         let cloned_manager = self.duties_manager.clone();
         let cloned_log = self.log.clone();
-        let current_epoch = self.current_slot.epoch(self.slots_per_epoch);
+        let current_epoch = self
+            .current_slot
+            .expect("The current slot must be updated before checking for duties")
+            .epoch(self.slots_per_epoch);
         // spawn a new thread separate to the runtime
         // TODO: Handle thread termination/timeout
         // TODO: Add duties thread back in, with channel to process duties in duty change.
@@ -340,14 +341,19 @@ impl<B: BeaconNodeDuties + 'static, S: Signer + 'static, E: EthSpec> Service<B,
 
     /// If there are any duties to process, spawn a separate thread and perform required actions.
     fn process_duties(&mut self) {
-        if let Some(work) = self.duties_manager.get_current_work(self.current_slot) {
+        if let Some(work) = self.duties_manager.get_current_work(
+            self.current_slot
+                .expect("The current slot must be updated before processing duties"),
+        ) {
             for (signer_index, work_type) in work {
                 if work_type.produce_block {
                     // we need to produce a block
                     // spawns a thread to produce a beacon block
                     let signers = self.duties_manager.signers.clone(); // this is an arc
                     let fork = self.fork.clone();
-                    let slot = self.current_slot;
+                    let slot = self
+                        .current_slot
+                        .expect("The current slot must be updated before processing duties");
                     let spec = self.spec.clone();
                     let beacon_node = self.beacon_block_client.clone();
                     let log = self.log.clone();