Merge pull request #180 from sigp/add-chain-benchmarks

Add chain benchmarks

Cargo.toml

@@ -1,6 +1,7 @@
 [workspace]
 members = [
-	"eth2/attestation_validation",
+	"eth2/attester",
+	"eth2/block_producer",
 	"eth2/genesis",
 	"eth2/naive_fork_choice",
 	"eth2/types",
@@ -12,9 +13,10 @@ members = [
 	"eth2/utils/ssz",
 	"eth2/utils/vec_shuffle",
 	"eth2/validator_induction",
-	"eth2/validator_shuffling",
 	"beacon_node",
 	"beacon_node/db",
+	"beacon_node/beacon_chain",
+	"beacon_node/beacon_chain/test_harness",
 	"protos",
 	"validator_client",
 ]

beacon_node/Cargo.toml

@@ -6,6 +6,7 @@ edition = "2018"
 
 [dependencies]
 bls = { path = "../eth2/utils/bls" }
+beacon_chain = { path = "beacon_chain" }
 grpcio = { version = "0.4", default-features = false, features = ["protobuf-codec"] }
 protobuf = "2.0.2"
 protos = { path = "../protos" }
@@ -13,8 +14,11 @@ clap = "2.32.0"
 db = { path = "db" }
 dirs = "1.0.3"
 futures = "0.1.23"
+genesis = { path = "../eth2/genesis" }
 slog = "^2.2.3"
+slot_clock = { path = "../eth2/utils/slot_clock" }
 slog-term = "^2.4.0"
 slog-async = "^2.3.0"
+types = { path = "../eth2/types" }
 ssz = { path = "../eth2/utils/ssz" }
 tokio = "0.1"

beacon_node/beacon_chain/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "beacon_chain"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+[dependencies]
+block_producer = { path = "../../eth2/block_producer" }
+bls = { path = "../../eth2/utils/bls" }
+boolean-bitfield = { path = "../../eth2/utils/boolean-bitfield" }
+db = { path = "../db" }
+failure = "0.1"
+failure_derive = "0.1"
+genesis = { path = "../../eth2/genesis" }
+hashing = { path = "../../eth2/utils/hashing" }
+parking_lot = "0.7"
+log = "0.4"
+env_logger = "0.6"
+serde = "1.0"
+serde_derive = "1.0"
+serde_json = "1.0"
+slot_clock = { path = "../../eth2/utils/slot_clock" }
+ssz = { path = "../../eth2/utils/ssz" }
+types = { path = "../../eth2/types" }

beacon_node/beacon_chain/src/attestation_aggregator.rs

@@ -0,0 +1,217 @@
+use std::collections::{HashMap, HashSet};
+use types::{
+    beacon_state::CommitteesError, AggregateSignature, Attestation, AttestationData, BeaconState,
+    Bitfield, ChainSpec, FreeAttestation, Signature,
+};
+
+const PHASE_0_CUSTODY_BIT: bool = false;
+
+/// Provides the functionality to:
+///
+///  - Recieve a `FreeAttestation` and aggregate it into an `Attestation` (or create a new if it
+///  doesn't exist).
+///  - Store all aggregated or created `Attestation`s.
+///  - Produce a list of attestations that would be valid for inclusion in some `BeaconState` (and
+///  therefore valid for inclusion in a `BeaconBlock`.
+///
+///  Note: `Attestations` are stored in memory and never deleted. This is not scalable and must be
+///  rectified in a future revision.
+pub struct AttestationAggregator {
+    store: HashMap<Vec<u8>, Attestation>,
+}
+
+pub struct Outcome {
+    pub valid: bool,
+    pub message: Message,
+}
+
+pub enum Message {
+    /// The free attestation was added to an existing attestation.
+    Aggregated,
+    /// The free attestation has already been aggregated to an existing attestation.
+    AggregationNotRequired,
+    /// The free attestation was transformed into a new attestation.
+    NewAttestationCreated,
+    /// The supplied `validator_index` is not in the committee for the given `shard` and `slot`.
+    BadValidatorIndex,
+    /// The given `signature` did not match the `pubkey` in the given
+    /// `state.validator_registry`.
+    BadSignature,
+    /// The given `slot` does not match the validators committee assignment.
+    BadSlot,
+    /// The given `shard` does not match the validators committee assignment.
+    BadShard,
+}
+
+macro_rules! some_or_invalid {
+    ($expression: expr, $error: expr) => {
+        match $expression {
+            Some(x) => x,
+            None => {
+                return Ok(Outcome {
+                    valid: false,
+                    message: $error,
+                });
+            }
+        }
+    };
+}
+
+impl AttestationAggregator {
+    /// Instantiates a new AttestationAggregator with an empty database.
+    pub fn new() -> Self {
+        Self {
+            store: HashMap::new(),
+        }
+    }
+
+    /// Accepts some `FreeAttestation`, validates it and either aggregates it upon some existing
+    /// `Attestation` or produces a new `Attestation`.
+    ///
+    /// The "validation" provided is not complete, instead the following points are checked:
+    ///  - The given `validator_index` is in the committee for the given `shard` for the given
+    ///  `slot`.
+    ///  - The signature is verified against that of the validator at `validator_index`.
+    pub fn process_free_attestation(
+        &mut self,
+        state: &BeaconState,
+        free_attestation: &FreeAttestation,
+        spec: &ChainSpec,
+    ) -> Result<Outcome, CommitteesError> {
+        let (slot, shard, committee_index) = some_or_invalid!(
+            state.attestation_slot_and_shard_for_validator(
+                free_attestation.validator_index as usize,
+                spec,
+            )?,
+            Message::BadValidatorIndex
+        );
+
+        if free_attestation.data.slot != slot {
+            return Ok(Outcome {
+                valid: false,
+                message: Message::BadSlot,
+            });
+        }
+        if free_attestation.data.shard != shard {
+            return Ok(Outcome {
+                valid: false,
+                message: Message::BadShard,
+            });
+        }
+
+        let signable_message = free_attestation.data.signable_message(PHASE_0_CUSTODY_BIT);
+
+        let validator_record = some_or_invalid!(
+            state
+                .validator_registry
+                .get(free_attestation.validator_index as usize),
+            Message::BadValidatorIndex
+        );
+
+        if !free_attestation
+            .signature
+            .verify(&signable_message, &validator_record.pubkey)
+        {
+            return Ok(Outcome {
+                valid: false,
+                message: Message::BadSignature,
+            });
+        }
+
+        if let Some(existing_attestation) = self.store.get(&signable_message) {
+            if let Some(updated_attestation) = aggregate_attestation(
+                existing_attestation,
+                &free_attestation.signature,
+                committee_index as usize,
+            ) {
+                self.store.insert(signable_message, updated_attestation);
+                Ok(Outcome {
+                    valid: true,
+                    message: Message::Aggregated,
+                })
+            } else {
+                Ok(Outcome {
+                    valid: true,
+                    message: Message::AggregationNotRequired,
+                })
+            }
+        } else {
+            let mut aggregate_signature = AggregateSignature::new();
+            aggregate_signature.add(&free_attestation.signature);
+            let mut aggregation_bitfield = Bitfield::new();
+            aggregation_bitfield.set(committee_index as usize, true);
+            let new_attestation = Attestation {
+                data: free_attestation.data.clone(),
+                aggregation_bitfield,
+                custody_bitfield: Bitfield::new(),
+                aggregate_signature,
+            };
+            self.store.insert(signable_message, new_attestation);
+            Ok(Outcome {
+                valid: true,
+                message: Message::NewAttestationCreated,
+            })
+        }
+    }
+
+    /// Returns all known attestations which are:
+    ///
+    /// - Valid for the given state
+    /// - Not already in `state.latest_attestations`.
+    pub fn get_attestations_for_state(
+        &self,
+        state: &BeaconState,
+        spec: &ChainSpec,
+    ) -> Vec<Attestation> {
+        let mut known_attestation_data: HashSet<AttestationData> = HashSet::new();
+
+        state.latest_attestations.iter().for_each(|attestation| {
+            known_attestation_data.insert(attestation.data.clone());
+        });
+
+        self.store
+            .values()
+            .filter_map(|attestation| {
+                if state
+                    .validate_attestation_without_signature(attestation, spec)
+                    .is_ok()
+                    && !known_attestation_data.contains(&attestation.data)
+                {
+                    Some(attestation.clone())
+                } else {
+                    None
+                }
+            })
+            .collect()
+    }
+}
+
+/// Produces a new `Attestation` where:
+///
+/// - `signature` is added to `Attestation.aggregate_signature`
+/// - Attestation.aggregation_bitfield[committee_index]` is set to true.
+fn aggregate_attestation(
+    existing_attestation: &Attestation,
+    signature: &Signature,
+    committee_index: usize,
+) -> Option<Attestation> {
+    let already_signed = existing_attestation
+        .aggregation_bitfield
+        .get(committee_index)
+        .unwrap_or(false);
+
+    if already_signed {
+        None
+    } else {
+        let mut aggregation_bitfield = existing_attestation.aggregation_bitfield.clone();
+        aggregation_bitfield.set(committee_index, true);
+        let mut aggregate_signature = existing_attestation.aggregate_signature.clone();
+        aggregate_signature.add(&signature);
+
+        Some(Attestation {
+            aggregation_bitfield,
+            aggregate_signature,
+            ..existing_attestation.clone()
+        })
+    }
+}

beacon_node/beacon_chain/src/attestation_targets.rs

@@ -0,0 +1,22 @@
+use std::collections::HashMap;
+use types::Hash256;
+
+pub struct AttestationTargets {
+    map: HashMap<u64, Hash256>,
+}
+
+impl AttestationTargets {
+    pub fn new() -> Self {
+        Self {
+            map: HashMap::new(),
+        }
+    }
+
+    pub fn get(&self, validator_index: u64) -> Option<&Hash256> {
+        self.map.get(&validator_index)
+    }
+
+    pub fn insert(&mut self, validator_index: u64, block_hash: Hash256) -> Option<Hash256> {
+        self.map.insert(validator_index, block_hash)
+    }
+}

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -0,0 +1,586 @@
+use db::{
+    stores::{BeaconBlockStore, BeaconStateStore},
+    ClientDB, DBError,
+};
+use genesis::{genesis_beacon_block, genesis_beacon_state};
+use log::{debug, trace};
+use parking_lot::{RwLock, RwLockReadGuard};
+use slot_clock::SlotClock;
+use ssz::ssz_encode;
+use std::sync::Arc;
+use types::{
+    beacon_state::{BlockProcessingError, CommitteesError, SlotProcessingError},
+    readers::{BeaconBlockReader, BeaconStateReader},
+    AttestationData, BeaconBlock, BeaconBlockBody, BeaconState, ChainSpec, Eth1Data,
+    FreeAttestation, Hash256, PublicKey, Signature,
+};
+
+use crate::attestation_aggregator::{AttestationAggregator, Outcome as AggregationOutcome};
+use crate::attestation_targets::AttestationTargets;
+use crate::block_graph::BlockGraph;
+use crate::checkpoint::CheckPoint;
+
+#[derive(Debug, PartialEq)]
+pub enum Error {
+    InsufficientValidators,
+    BadRecentBlockRoots,
+    CommitteesError(CommitteesError),
+    DBInconsistent(String),
+    DBError(String),
+}
+
+#[derive(Debug, PartialEq)]
+pub enum ValidBlock {
+    /// The block was sucessfully processed.
+    Processed,
+}
+
+#[derive(Debug, PartialEq)]
+pub enum InvalidBlock {
+    /// The block slot is greater than the present slot.
+    FutureSlot,
+    /// The block state_root does not match the generated state.
+    StateRootMismatch,
+    /// The blocks parent_root is unknown.
+    ParentUnknown,
+    /// There was an error whilst advancing the parent state to the present slot. This condition
+    /// should not occur, it likely represents an internal error.
+    SlotProcessingError(SlotProcessingError),
+    /// The block could not be applied to the state, it is invalid.
+    PerBlockProcessingError(BlockProcessingError),
+}
+
+#[derive(Debug, PartialEq)]
+pub enum BlockProcessingOutcome {
+    /// The block was sucessfully validated.
+    ValidBlock(ValidBlock),
+    /// The block was not sucessfully validated.
+    InvalidBlock(InvalidBlock),
+}
+
+pub struct BeaconChain<T: ClientDB + Sized, U: SlotClock> {
+    pub block_store: Arc<BeaconBlockStore<T>>,
+    pub state_store: Arc<BeaconStateStore<T>>,
+    pub slot_clock: U,
+    pub block_graph: BlockGraph,
+    pub attestation_aggregator: RwLock<AttestationAggregator>,
+    canonical_head: RwLock<CheckPoint>,
+    finalized_head: RwLock<CheckPoint>,
+    justified_head: RwLock<CheckPoint>,
+    pub state: RwLock<BeaconState>,
+    pub latest_attestation_targets: RwLock<AttestationTargets>,
+    pub spec: ChainSpec,
+}
+
+impl<T, U> BeaconChain<T, U>
+where
+    T: ClientDB,
+    U: SlotClock,
+{
+    /// Instantiate a new Beacon Chain, from genesis.
+    pub fn genesis(
+        state_store: Arc<BeaconStateStore<T>>,
+        block_store: Arc<BeaconBlockStore<T>>,
+        slot_clock: U,
+        spec: ChainSpec,
+    ) -> Result<Self, Error> {
+        if spec.initial_validators.is_empty() {
+            return Err(Error::InsufficientValidators);
+        }
+
+        let genesis_state = genesis_beacon_state(&spec);
+        let state_root = genesis_state.canonical_root();
+        state_store.put(&state_root, &ssz_encode(&genesis_state)[..])?;
+
+        let genesis_block = genesis_beacon_block(state_root, &spec);
+        let block_root = genesis_block.canonical_root();
+        block_store.put(&block_root, &ssz_encode(&genesis_block)[..])?;
+
+        let block_graph = BlockGraph::new();
+        block_graph.add_leaf(&Hash256::zero(), block_root.clone());
+
+        let finalized_head = RwLock::new(CheckPoint::new(
+            genesis_block.clone(),
+            block_root.clone(),
+            genesis_state.clone(),
+            state_root.clone(),
+        ));
+        let justified_head = RwLock::new(CheckPoint::new(
+            genesis_block.clone(),
+            block_root.clone(),
+            genesis_state.clone(),
+            state_root.clone(),
+        ));
+        let canonical_head = RwLock::new(CheckPoint::new(
+            genesis_block.clone(),
+            block_root.clone(),
+            genesis_state.clone(),
+            state_root.clone(),
+        ));
+        let attestation_aggregator = RwLock::new(AttestationAggregator::new());
+
+        let latest_attestation_targets = RwLock::new(AttestationTargets::new());
+
+        Ok(Self {
+            block_store,
+            state_store,
+            slot_clock,
+            block_graph,
+            attestation_aggregator,
+            state: RwLock::new(genesis_state.clone()),
+            justified_head,
+            finalized_head,
+            canonical_head,
+            latest_attestation_targets,
+            spec: spec,
+        })
+    }
+
+    /// Update the canonical head to some new values.
+    pub fn update_canonical_head(
+        &self,
+        new_beacon_block: BeaconBlock,
+        new_beacon_block_root: Hash256,
+        new_beacon_state: BeaconState,
+        new_beacon_state_root: Hash256,
+    ) {
+        let mut head = self.canonical_head.write();
+        head.update(
+            new_beacon_block,
+            new_beacon_block_root,
+            new_beacon_state,
+            new_beacon_state_root,
+        );
+    }
+
+    /// Returns a read-lock guarded `CheckPoint` struct for reading the head (as chosen by the
+    /// fork-choice rule).
+    ///
+    /// It is important to note that the `beacon_state` returned may not match the present slot. It
+    /// is the state as it was when the head block was recieved, which could be some slots prior to
+    /// now.
+    pub fn head(&self) -> RwLockReadGuard<CheckPoint> {
+        self.canonical_head.read()
+    }
+
+    /// Update the justified head to some new values.
+    pub fn update_finalized_head(
+        &self,
+        new_beacon_block: BeaconBlock,
+        new_beacon_block_root: Hash256,
+        new_beacon_state: BeaconState,
+        new_beacon_state_root: Hash256,
+    ) {
+        let mut finalized_head = self.finalized_head.write();
+        finalized_head.update(
+            new_beacon_block,
+            new_beacon_block_root,
+            new_beacon_state,
+            new_beacon_state_root,
+        );
+    }
+
+    /// Returns a read-lock guarded `CheckPoint` struct for reading the justified head (as chosen,
+    /// indirectly,  by the fork-choice rule).
+    pub fn finalized_head(&self) -> RwLockReadGuard<CheckPoint> {
+        self.finalized_head.read()
+    }
+
+    /// Advance the `self.state` `BeaconState` to the supplied slot.
+    ///
+    /// This will perform per_slot and per_epoch processing as required.
+    ///
+    /// The `previous_block_root` will be set to the root of the current head block (as determined
+    /// by the fork-choice rule).
+    ///
+    /// It is important to note that this is _not_ the state corresponding to the canonical head
+    /// block, instead it is that state which may or may not have had additional per slot/epoch
+    /// processing applied to it.
+    pub fn advance_state(&self, slot: u64) -> Result<(), SlotProcessingError> {
+        let state_slot = self.state.read().slot;
+        let head_block_root = self.head().beacon_block_root;
+        for _ in state_slot..slot {
+            self.state
+                .write()
+                .per_slot_processing(head_block_root.clone(), &self.spec)?;
+        }
+        Ok(())
+    }
+
+    /// Returns the the validator index (if any) for the given public key.
+    ///
+    /// Information is retrieved from the present `beacon_state.validator_registry`.
+    pub fn validator_index(&self, pubkey: &PublicKey) -> Option<usize> {
+        for (i, validator) in self
+            .head()
+            .beacon_state
+            .validator_registry
+            .iter()
+            .enumerate()
+        {
+            if validator.pubkey == *pubkey {
+                return Some(i);
+            }
+        }
+        None
+    }
+
+    /// Returns the number of slots the validator has been required to propose.
+    ///
+    /// Returns `None` if the `validator_index` is invalid.
+    ///
+    /// Information is retrieved from the present `beacon_state.validator_registry`.
+    pub fn proposer_slots(&self, validator_index: usize) -> Option<u64> {
+        if let Some(validator) = self.state.read().validator_registry.get(validator_index) {
+            Some(validator.proposer_slots)
+        } else {
+            None
+        }
+    }
+
+    /// Reads the slot clock, returns `None` if the slot is unavailable.
+    ///
+    /// The slot might be unavailable due to an error with the system clock, or if the present time
+    /// is before genesis (i.e., a negative slot).
+    ///
+    /// This is distinct to `present_slot`, which simply reads the latest state. If a
+    /// call to `read_slot_clock` results in a higher slot than a call to `present_slot`,
+    /// `self.state` should undergo per slot processing.
+    pub fn read_slot_clock(&self) -> Option<u64> {
+        match self.slot_clock.present_slot() {
+            Ok(some_slot) => some_slot,
+            _ => None,
+        }
+    }
+
+    /// Returns slot of the present state.
+    ///
+    /// This is distinct to `read_slot_clock`, which reads from the actual system clock. If
+    /// `self.state` has not been transitioned it is possible for the system clock to be on a
+    /// different slot to what is returned from this call.
+    pub fn present_slot(&self) -> u64 {
+        self.state.read().slot
+    }
+
+    /// Returns the block proposer for a given slot.
+    ///
+    /// Information is read from the present `beacon_state` shuffling, so only information from the
+    /// present and prior epoch is available.
+    pub fn block_proposer(&self, slot: u64) -> Result<usize, CommitteesError> {
+        let index = self
+            .state
+            .read()
+            .get_beacon_proposer_index(slot, &self.spec)?;
+
+        Ok(index)
+    }
+
+    /// Returns the justified slot for the present state.
+    pub fn justified_slot(&self) -> u64 {
+        self.state.read().justified_slot
+    }
+
+    /// Returns the attestation slot and shard for a given validator index.
+    ///
+    /// Information is read from the current state, so only information from the present and prior
+    /// epoch is available.
+    pub fn validator_attestion_slot_and_shard(
+        &self,
+        validator_index: usize,
+    ) -> Result<Option<(u64, u64)>, CommitteesError> {
+        if let Some((slot, shard, _committee)) = self
+            .state
+            .read()
+            .attestation_slot_and_shard_for_validator(validator_index, &self.spec)?
+        {
+            Ok(Some((slot, shard)))
+        } else {
+            Ok(None)
+        }
+    }
+
+    /// Produce an `AttestationData` that is valid for the present `slot` and given `shard`.
+    pub fn produce_attestation_data(&self, shard: u64) -> Result<AttestationData, Error> {
+        let justified_slot = self.justified_slot();
+        let justified_block_root = self
+            .state
+            .read()
+            .get_block_root(justified_slot, &self.spec)
+            .ok_or_else(|| Error::BadRecentBlockRoots)?
+            .clone();
+
+        let epoch_boundary_root = self
+            .state
+            .read()
+            .get_block_root(
+                self.state.read().current_epoch_start_slot(&self.spec),
+                &self.spec,
+            )
+            .ok_or_else(|| Error::BadRecentBlockRoots)?
+            .clone();
+
+        Ok(AttestationData {
+            slot: self.state.read().slot,
+            shard,
+            beacon_block_root: self.head().beacon_block_root.clone(),
+            epoch_boundary_root,
+            shard_block_root: Hash256::zero(),
+            latest_crosslink_root: Hash256::zero(),
+            justified_slot,
+            justified_block_root,
+        })
+    }
+
+    /// Validate a `FreeAttestation` and either:
+    ///
+    /// - Create a new `Attestation`.
+    /// - Aggregate it to an existing `Attestation`.
+    pub fn process_free_attestation(
+        &self,
+        free_attestation: FreeAttestation,
+    ) -> Result<AggregationOutcome, Error> {
+        self.attestation_aggregator
+            .write()
+            .process_free_attestation(&self.state.read(), &free_attestation, &self.spec)
+            .map_err(|e| e.into())
+    }
+
+    /// Set the latest attestation target for some validator.
+    pub fn insert_latest_attestation_target(&self, validator_index: u64, block_root: Hash256) {
+        let mut targets = self.latest_attestation_targets.write();
+        targets.insert(validator_index, block_root);
+    }
+
+    /// Get the latest attestation target for some validator.
+    pub fn get_latest_attestation_target(&self, validator_index: u64) -> Option<Hash256> {
+        let targets = self.latest_attestation_targets.read();
+
+        match targets.get(validator_index) {
+            Some(hash) => Some(hash.clone()),
+            None => None,
+        }
+    }
+
+    /// Dumps the entire canonical chain, from the head to genesis to a vector for analysis.
+    ///
+    /// This could be a very expensive operation and should only be done in testing/analysis
+    /// activities.
+    pub fn chain_dump(&self) -> Result<Vec<CheckPoint>, Error> {
+        let mut dump = vec![];
+
+        let mut last_slot = CheckPoint {
+            beacon_block: self.head().beacon_block.clone(),
+            beacon_block_root: self.head().beacon_block_root,
+            beacon_state: self.head().beacon_state.clone(),
+            beacon_state_root: self.head().beacon_state_root,
+        };
+
+        dump.push(last_slot.clone());
+
+        loop {
+            let beacon_block_root = last_slot.beacon_block.parent_root;
+
+            if beacon_block_root == self.spec.zero_hash {
+                break; // Genesis has been reached.
+            }
+
+            let beacon_block = self
+                .block_store
+                .get_deserialized(&beacon_block_root)?
+                .ok_or_else(|| {
+                    Error::DBInconsistent(format!("Missing block {}", beacon_block_root))
+                })?;
+            let beacon_state_root = beacon_block.state_root;
+            let beacon_state = self
+                .state_store
+                .get_deserialized(&beacon_state_root)?
+                .ok_or_else(|| {
+                    Error::DBInconsistent(format!("Missing state {}", beacon_state_root))
+                })?;
+
+            let slot = CheckPoint {
+                beacon_block,
+                beacon_block_root,
+                beacon_state,
+                beacon_state_root,
+            };
+
+            dump.push(slot.clone());
+            last_slot = slot;
+        }
+
+        Ok(dump)
+    }
+
+    /// Accept some block and attempt to add it to block DAG.
+    ///
+    /// Will accept blocks from prior slots, however it will reject any block from a future slot.
+    pub fn process_block(&self, block: BeaconBlock) -> Result<BlockProcessingOutcome, Error> {
+        debug!("Processing block with slot {}...", block.slot());
+
+        let block_root = block.canonical_root();
+
+        let present_slot = self.present_slot();
+
+        if block.slot > present_slot {
+            return Ok(BlockProcessingOutcome::InvalidBlock(
+                InvalidBlock::FutureSlot,
+            ));
+        }
+
+        // Load the blocks parent block from the database, returning invalid if that block is not
+        // found.
+        let parent_block_root = block.parent_root;
+        let parent_block = match self.block_store.get_reader(&parent_block_root)? {
+            Some(parent_root) => parent_root,
+            None => {
+                return Ok(BlockProcessingOutcome::InvalidBlock(
+                    InvalidBlock::ParentUnknown,
+                ));
+            }
+        };
+
+        // Load the parent blocks state from the database, returning an error if it is not found.
+        // It is an error because if know the parent block we should also know the parent state.
+        let parent_state_root = parent_block.state_root();
+        let parent_state = self
+            .state_store
+            .get_reader(&parent_state_root)?
+            .ok_or(Error::DBInconsistent(format!(
+                "Missing state {}",
+                parent_state_root
+            )))?
+            .into_beacon_state()
+            .ok_or(Error::DBInconsistent(format!(
+                "State SSZ invalid {}",
+                parent_state_root
+            )))?;
+
+        // TODO: check the block proposer signature BEFORE doing a state transition. This will
+        // significantly lower exposure surface to DoS attacks.
+
+        // Transition the parent state to the present slot.
+        let mut state = parent_state;
+        for _ in state.slot..present_slot {
+            if let Err(e) = state.per_slot_processing(parent_block_root.clone(), &self.spec) {
+                return Ok(BlockProcessingOutcome::InvalidBlock(
+                    InvalidBlock::SlotProcessingError(e),
+                ));
+            }
+        }
+
+        // Apply the recieved block to its parent state (which has been transitioned into this
+        // slot).
+        if let Err(e) = state.per_block_processing(&block, &self.spec) {
+            return Ok(BlockProcessingOutcome::InvalidBlock(
+                InvalidBlock::PerBlockProcessingError(e),
+            ));
+        }
+
+        let state_root = state.canonical_root();
+
+        if block.state_root != state_root {
+            return Ok(BlockProcessingOutcome::InvalidBlock(
+                InvalidBlock::StateRootMismatch,
+            ));
+        }
+
+        // Store the block and state.
+        self.block_store.put(&block_root, &ssz_encode(&block)[..])?;
+        self.state_store.put(&state_root, &ssz_encode(&state)[..])?;
+
+        // Update the block DAG.
+        self.block_graph
+            .add_leaf(&parent_block_root, block_root.clone());
+
+        // If the parent block was the parent_block, automatically update the canonical head.
+        //
+        // TODO: this is a first-in-best-dressed scenario that is not ideal; fork_choice should be
+        // run instead.
+        if self.head().beacon_block_root == parent_block_root {
+            self.update_canonical_head(
+                block.clone(),
+                block_root.clone(),
+                state.clone(),
+                state_root.clone(),
+            );
+            // Update the local state variable.
+            *self.state.write() = state.clone();
+        }
+
+        Ok(BlockProcessingOutcome::ValidBlock(ValidBlock::Processed))
+    }
+
+    /// Produce a new block at the present slot.
+    ///
+    /// The produced block will not be inheriently valid, it must be signed by a block producer.
+    /// Block signing is out of the scope of this function and should be done by a separate program.
+    pub fn produce_block(&self, randao_reveal: Signature) -> Option<(BeaconBlock, BeaconState)> {
+        debug!("Producing block at slot {}...", self.state.read().slot);
+
+        let mut state = self.state.read().clone();
+
+        trace!("Finding attestations for new block...");
+
+        let attestations = self
+            .attestation_aggregator
+            .read()
+            .get_attestations_for_state(&state, &self.spec);
+
+        trace!(
+            "Inserting {} attestation(s) into new block.",
+            attestations.len()
+        );
+
+        let parent_root = state
+            .get_block_root(state.slot.saturating_sub(1), &self.spec)?
+            .clone();
+
+        let mut block = BeaconBlock {
+            slot: state.slot,
+            parent_root,
+            state_root: Hash256::zero(), // Updated after the state is calculated.
+            randao_reveal: randao_reveal,
+            eth1_data: Eth1Data {
+                // TODO: replace with real data
+                deposit_root: Hash256::zero(),
+                block_hash: Hash256::zero(),
+            },
+            signature: self.spec.empty_signature.clone(), // To be completed by a validator.
+            body: BeaconBlockBody {
+                proposer_slashings: vec![],
+                casper_slashings: vec![],
+                attestations: attestations,
+                custody_reseeds: vec![],
+                custody_challenges: vec![],
+                custody_responses: vec![],
+                deposits: vec![],
+                exits: vec![],
+            },
+        };
+
+        state
+            .per_block_processing_without_verifying_block_signature(&block, &self.spec)
+            .ok()?;
+
+        let state_root = state.canonical_root();
+
+        block.state_root = state_root;
+
+        trace!("Block produced.");
+
+        Some((block, state))
+    }
+}
+
+impl From<DBError> for Error {
+    fn from(e: DBError) -> Error {
+        Error::DBError(e.message)
+    }
+}
+
+impl From<CommitteesError> for Error {
+    fn from(e: CommitteesError) -> Error {
+        Error::CommitteesError(e)
+    }
+}

beacon_node/beacon_chain/src/block_graph.rs

@@ -0,0 +1,44 @@
+use parking_lot::{RwLock, RwLockReadGuard};
+use std::collections::HashSet;
+use types::Hash256;
+
+/// Maintains a view of the block DAG, also known as the "blockchain" (except, it tracks multiple
+/// chains eminating from a single root instead of just the head of some canonical chain).
+///
+/// The BlockGraph does not store the blocks, instead it tracks the block hashes of blocks at the
+/// tip of the DAG. It is out of the scope of the object to retrieve blocks.
+///
+/// Presently, the DAG root (genesis block) is not tracked.
+///
+/// The BlogGraph is thread-safe due to internal RwLocks.
+pub struct BlockGraph {
+    pub leaves: RwLock<HashSet<Hash256>>,
+}
+
+impl BlockGraph {
+    /// Create a new block graph without any leaves.
+    pub fn new() -> Self {
+        Self {
+            leaves: RwLock::new(HashSet::new()),
+        }
+    }
+    /// Add a new leaf to the block hash graph. Returns `true` if the leaf was built upon another
+    /// leaf.
+    pub fn add_leaf(&self, parent: &Hash256, leaf: Hash256) -> bool {
+        let mut leaves = self.leaves.write();
+
+        if leaves.contains(parent) {
+            leaves.remove(parent);
+            leaves.insert(leaf);
+            true
+        } else {
+            leaves.insert(leaf);
+            false
+        }
+    }
+
+    /// Returns a read-guarded HashSet of all leaf blocks.
+    pub fn leaves(&self) -> RwLockReadGuard<HashSet<Hash256>> {
+        self.leaves.read()
+    }
+}

beacon_node/beacon_chain/src/checkpoint.rs

@@ -0,0 +1,43 @@
+use serde_derive::Serialize;
+use types::{BeaconBlock, BeaconState, Hash256};
+
+/// Represents some block and it's associated state. Generally, this will be used for tracking the
+/// head, justified head and finalized head.
+#[derive(PartialEq, Clone, Serialize)]
+pub struct CheckPoint {
+    pub beacon_block: BeaconBlock,
+    pub beacon_block_root: Hash256,
+    pub beacon_state: BeaconState,
+    pub beacon_state_root: Hash256,
+}
+
+impl CheckPoint {
+    /// Create a new checkpoint.
+    pub fn new(
+        beacon_block: BeaconBlock,
+        beacon_block_root: Hash256,
+        beacon_state: BeaconState,
+        beacon_state_root: Hash256,
+    ) -> Self {
+        Self {
+            beacon_block,
+            beacon_block_root,
+            beacon_state,
+            beacon_state_root,
+        }
+    }
+
+    /// Update all fields of the checkpoint.
+    pub fn update(
+        &mut self,
+        beacon_block: BeaconBlock,
+        beacon_block_root: Hash256,
+        beacon_state: BeaconState,
+        beacon_state_root: Hash256,
+    ) {
+        self.beacon_block = beacon_block;
+        self.beacon_block_root = beacon_block_root;
+        self.beacon_state = beacon_state;
+        self.beacon_state_root = beacon_state_root;
+    }
+}

beacon_node/beacon_chain/src/lib.rs

@@ -0,0 +1,9 @@
+mod attestation_aggregator;
+mod attestation_targets;
+mod beacon_chain;
+mod block_graph;
+mod checkpoint;
+mod lmd_ghost;
+
+pub use self::beacon_chain::{BeaconChain, Error};
+pub use self::checkpoint::CheckPoint;

beacon_node/beacon_chain/src/lmd_ghost.rs

@@ -0,0 +1,196 @@
+use crate::BeaconChain;
+use db::{
+    stores::{BeaconBlockAtSlotError, BeaconBlockStore},
+    ClientDB, DBError,
+};
+use slot_clock::{SlotClock, TestingSlotClockError};
+use std::collections::HashSet;
+use std::sync::Arc;
+use types::{
+    readers::{BeaconBlockReader, BeaconStateReader},
+    validator_registry::get_active_validator_indices,
+    Hash256,
+};
+
+#[derive(Debug, PartialEq)]
+pub enum Error {
+    DBError(String),
+    MissingBeaconState(Hash256),
+    InvalidBeaconState(Hash256),
+    MissingBeaconBlock(Hash256),
+    InvalidBeaconBlock(Hash256),
+}
+
+impl<T, U> BeaconChain<T, U>
+where
+    T: ClientDB,
+    U: SlotClock,
+    Error: From<<U as SlotClock>::Error>,
+{
+    /// Run the fork-choice rule on the current chain, updating the canonical head, if required.
+    pub fn fork_choice(&self) -> Result<(), Error> {
+        let present_head = &self.finalized_head().beacon_block_root;
+
+        let new_head = self.slow_lmd_ghost(&self.finalized_head().beacon_block_root)?;
+
+        if new_head != *present_head {
+            let block = self
+                .block_store
+                .get_deserialized(&new_head)?
+                .ok_or_else(|| Error::MissingBeaconBlock(new_head))?;
+            let block_root = block.canonical_root();
+
+            let state = self
+                .state_store
+                .get_deserialized(&block.state_root)?
+                .ok_or_else(|| Error::MissingBeaconState(block.state_root))?;
+            let state_root = state.canonical_root();
+
+            self.update_canonical_head(block, block_root, state, state_root);
+        }
+
+        Ok(())
+    }
+
+    /// A very inefficient implementation of LMD ghost.
+    pub fn slow_lmd_ghost(&self, start_hash: &Hash256) -> Result<Hash256, Error> {
+        let start = self
+            .block_store
+            .get_reader(&start_hash)?
+            .ok_or(Error::MissingBeaconBlock(*start_hash))?;
+
+        let start_state_root = start.state_root();
+
+        let state = self
+            .state_store
+            .get_reader(&start_state_root)?
+            .ok_or(Error::MissingBeaconState(start_state_root))?
+            .into_beacon_state()
+            .ok_or(Error::InvalidBeaconState(start_state_root))?;
+
+        let active_validator_indices =
+            get_active_validator_indices(&state.validator_registry, start.slot());
+
+        let mut attestation_targets = Vec::with_capacity(active_validator_indices.len());
+        for i in active_validator_indices {
+            if let Some(target) = self.get_latest_attestation_target(i as u64) {
+                attestation_targets.push(target);
+            }
+        }
+
+        let mut head_hash = Hash256::zero();
+        let mut head_vote_count = 0;
+
+        loop {
+            let child_hashes_and_slots = get_child_hashes_and_slots(
+                &self.block_store,
+                &head_hash,
+                &self.block_graph.leaves(),
+            )?;
+
+            if child_hashes_and_slots.len() == 0 {
+                break;
+            }
+
+            for (child_hash, child_slot) in child_hashes_and_slots {
+                let vote_count = get_vote_count(
+                    &self.block_store,
+                    &attestation_targets[..],
+                    &child_hash,
+                    child_slot,
+                )?;
+
+                if vote_count > head_vote_count {
+                    head_hash = child_hash;
+                    head_vote_count = vote_count;
+                }
+            }
+        }
+
+        Ok(head_hash)
+    }
+}
+
+/// Get the total number of votes for some given block root.
+///
+/// The vote count is incrememented each time an attestation target votes for a block root.
+fn get_vote_count<T: ClientDB>(
+    block_store: &Arc<BeaconBlockStore<T>>,
+    attestation_targets: &[Hash256],
+    block_root: &Hash256,
+    slot: u64,
+) -> Result<u64, Error> {
+    let mut count = 0;
+    for target in attestation_targets {
+        let (root_at_slot, _) = block_store
+            .block_at_slot(&block_root, slot)?
+            .ok_or(Error::MissingBeaconBlock(*block_root))?;
+        if root_at_slot == *target {
+            count += 1;
+        }
+    }
+    Ok(count)
+}
+
+/// Starting from some `leaf_hashes`, recurse back down each branch until the `root_hash`, adding
+/// each `block_root` and `slot` to a HashSet.
+fn get_child_hashes_and_slots<T: ClientDB>(
+    block_store: &Arc<BeaconBlockStore<T>>,
+    root_hash: &Hash256,
+    leaf_hashes: &HashSet<Hash256>,
+) -> Result<HashSet<(Hash256, u64)>, Error> {
+    let mut hash_set = HashSet::new();
+
+    for leaf_hash in leaf_hashes {
+        let mut current_hash = *leaf_hash;
+
+        loop {
+            if let Some(block_reader) = block_store.get_reader(&current_hash)? {
+                let parent_root = block_reader.parent_root();
+
+                let new_hash = hash_set.insert((current_hash, block_reader.slot()));
+
+                // If the hash just added was already in the set, break the loop.
+                //
+                // In such a case, the present branch has merged with a branch that is already in
+                // the set.
+                if !new_hash {
+                    break;
+                }
+
+                // The branch is exhausted if the parent of this block is the root_hash.
+                if parent_root == *root_hash {
+                    break;
+                }
+
+                current_hash = parent_root.clone();
+            } else {
+                return Err(Error::MissingBeaconBlock(current_hash));
+            }
+        }
+    }
+
+    Ok(hash_set)
+}
+
+impl From<DBError> for Error {
+    fn from(e: DBError) -> Error {
+        Error::DBError(e.message)
+    }
+}
+
+impl From<BeaconBlockAtSlotError> for Error {
+    fn from(e: BeaconBlockAtSlotError) -> Error {
+        match e {
+            BeaconBlockAtSlotError::UnknownBeaconBlock(h) => Error::MissingBeaconBlock(h),
+            BeaconBlockAtSlotError::InvalidBeaconBlock(h) => Error::InvalidBeaconBlock(h),
+            BeaconBlockAtSlotError::DBError(msg) => Error::DBError(msg),
+        }
+    }
+}
+
+impl From<TestingSlotClockError> for Error {
+    fn from(_: TestingSlotClockError) -> Error {
+        unreachable!(); // Testing clock never throws an error.
+    }
+}

beacon_node/beacon_chain/test_harness/Cargo.toml

@@ -0,0 +1,34 @@
+[package]
+name = "test_harness"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+[[bench]]
+name = "state_transition"
+harness = false
+
+[dev-dependencies]
+criterion = "0.2"
+
+[dependencies]
+attester = { path = "../../../eth2/attester" }
+beacon_chain = { path = "../../beacon_chain" }
+block_producer = { path = "../../../eth2/block_producer" }
+bls = { path = "../../../eth2/utils/bls" }
+boolean-bitfield = { path = "../../../eth2/utils/boolean-bitfield" }
+db = { path = "../../db" }
+parking_lot = "0.7"
+failure = "0.1"
+failure_derive = "0.1"
+genesis = { path = "../../../eth2/genesis" }
+hashing = { path = "../../../eth2/utils/hashing" }
+log = "0.4"
+env_logger = "0.6.0"
+rayon = "1.0"
+serde = "1.0"
+serde_derive = "1.0"
+serde_json = "1.0"
+slot_clock = { path = "../../../eth2/utils/slot_clock" }
+ssz = { path = "../../../eth2/utils/ssz" }
+types = { path = "../../../eth2/types" }

beacon_node/beacon_chain/test_harness/benches/state_transition.rs

@@ -0,0 +1,68 @@
+use criterion::Criterion;
+use criterion::{black_box, criterion_group, criterion_main, Benchmark};
+// use env_logger::{Builder, Env};
+use test_harness::BeaconChainHarness;
+use types::{ChainSpec, Hash256};
+
+fn mid_epoch_state_transition(c: &mut Criterion) {
+    // Builder::from_env(Env::default().default_filter_or("debug")).init();
+
+    let validator_count = 1000;
+    let mut rig = BeaconChainHarness::new(ChainSpec::foundation(), validator_count);
+
+    let epoch_depth = (rig.spec.epoch_length * 2) + (rig.spec.epoch_length / 2);
+
+    for _ in 0..epoch_depth {
+        rig.advance_chain_with_block();
+    }
+
+    let state = rig.beacon_chain.state.read().clone();
+
+    assert!((state.slot + 1) % rig.spec.epoch_length != 0);
+
+    c.bench_function("mid-epoch state transition 10k validators", move |b| {
+        let state = state.clone();
+        b.iter(|| {
+            let mut state = state.clone();
+            black_box(state.per_slot_processing(Hash256::zero(), &rig.spec))
+        })
+    });
+}
+
+fn epoch_boundary_state_transition(c: &mut Criterion) {
+    // Builder::from_env(Env::default().default_filter_or("debug")).init();
+
+    let validator_count = 10000;
+    let mut rig = BeaconChainHarness::new(ChainSpec::foundation(), validator_count);
+
+    let epoch_depth = rig.spec.epoch_length * 2;
+
+    for _ in 0..(epoch_depth - 1) {
+        rig.advance_chain_with_block();
+    }
+
+    let state = rig.beacon_chain.state.read().clone();
+
+    assert_eq!((state.slot + 1) % rig.spec.epoch_length, 0);
+
+    c.bench(
+        "routines",
+        Benchmark::new("routine_1", move |b| {
+            let state = state.clone();
+            b.iter(|| {
+                let mut state = state.clone();
+                black_box(black_box(
+                    state.per_slot_processing(Hash256::zero(), &rig.spec),
+                ))
+            })
+        })
+        .sample_size(5), // sample size is low because function is sloooow.
+    );
+}
+
+criterion_group!(
+    benches,
+    mid_epoch_state_transition,
+    epoch_boundary_state_transition
+);
+criterion_main!(benches);

beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs

@@ -0,0 +1,235 @@
+use super::ValidatorHarness;
+use beacon_chain::BeaconChain;
+pub use beacon_chain::{CheckPoint, Error as BeaconChainError};
+use db::{
+    stores::{BeaconBlockStore, BeaconStateStore},
+    MemoryDB,
+};
+use log::debug;
+use rayon::prelude::*;
+use slot_clock::TestingSlotClock;
+use std::collections::HashSet;
+use std::fs::File;
+use std::io::prelude::*;
+use std::iter::FromIterator;
+use std::sync::Arc;
+use types::{BeaconBlock, ChainSpec, FreeAttestation, Keypair, Validator};
+
+/// The beacon chain harness simulates a single beacon node with `validator_count` validators connected
+/// to it. Each validator is provided a borrow to the beacon chain, where it may read
+/// information and submit blocks/attesations for processing.
+///
+/// This test harness is useful for testing validator and internal state transition logic. It
+/// is not useful for testing that multiple beacon nodes can reach consensus.
+pub struct BeaconChainHarness {
+    pub db: Arc<MemoryDB>,
+    pub beacon_chain: Arc<BeaconChain<MemoryDB, TestingSlotClock>>,
+    pub block_store: Arc<BeaconBlockStore<MemoryDB>>,
+    pub state_store: Arc<BeaconStateStore<MemoryDB>>,
+    pub validators: Vec<ValidatorHarness>,
+    pub spec: Arc<ChainSpec>,
+}
+
+impl BeaconChainHarness {
+    /// Create a new harness with:
+    ///
+    /// - A keypair, `BlockProducer` and `Attester` for each validator.
+    /// - A new BeaconChain struct where the given validators are in the genesis.
+    pub fn new(mut spec: ChainSpec, validator_count: usize) -> Self {
+        let db = Arc::new(MemoryDB::open());
+        let block_store = Arc::new(BeaconBlockStore::new(db.clone()));
+        let state_store = Arc::new(BeaconStateStore::new(db.clone()));
+
+        let slot_clock = TestingSlotClock::new(spec.genesis_slot);
+
+        // Remove the validators present in the spec (if any).
+        spec.initial_validators = Vec::with_capacity(validator_count);
+        spec.initial_balances = Vec::with_capacity(validator_count);
+
+        debug!("Generating validator keypairs...");
+
+        let keypairs: Vec<Keypair> = (0..validator_count)
+            .collect::<Vec<usize>>()
+            .par_iter()
+            .map(|_| Keypair::random())
+            .collect();
+
+        debug!("Creating validator records...");
+
+        spec.initial_validators = keypairs
+            .par_iter()
+            .map(|keypair| Validator {
+                pubkey: keypair.pk.clone(),
+                activation_slot: 0,
+                ..std::default::Default::default()
+            })
+            .collect();
+
+        debug!("Setting validator balances...");
+
+        spec.initial_balances = spec
+            .initial_validators
+            .par_iter()
+            .map(|_| 32_000_000_000) // 32 ETH
+            .collect();
+
+        debug!("Creating the BeaconChain...");
+
+        // Create the Beacon Chain
+        let beacon_chain = Arc::new(
+            BeaconChain::genesis(
+                state_store.clone(),
+                block_store.clone(),
+                slot_clock,
+                spec.clone(),
+            )
+            .unwrap(),
+        );
+
+        let spec = Arc::new(spec);
+
+        debug!("Creating validator producer and attester instances...");
+
+        // Spawn the test validator instances.
+        let validators: Vec<ValidatorHarness> = keypairs
+            .iter()
+            .map(|keypair| {
+                ValidatorHarness::new(keypair.clone(), beacon_chain.clone(), spec.clone())
+            })
+            .collect();
+
+        debug!("Created {} ValidatorHarnesss", validators.len());
+
+        Self {
+            db,
+            beacon_chain,
+            block_store,
+            state_store,
+            validators,
+            spec,
+        }
+    }
+
+    /// Move the `slot_clock` for the `BeaconChain` forward one slot.
+    ///
+    /// This is the equivalent of advancing a system clock forward one `SLOT_DURATION`.
+    ///
+    /// Returns the new slot.
+    pub fn increment_beacon_chain_slot(&mut self) -> u64 {
+        let slot = self.beacon_chain.present_slot() + 1;
+
+        debug!("Incrementing BeaconChain slot to {}.", slot);
+
+        self.beacon_chain.slot_clock.set_slot(slot);
+        self.beacon_chain.advance_state(slot).unwrap();
+        slot
+    }
+
+    /// Gather the `FreeAttestation`s from the valiators.
+    ///
+    /// Note: validators will only produce attestations _once per slot_. So, if you call this twice
+    /// you'll only get attestations on the first run.
+    pub fn gather_free_attesations(&mut self) -> Vec<FreeAttestation> {
+        let present_slot = self.beacon_chain.present_slot();
+
+        let attesting_validators = self
+            .beacon_chain
+            .state
+            .read()
+            .get_crosslink_committees_at_slot(present_slot, &self.spec)
+            .unwrap()
+            .iter()
+            .fold(vec![], |mut acc, (committee, _slot)| {
+                acc.append(&mut committee.clone());
+                acc
+            });
+        let attesting_validators: HashSet<usize> =
+            HashSet::from_iter(attesting_validators.iter().cloned());
+
+        let free_attestations: Vec<FreeAttestation> = self
+            .validators
+            .par_iter_mut()
+            .enumerate()
+            .filter_map(|(i, validator)| {
+                if attesting_validators.contains(&i) {
+                    // Advance the validator slot.
+                    validator.set_slot(present_slot);
+
+                    // Prompt the validator to produce an attestation (if required).
+                    validator.produce_free_attestation().ok()
+                } else {
+                    None
+                }
+            })
+            .collect();
+
+        debug!(
+            "Gathered {} FreeAttestations for slot {}.",
+            free_attestations.len(),
+            present_slot
+        );
+
+        free_attestations
+    }
+
+    /// Get the block from the proposer for the slot.
+    ///
+    /// Note: the validator will only produce it _once per slot_. So, if you call this twice you'll
+    /// only get a block once.
+    pub fn produce_block(&mut self) -> BeaconBlock {
+        let present_slot = self.beacon_chain.present_slot();
+
+        let proposer = self.beacon_chain.block_proposer(present_slot).unwrap();
+
+        debug!(
+            "Producing block from validator #{} for slot {}.",
+            proposer, present_slot
+        );
+
+        // Ensure the validators slot clock is accurate.
+        self.validators[proposer].set_slot(present_slot);
+        self.validators[proposer].produce_block().unwrap()
+    }
+
+    /// Advances the chain with a BeaconBlock and attestations from all validators.
+    ///
+    /// This is the ideal scenario for the Beacon Chain, 100% honest participation from
+    /// validators.
+    pub fn advance_chain_with_block(&mut self) {
+        self.increment_beacon_chain_slot();
+
+        // Produce a new block.
+        let block = self.produce_block();
+        debug!("Submitting block for processing...");
+        self.beacon_chain.process_block(block).unwrap();
+        debug!("...block processed by BeaconChain.");
+
+        debug!("Producing free attestations...");
+
+        // Produce new attestations.
+        let free_attestations = self.gather_free_attesations();
+
+        debug!("Processing free attestations...");
+
+        free_attestations.par_iter().for_each(|free_attestation| {
+            self.beacon_chain
+                .process_free_attestation(free_attestation.clone())
+                .unwrap();
+        });
+
+        debug!("Free attestations processed.");
+    }
+
+    /// Dump all blocks and states from the canonical beacon chain.
+    pub fn chain_dump(&self) -> Result<Vec<CheckPoint>, BeaconChainError> {
+        self.beacon_chain.chain_dump()
+    }
+
+    /// Write the output of `chain_dump` to a JSON file.
+    pub fn dump_to_file(&self, filename: String, chain_dump: &Vec<CheckPoint>) {
+        let json = serde_json::to_string(chain_dump).unwrap();
+        let mut file = File::create(filename).unwrap();
+        file.write_all(json.as_bytes())
+            .expect("Failed writing dump to file.");
+    }
+}

beacon_node/beacon_chain/test_harness/src/lib.rs

@@ -0,0 +1,5 @@
+mod beacon_chain_harness;
+mod validator_harness;
+
+pub use self::beacon_chain_harness::BeaconChainHarness;
+pub use self::validator_harness::ValidatorHarness;

beacon_node/beacon_chain/test_harness/src/validator_harness/direct_beacon_node.rs

@@ -0,0 +1,121 @@
+use attester::{
+    BeaconNode as AttesterBeaconNode, BeaconNodeError as NodeError,
+    PublishOutcome as AttestationPublishOutcome,
+};
+use beacon_chain::BeaconChain;
+use block_producer::{
+    BeaconNode as BeaconBlockNode, BeaconNodeError as BeaconBlockNodeError,
+    PublishOutcome as BlockPublishOutcome,
+};
+use db::ClientDB;
+use parking_lot::RwLock;
+use slot_clock::SlotClock;
+use std::sync::Arc;
+use types::{AttestationData, BeaconBlock, FreeAttestation, PublicKey, Signature};
+
+// mod attester;
+// mod producer;
+
+/// Connect directly to a borrowed `BeaconChain` instance so an attester/producer can request/submit
+/// blocks/attestations.
+///
+/// `BeaconBlock`s and `FreeAttestation`s are not actually published to the `BeaconChain`, instead
+/// they are stored inside this struct. This is to allow one to benchmark the submission of the
+/// block/attestation directly, or modify it before submission.
+pub struct DirectBeaconNode<T: ClientDB, U: SlotClock> {
+    beacon_chain: Arc<BeaconChain<T, U>>,
+    published_blocks: RwLock<Vec<BeaconBlock>>,
+    published_attestations: RwLock<Vec<FreeAttestation>>,
+}
+
+impl<T: ClientDB, U: SlotClock> DirectBeaconNode<T, U> {
+    pub fn new(beacon_chain: Arc<BeaconChain<T, U>>) -> Self {
+        Self {
+            beacon_chain,
+            published_blocks: RwLock::new(vec![]),
+            published_attestations: RwLock::new(vec![]),
+        }
+    }
+
+    /// Get the last published block (if any).
+    pub fn last_published_block(&self) -> Option<BeaconBlock> {
+        Some(self.published_blocks.read().last()?.clone())
+    }
+
+    /// Get the last published attestation (if any).
+    pub fn last_published_free_attestation(&self) -> Option<FreeAttestation> {
+        Some(self.published_attestations.read().last()?.clone())
+    }
+}
+
+impl<T: ClientDB, U: SlotClock> AttesterBeaconNode for DirectBeaconNode<T, U> {
+    fn produce_attestation_data(
+        &self,
+        _slot: u64,
+        shard: u64,
+    ) -> Result<Option<AttestationData>, NodeError> {
+        match self.beacon_chain.produce_attestation_data(shard) {
+            Ok(attestation_data) => Ok(Some(attestation_data)),
+            Err(e) => Err(NodeError::RemoteFailure(format!("{:?}", e))),
+        }
+    }
+
+    fn publish_attestation_data(
+        &self,
+        free_attestation: FreeAttestation,
+    ) -> Result<AttestationPublishOutcome, NodeError> {
+        self.published_attestations.write().push(free_attestation);
+        Ok(AttestationPublishOutcome::ValidAttestation)
+    }
+}
+
+impl<T: ClientDB, U: SlotClock> BeaconBlockNode for DirectBeaconNode<T, U> {
+    /// Requests the `proposer_nonce` from the `BeaconChain`.
+    fn proposer_nonce(&self, pubkey: &PublicKey) -> Result<u64, BeaconBlockNodeError> {
+        let validator_index = self
+            .beacon_chain
+            .validator_index(pubkey)
+            .ok_or_else(|| BeaconBlockNodeError::RemoteFailure("pubkey unknown.".to_string()))?;
+
+        self.beacon_chain
+            .proposer_slots(validator_index)
+            .ok_or_else(|| {
+                BeaconBlockNodeError::RemoteFailure("validator_index unknown.".to_string())
+            })
+    }
+
+    /// Requests a new `BeaconBlock from the `BeaconChain`.
+    fn produce_beacon_block(
+        &self,
+        slot: u64,
+        randao_reveal: &Signature,
+    ) -> Result<Option<BeaconBlock>, BeaconBlockNodeError> {
+        let (block, _state) = self
+            .beacon_chain
+            .produce_block(randao_reveal.clone())
+            .ok_or_else(|| {
+                BeaconBlockNodeError::RemoteFailure(format!("Did not produce block."))
+            })?;
+
+        if block.slot == slot {
+            Ok(Some(block))
+        } else {
+            Err(BeaconBlockNodeError::RemoteFailure(
+                "Unable to produce at non-current slot.".to_string(),
+            ))
+        }
+    }
+
+    /// A block is not _actually_ published to the `BeaconChain`, instead it is stored in the
+    /// `published_block_vec` and a successful `ValidBlock` is returned to the caller.
+    ///
+    /// The block may be retrieved and then applied to the `BeaconChain` manually, potentially in a
+    /// benchmarking scenario.
+    fn publish_beacon_block(
+        &self,
+        block: BeaconBlock,
+    ) -> Result<BlockPublishOutcome, BeaconBlockNodeError> {
+        self.published_blocks.write().push(block);
+        Ok(BlockPublishOutcome::ValidBlock)
+    }
+}

beacon_node/beacon_chain/test_harness/src/validator_harness/direct_duties.rs

@@ -0,0 +1,69 @@
+use attester::{
+    DutiesReader as AttesterDutiesReader, DutiesReaderError as AttesterDutiesReaderError,
+};
+use beacon_chain::BeaconChain;
+use block_producer::{
+    DutiesReader as ProducerDutiesReader, DutiesReaderError as ProducerDutiesReaderError,
+};
+use db::ClientDB;
+use slot_clock::SlotClock;
+use std::sync::Arc;
+use types::PublicKey;
+
+/// Connects directly to a borrowed `BeaconChain` and reads attester/proposer duties directly from
+/// it.
+pub struct DirectDuties<T: ClientDB, U: SlotClock> {
+    beacon_chain: Arc<BeaconChain<T, U>>,
+    pubkey: PublicKey,
+}
+
+impl<T: ClientDB, U: SlotClock> DirectDuties<T, U> {
+    pub fn new(pubkey: PublicKey, beacon_chain: Arc<BeaconChain<T, U>>) -> Self {
+        Self {
+            beacon_chain,
+            pubkey,
+        }
+    }
+}
+
+impl<T: ClientDB, U: SlotClock> ProducerDutiesReader for DirectDuties<T, U> {
+    fn is_block_production_slot(&self, slot: u64) -> Result<bool, ProducerDutiesReaderError> {
+        let validator_index = self
+            .beacon_chain
+            .validator_index(&self.pubkey)
+            .ok_or_else(|| ProducerDutiesReaderError::UnknownValidator)?;
+
+        match self.beacon_chain.block_proposer(slot) {
+            Ok(proposer) if proposer == validator_index => Ok(true),
+            Ok(_) => Ok(false),
+            Err(_) => Err(ProducerDutiesReaderError::UnknownEpoch),
+        }
+    }
+}
+
+impl<T: ClientDB, U: SlotClock> AttesterDutiesReader for DirectDuties<T, U> {
+    fn validator_index(&self) -> Option<u64> {
+        match self.beacon_chain.validator_index(&self.pubkey) {
+            Some(index) => Some(index as u64),
+            None => None,
+        }
+    }
+
+    fn attestation_shard(&self, slot: u64) -> Result<Option<u64>, AttesterDutiesReaderError> {
+        if let Some(validator_index) = self.validator_index() {
+            match self
+                .beacon_chain
+                .validator_attestion_slot_and_shard(validator_index as usize)
+            {
+                Ok(Some((attest_slot, attest_shard))) if attest_slot == slot => {
+                    Ok(Some(attest_shard))
+                }
+                Ok(Some(_)) => Ok(None),
+                Ok(None) => Err(AttesterDutiesReaderError::UnknownEpoch),
+                Err(_) => panic!("Error when getting validator attestation shard."),
+            }
+        } else {
+            Err(AttesterDutiesReaderError::UnknownValidator)
+        }
+    }
+}

beacon_node/beacon_chain/test_harness/src/validator_harness/local_signer.rs

@@ -0,0 +1,47 @@
+use attester::Signer as AttesterSigner;
+use block_producer::Signer as BlockProposerSigner;
+use std::sync::RwLock;
+use types::{Keypair, Signature};
+
+/// A test-only struct used to perform signing for a proposer or attester.
+pub struct LocalSigner {
+    keypair: Keypair,
+    should_sign: RwLock<bool>,
+}
+
+impl LocalSigner {
+    /// Produce a new TestSigner with signing enabled by default.
+    pub fn new(keypair: Keypair) -> Self {
+        Self {
+            keypair,
+            should_sign: RwLock::new(true),
+        }
+    }
+
+    /// If set to `false`, the service will refuse to sign all messages. Otherwise, all messages
+    /// will be signed.
+    pub fn enable_signing(&self, enabled: bool) {
+        *self.should_sign.write().unwrap() = enabled;
+    }
+
+    /// Sign some message.
+    fn bls_sign(&self, message: &[u8]) -> Option<Signature> {
+        Some(Signature::new(message, &self.keypair.sk))
+    }
+}
+
+impl BlockProposerSigner for LocalSigner {
+    fn sign_block_proposal(&self, message: &[u8]) -> Option<Signature> {
+        self.bls_sign(message)
+    }
+
+    fn sign_randao_reveal(&self, message: &[u8]) -> Option<Signature> {
+        self.bls_sign(message)
+    }
+}
+
+impl AttesterSigner for LocalSigner {
+    fn sign_attestation_message(&self, message: &[u8]) -> Option<Signature> {
+        self.bls_sign(message)
+    }
+}

beacon_node/beacon_chain/test_harness/src/validator_harness/mod.rs

@@ -0,0 +1,6 @@
+mod direct_beacon_node;
+mod direct_duties;
+mod local_signer;
+mod validator_harness;
+
+pub use self::validator_harness::ValidatorHarness;

beacon_node/beacon_chain/test_harness/src/validator_harness/validator_harness.rs

@@ -0,0 +1,133 @@
+use super::direct_beacon_node::DirectBeaconNode;
+use super::direct_duties::DirectDuties;
+use super::local_signer::LocalSigner;
+use attester::PollOutcome as AttestationPollOutcome;
+use attester::{Attester, Error as AttestationPollError};
+use beacon_chain::BeaconChain;
+use block_producer::PollOutcome as BlockPollOutcome;
+use block_producer::{BlockProducer, Error as BlockPollError};
+use db::MemoryDB;
+use slot_clock::TestingSlotClock;
+use std::sync::Arc;
+use types::{BeaconBlock, ChainSpec, FreeAttestation, Keypair};
+
+#[derive(Debug, PartialEq)]
+pub enum BlockProduceError {
+    DidNotProduce(BlockPollOutcome),
+    PollError(BlockPollError),
+}
+
+#[derive(Debug, PartialEq)]
+pub enum AttestationProduceError {
+    DidNotProduce(AttestationPollOutcome),
+    PollError(AttestationPollError),
+}
+
+/// A `BlockProducer` and `Attester` which sign using a common keypair.
+///
+/// The test validator connects directly to a borrowed `BeaconChain` struct. It is useful for
+/// testing that the core proposer and attester logic is functioning. Also for supporting beacon
+/// chain tests.
+pub struct ValidatorHarness {
+    pub block_producer: BlockProducer<
+        TestingSlotClock,
+        DirectBeaconNode<MemoryDB, TestingSlotClock>,
+        DirectDuties<MemoryDB, TestingSlotClock>,
+        LocalSigner,
+    >,
+    pub attester: Attester<
+        TestingSlotClock,
+        DirectBeaconNode<MemoryDB, TestingSlotClock>,
+        DirectDuties<MemoryDB, TestingSlotClock>,
+        LocalSigner,
+    >,
+    pub spec: Arc<ChainSpec>,
+    pub epoch_map: Arc<DirectDuties<MemoryDB, TestingSlotClock>>,
+    pub keypair: Keypair,
+    pub beacon_node: Arc<DirectBeaconNode<MemoryDB, TestingSlotClock>>,
+    pub slot_clock: Arc<TestingSlotClock>,
+    pub signer: Arc<LocalSigner>,
+}
+
+impl ValidatorHarness {
+    /// Create a new ValidatorHarness that signs with the given keypair, operates per the given spec and connects to the
+    /// supplied beacon node.
+    ///
+    /// A `BlockProducer` and `Attester` is created..
+    pub fn new(
+        keypair: Keypair,
+        beacon_chain: Arc<BeaconChain<MemoryDB, TestingSlotClock>>,
+        spec: Arc<ChainSpec>,
+    ) -> Self {
+        let slot_clock = Arc::new(TestingSlotClock::new(spec.genesis_slot));
+        let signer = Arc::new(LocalSigner::new(keypair.clone()));
+        let beacon_node = Arc::new(DirectBeaconNode::new(beacon_chain.clone()));
+        let epoch_map = Arc::new(DirectDuties::new(keypair.pk.clone(), beacon_chain.clone()));
+
+        let block_producer = BlockProducer::new(
+            spec.clone(),
+            keypair.pk.clone(),
+            epoch_map.clone(),
+            slot_clock.clone(),
+            beacon_node.clone(),
+            signer.clone(),
+        );
+
+        let attester = Attester::new(
+            epoch_map.clone(),
+            slot_clock.clone(),
+            beacon_node.clone(),
+            signer.clone(),
+        );
+
+        Self {
+            block_producer,
+            attester,
+            spec,
+            epoch_map,
+            keypair,
+            beacon_node,
+            slot_clock,
+            signer,
+        }
+    }
+
+    /// Run the `poll` function on the `BlockProducer` and produce a block.
+    ///
+    /// An error is returned if the producer refuses to produce.
+    pub fn produce_block(&mut self) -> Result<BeaconBlock, BlockProduceError> {
+        // Using `DirectBeaconNode`, the validator will always return sucessufully if it tries to
+        // publish a block.
+        match self.block_producer.poll() {
+            Ok(BlockPollOutcome::BlockProduced(_)) => {}
+            Ok(outcome) => return Err(BlockProduceError::DidNotProduce(outcome)),
+            Err(error) => return Err(BlockProduceError::PollError(error)),
+        };
+        Ok(self
+            .beacon_node
+            .last_published_block()
+            .expect("Unable to obtain produced block."))
+    }
+
+    /// Run the `poll` function on the `Attester` and produce a `FreeAttestation`.
+    ///
+    /// An error is returned if the attester refuses to attest.
+    pub fn produce_free_attestation(&mut self) -> Result<FreeAttestation, AttestationProduceError> {
+        match self.attester.poll() {
+            Ok(AttestationPollOutcome::AttestationProduced(_)) => {}
+            Ok(outcome) => return Err(AttestationProduceError::DidNotProduce(outcome)),
+            Err(error) => return Err(AttestationProduceError::PollError(error)),
+        };
+        Ok(self
+            .beacon_node
+            .last_published_free_attestation()
+            .expect("Unable to obtain produced attestation."))
+    }
+
+    /// Set the validators slot clock to the specified slot.
+    ///
+    /// The validators slot clock will always read this value until it is set to something else.
+    pub fn set_slot(&mut self, slot: u64) {
+        self.slot_clock.set_slot(slot)
+    }
+}

beacon_node/beacon_chain/test_harness/tests/chain.rs

@@ -0,0 +1,47 @@
+use env_logger::{Builder, Env};
+use log::debug;
+use test_harness::BeaconChainHarness;
+use types::ChainSpec;
+
+#[test]
+#[ignore]
+fn it_can_build_on_genesis_block() {
+    let mut spec = ChainSpec::foundation();
+    spec.genesis_slot = spec.epoch_length * 8;
+
+    /*
+    spec.shard_count = spec.shard_count / 8;
+    spec.target_committee_size = spec.target_committee_size / 8;
+    */
+    let validator_count = 1000;
+
+    let mut harness = BeaconChainHarness::new(spec, validator_count as usize);
+
+    harness.advance_chain_with_block();
+}
+
+#[test]
+#[ignore]
+fn it_can_produce_past_first_epoch_boundary() {
+    Builder::from_env(Env::default().default_filter_or("debug")).init();
+
+    let validator_count = 100;
+
+    debug!("Starting harness build...");
+
+    let mut harness = BeaconChainHarness::new(ChainSpec::foundation(), validator_count);
+
+    debug!("Harness built, tests starting..");
+
+    let blocks = harness.spec.epoch_length * 3 + 1;
+
+    for i in 0..blocks {
+        harness.advance_chain_with_block();
+        debug!("Produced block {}/{}.", i, blocks);
+    }
+    let dump = harness.chain_dump().expect("Chain dump failed.");
+
+    assert_eq!(dump.len() as u64, blocks + 1); // + 1 for genesis block.
+
+    harness.dump_to_file("/tmp/chaindump.json".to_string(), &dump);
+}

beacon_node/db/src/stores/beacon_block_store.rs

@@ -6,8 +6,8 @@ use types::{readers::BeaconBlockReader, BeaconBlock, Hash256};
 
 #[derive(Clone, Debug, PartialEq)]
 pub enum BeaconBlockAtSlotError {
-    UnknownBeaconBlock,
+    UnknownBeaconBlock(Hash256),
-    InvalidBeaconBlock,
+    InvalidBeaconBlock(Hash256),
     DBError(String),
 }
 
@@ -26,6 +26,18 @@ impl<T: ClientDB> BeaconBlockStore<T> {
         Self { db }
     }
 
+    pub fn get_deserialized(&self, hash: &Hash256) -> Result<Option<BeaconBlock>, DBError> {
+        match self.get(&hash)? {
+            None => Ok(None),
+            Some(ssz) => {
+                let (block, _) = BeaconBlock::ssz_decode(&ssz, 0).map_err(|_| DBError {
+                    message: "Bad BeaconBlock SSZ.".to_string(),
+                })?;
+                Ok(Some(block))
+            }
+        }
+    }
+
     /// Retuns an object implementing `BeaconBlockReader`, or `None` (if hash not known).
     ///
     /// Note: Presently, this function fully deserializes a `BeaconBlock` and returns that. In the
@@ -73,7 +85,7 @@ impl<T: ClientDB> BeaconBlockStore<T> {
                     current_hash = block_reader.parent_root();
                 }
             } else {
-                break Err(BeaconBlockAtSlotError::UnknownBeaconBlock);
+                break Err(BeaconBlockAtSlotError::UnknownBeaconBlock(current_hash));
             }
         }
     }
@@ -145,7 +157,7 @@ mod tests {
         db.put(DB_COLUMN, hash, ssz).unwrap();
         assert_eq!(
             store.block_at_slot(other_hash, 42),
-            Err(BeaconBlockAtSlotError::UnknownBeaconBlock)
+            Err(BeaconBlockAtSlotError::UnknownBeaconBlock(*other_hash))
         );
     }
 
@@ -243,7 +255,11 @@ mod tests {
         let ssz = bs.block_at_slot(&hashes[4], 6).unwrap();
         assert_eq!(ssz, None);
 
-        let ssz = bs.block_at_slot(&Hash256::from("unknown".as_bytes()), 2);
+        let bad_hash = &Hash256::from("unknown".as_bytes());
-        assert_eq!(ssz, Err(BeaconBlockAtSlotError::UnknownBeaconBlock));
+        let ssz = bs.block_at_slot(bad_hash, 2);
+        assert_eq!(
+            ssz,
+            Err(BeaconBlockAtSlotError::UnknownBeaconBlock(*bad_hash))
+        );
     }
 }

beacon_node/db/src/stores/beacon_state_store.rs

@@ -19,6 +19,18 @@ impl<T: ClientDB> BeaconStateStore<T> {
         Self { db }
     }
 
+    pub fn get_deserialized(&self, hash: &Hash256) -> Result<Option<BeaconState>, DBError> {
+        match self.get(&hash)? {
+            None => Ok(None),
+            Some(ssz) => {
+                let (state, _) = BeaconState::ssz_decode(&ssz, 0).map_err(|_| DBError {
+                    message: "Bad State SSZ.".to_string(),
+                })?;
+                Ok(Some(state))
+            }
+        }
+    }
+
     /// Retuns an object implementing `BeaconStateReader`, or `None` (if hash not known).
     ///
     /// Note: Presently, this function fully deserializes a `BeaconState` and returns that. In the

beacon_node/db/src/stores/mod.rs

@@ -12,8 +12,6 @@ pub use self::beacon_state_store::BeaconStateStore;
 pub use self::pow_chain_store::PoWChainStore;
 pub use self::validator_store::{ValidatorStore, ValidatorStoreError};
 
-use super::bls;
-
 pub const BLOCKS_DB_COLUMN: &str = "blocks";
 pub const STATES_DB_COLUMN: &str = "states";
 pub const POW_CHAIN_DB_COLUMN: &str = "powchain";

beacon_node/db/src/stores/validator_store.rs

@@ -1,9 +1,9 @@
 extern crate bytes;
 
 use self::bytes::{BufMut, BytesMut};
-use super::bls::PublicKey;
 use super::VALIDATOR_DB_COLUMN as DB_COLUMN;
 use super::{ClientDB, DBError};
+use bls::PublicKey;
 use ssz::{ssz_encode, Decodable};
 use std::sync::Arc;
 
@@ -80,8 +80,8 @@ impl<T: ClientDB> ValidatorStore<T> {
 #[cfg(test)]
 mod tests {
     use super::super::super::MemoryDB;
-    use super::super::bls::Keypair;
     use super::*;
+    use bls::Keypair;
 
     #[test]
     fn test_prefix_bytes() {

beacon_node/src/beacon_chain/block_processing.rs

@@ -1,72 +0,0 @@
-use super::{BeaconChain, ClientDB, DBError, SlotClock};
-use slot_clock::TestingSlotClockError;
-use ssz::{ssz_encode, Encodable};
-use types::{readers::BeaconBlockReader, Hash256};
-
-#[derive(Debug, PartialEq)]
-pub enum Outcome {
-    FutureSlot,
-    Processed,
-
-    NewCanonicalBlock,
-    NewReorgBlock,
-    NewForkBlock,
-}
-
-#[derive(Debug, PartialEq)]
-pub enum Error {
-    DBError(String),
-    NotImplemented,
-    PresentSlotIsNone,
-}
-
-impl<T, U> BeaconChain<T, U>
-where
-    T: ClientDB,
-    U: SlotClock,
-    Error: From<<U as SlotClock>::Error>,
-{
-    pub fn process_block<V>(&mut self, block: &V) -> Result<(Outcome, Hash256), Error>
-    where
-        V: BeaconBlockReader + Encodable + Sized,
-    {
-        let block_root = block.canonical_root();
-
-        let present_slot = self
-            .slot_clock
-            .present_slot()?
-            .ok_or(Error::PresentSlotIsNone)?;
-
-        // Block from future slots (i.e., greater than the present slot) should not be processed.
-        if block.slot() > present_slot {
-            return Ok((Outcome::FutureSlot, block_root));
-        }
-
-        // TODO: block processing has been removed.
-        // https://github.com/sigp/lighthouse/issues/98
-
-        // Update leaf blocks.
-        self.block_store.put(&block_root, &ssz_encode(block)[..])?;
-        if self.leaf_blocks.contains(&block.parent_root()) {
-            self.leaf_blocks.remove(&block.parent_root());
-        }
-        if self.canonical_leaf_block == block.parent_root() {
-            self.canonical_leaf_block = block_root;
-        }
-        self.leaf_blocks.insert(block_root);
-
-        Ok((Outcome::Processed, block_root))
-    }
-}
-
-impl From<DBError> for Error {
-    fn from(e: DBError) -> Error {
-        Error::DBError(e.message)
-    }
-}
-
-impl From<TestingSlotClockError> for Error {
-    fn from(_: TestingSlotClockError) -> Error {
-        unreachable!(); // Testing clock never throws an error.
-    }
-}

beacon_node/src/beacon_chain/block_production.rs

@@ -1,75 +0,0 @@
-use super::{BeaconChain, ClientDB, DBError, SlotClock};
-use slot_clock::TestingSlotClockError;
-use types::{
-    readers::{BeaconBlockReader, BeaconStateReader},
-    BeaconBlock, BeaconState, Hash256,
-};
-
-#[derive(Debug, PartialEq)]
-pub enum Error {
-    DBError(String),
-    PresentSlotIsNone,
-}
-
-impl<T, U> BeaconChain<T, U>
-where
-    T: ClientDB,
-    U: SlotClock,
-    Error: From<<U as SlotClock>::Error>,
-{
-    pub fn produce_block(&mut self) -> Result<(BeaconBlock, BeaconState), Error> {
-        /*
-         * Important: this code is a big stub and only exists to ensure that tests pass.
-         *
-         * https://github.com/sigp/lighthouse/issues/107
-         */
-        let present_slot = self
-            .slot_clock
-            .present_slot()?
-            .ok_or(Error::PresentSlotIsNone)?;
-        let parent_root = self.canonical_leaf_block;
-        let parent_block_reader = self
-            .block_store
-            .get_reader(&parent_root)?
-            .ok_or_else(|| Error::DBError("Block not found.".to_string()))?;
-        let parent_state_reader = self
-            .state_store
-            .get_reader(&parent_block_reader.state_root())?
-            .ok_or_else(|| Error::DBError("State not found.".to_string()))?;
-
-        let parent_block = parent_block_reader
-            .into_beacon_block()
-            .ok_or_else(|| Error::DBError("Bad parent block SSZ.".to_string()))?;
-        let mut block = BeaconBlock {
-            slot: present_slot,
-            parent_root,
-            state_root: Hash256::zero(), // Updated after the state is calculated.
-            ..parent_block
-        };
-
-        let parent_state = parent_state_reader
-            .into_beacon_state()
-            .ok_or_else(|| Error::DBError("Bad parent block SSZ.".to_string()))?;
-        let state = BeaconState {
-            slot: present_slot,
-            ..parent_state
-        };
-        let state_root = state.canonical_root();
-
-        block.state_root = state_root;
-
-        Ok((block, state))
-    }
-}
-
-impl From<DBError> for Error {
-    fn from(e: DBError) -> Error {
-        Error::DBError(e.message)
-    }
-}
-
-impl From<TestingSlotClockError> for Error {
-    fn from(_: TestingSlotClockError) -> Error {
-        unreachable!(); // Testing clock never throws an error.
-    }
-}

beacon_node/src/beacon_chain/mod.rs

@@ -1,80 +0,0 @@
-mod block_processing;
-mod block_production;
-
-use db::{
-    stores::{BeaconBlockStore, BeaconStateStore},
-    ClientDB, DBError,
-};
-use genesis::{genesis_beacon_block, genesis_beacon_state, GenesisError};
-use slot_clock::SlotClock;
-use ssz::ssz_encode;
-use std::collections::HashSet;
-use std::sync::Arc;
-use types::{ChainSpec, Hash256};
-
-pub use crate::block_processing::Outcome as BlockProcessingOutcome;
-
-#[derive(Debug, PartialEq)]
-pub enum BeaconChainError {
-    InsufficientValidators,
-    GenesisError(GenesisError),
-    DBError(String),
-}
-
-pub struct BeaconChain<T: ClientDB + Sized, U: SlotClock> {
-    pub block_store: Arc<BeaconBlockStore<T>>,
-    pub state_store: Arc<BeaconStateStore<T>>,
-    pub slot_clock: U,
-    pub leaf_blocks: HashSet<Hash256>,
-    pub canonical_leaf_block: Hash256,
-    pub spec: ChainSpec,
-}
-
-impl<T, U> BeaconChain<T, U>
-where
-    T: ClientDB,
-    U: SlotClock,
-{
-    pub fn genesis(
-        state_store: Arc<BeaconStateStore<T>>,
-        block_store: Arc<BeaconBlockStore<T>>,
-        slot_clock: U,
-        spec: ChainSpec,
-    ) -> Result<Self, BeaconChainError> {
-        if spec.initial_validators.is_empty() {
-            return Err(BeaconChainError::InsufficientValidators);
-        }
-
-        let genesis_state = genesis_beacon_state(&spec)?;
-        let state_root = genesis_state.canonical_root();
-        state_store.put(&state_root, &ssz_encode(&genesis_state)[..])?;
-
-        let genesis_block = genesis_beacon_block(state_root, &spec);
-        let block_root = genesis_block.canonical_root();
-        block_store.put(&block_root, &ssz_encode(&genesis_block)[..])?;
-
-        let mut leaf_blocks = HashSet::new();
-        leaf_blocks.insert(block_root);
-
-        Ok(Self {
-            block_store,
-            state_store,
-            slot_clock,
-            leaf_blocks,
-            canonical_leaf_block: block_root,
-            spec,
-        })
-    }
-}
-
-impl From<DBError> for BeaconChainError {
-    fn from(e: DBError) -> BeaconChainError {
-        BeaconChainError::DBError(e.message)
-    }
-}
-
-impl From<GenesisError> for BeaconChainError {
-    fn from(e: GenesisError) -> BeaconChainError {
-        BeaconChainError::GenesisError(e)
-    }
-}

beacon_node/src/beacon_chain/tests/chain_test.rs

@@ -1,49 +0,0 @@
-use chain::{BeaconChain, BlockProcessingOutcome};
-use db::{
-    stores::{BeaconBlockStore, BeaconStateStore},
-    MemoryDB,
-};
-use slot_clock::TestingSlotClock;
-use std::sync::Arc;
-use types::ChainSpec;
-
-fn in_memory_test_stores() -> (
-    Arc<MemoryDB>,
-    Arc<BeaconBlockStore<MemoryDB>>,
-    Arc<BeaconStateStore<MemoryDB>>,
-) {
-    let db = Arc::new(MemoryDB::open());
-    let block_store = Arc::new(BeaconBlockStore::new(db.clone()));
-    let state_store = Arc::new(BeaconStateStore::new(db.clone()));
-    (db, block_store, state_store)
-}
-
-fn in_memory_test_chain(
-    spec: ChainSpec,
-) -> (Arc<MemoryDB>, BeaconChain<MemoryDB, TestingSlotClock>) {
-    let (db, block_store, state_store) = in_memory_test_stores();
-    let slot_clock = TestingSlotClock::new(0);
-
-    let chain = BeaconChain::genesis(state_store, block_store, slot_clock, spec);
-    (db, chain.unwrap())
-}
-
-#[test]
-fn it_constructs() {
-    let (_db, _chain) = in_memory_test_chain(ChainSpec::foundation());
-}
-
-#[test]
-fn it_produces() {
-    let (_db, mut chain) = in_memory_test_chain(ChainSpec::foundation());
-    let (_block, _state) = chain.produce_block().unwrap();
-}
-
-#[test]
-fn it_processes_a_block_it_produces() {
-    let (_db, mut chain) = in_memory_test_chain(ChainSpec::foundation());
-    let (block, _state) = chain.produce_block().unwrap();
-    let (outcome, new_block_hash) = chain.process_block(&block).unwrap();
-    assert_eq!(outcome, BlockProcessingOutcome::Processed);
-    assert_eq!(chain.canonical_leaf_block, new_block_hash);
-}

beacon_node/src/beacon_chain/transition.rs

@@ -1,29 +0,0 @@
-use super::BeaconChain;
-use db::ClientDB;
-use state_transition::{extend_active_state, StateTransitionError};
-use types::{ActiveState, BeaconBlock, CrystallizedState, Hash256};
-
-impl<T, U> BeaconChain<T, U>
-where
-    T: ClientDB + Sized,
-{
-    pub(crate) fn transition_states(
-        &self,
-        act_state: &ActiveState,
-        cry_state: &CrystallizedState,
-        block: &BeaconBlock,
-        block_hash: &Hash256,
-    ) -> Result<(ActiveState, Option<CrystallizedState>), StateTransitionError> {
-        let state_recalc_distance = block
-            .slot
-            .checked_sub(cry_state.last_state_recalculation_slot)
-            .ok_or(StateTransitionError::BlockSlotBeforeRecalcSlot)?;
-
-        if state_recalc_distance >= u64::from(self.spec.epoch_length) {
-            panic!("Not implemented!")
-        } else {
-            let new_act_state = extend_active_state(act_state, block, block_hash)?;
-            Ok((new_act_state, None))
-        }
-    }
-}

beacon_node/src/main.rs

@@ -7,8 +7,16 @@ use std::path::PathBuf;
 
 use crate::config::LighthouseConfig;
 use crate::rpc::start_server;
+use beacon_chain::BeaconChain;
 use clap::{App, Arg};
+use db::{
+    stores::{BeaconBlockStore, BeaconStateStore},
+    MemoryDB,
+};
 use slog::{error, info, o, Drain};
+use slot_clock::SystemTimeSlotClock;
+use std::sync::Arc;
+use types::ChainSpec;
 
 fn main() {
     let decorator = slog_term::TermDecorator::new().build();
@@ -58,6 +66,23 @@ fn main() {
           "data_dir" => &config.data_dir.to_str(),
           "port" => &config.p2p_listen_port);
 
+    // Specification (presently fixed to foundation).
+    let spec = ChainSpec::foundation();
+
+    // Database (presently in-memory)
+    let db = Arc::new(MemoryDB::open());
+    let block_store = Arc::new(BeaconBlockStore::new(db.clone()));
+    let state_store = Arc::new(BeaconStateStore::new(db.clone()));
+
+    // Slot clock
+    let slot_clock = SystemTimeSlotClock::new(spec.genesis_time, spec.slot_duration)
+        .expect("Unable to load SystemTimeSlotClock");
+
+    // Genesis chain
+    // TODO: persist chain to storage.
+    let _chain_result =
+        BeaconChain::genesis(state_store.clone(), block_store.clone(), slot_clock, spec);
+
     let _server = start_server(log.clone());
 
     loop {

eth2/attestation_validation/Cargo.toml

@@ -1,12 +0,0 @@
-[package]
-name = "attestation_validation"
-version = "0.1.0"
-authors = ["Paul Hauner <paul@paulhauner.com>"]
-edition = "2018"
-
-[dependencies]
-bls = { path = "../utils/bls" }
-db = { path = "../../beacon_node/db" }
-hashing = { path = "../utils/hashing" }
-ssz = { path = "../utils/ssz" }
-types = { path = "../types" }

eth2/attestation_validation/src/block_inclusion.rs

@@ -1,246 +0,0 @@
-use super::{Error, Invalid, Outcome};
-
-/// Check that an attestation is valid to be included in some block.
-pub fn validate_attestation_for_block(
-    attestation_slot: u64,
-    block_slot: u64,
-    parent_block_slot: u64,
-    min_attestation_inclusion_delay: u64,
-    epoch_length: u64,
-) -> Result<Outcome, Error> {
-    /*
-     * There is a delay before an attestation may be included in a block, quantified by
-     * `slots` and defined as `min_attestation_inclusion_delay`.
-     *
-     * So, an attestation must be at least `min_attestation_inclusion_delay` slots "older" than the
-     * block it is contained in.
-     */
-    verify_or!(
-        // TODO: this differs from the spec as it does not handle underflows correctly.
-        // https://github.com/sigp/lighthouse/issues/95
-        attestation_slot < block_slot.saturating_sub(min_attestation_inclusion_delay - 1),
-        reject!(Invalid::AttestationTooRecent)
-    );
-
-    /*
-     * A block may not include attestations reference slots more than an epoch length + 1 prior to
-     * the block slot.
-     */
-    verify_or!(
-        attestation_slot >= parent_block_slot.saturating_sub(epoch_length + 1),
-        reject!(Invalid::AttestationTooOld)
-    );
-
-    accept!()
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    /*
-     * Invalid::AttestationTooOld tests.
-     */
-
-    #[test]
-    fn test_inclusion_too_old_minimal() {
-        let min_attestation_inclusion_delay = 10;
-        let epoch_length = 20;
-        let block_slot = 100;
-        let parent_block_slot = block_slot - 1;
-        let attestation_slot = block_slot - min_attestation_inclusion_delay;
-
-        let outcome = validate_attestation_for_block(
-            attestation_slot,
-            block_slot,
-            parent_block_slot,
-            min_attestation_inclusion_delay,
-            epoch_length,
-        );
-        assert_eq!(outcome, Ok(Outcome::Valid));
-    }
-
-    #[test]
-    fn test_inclusion_too_old_maximal() {
-        let min_attestation_inclusion_delay = 10;
-        let epoch_length = 20;
-        let block_slot = 100;
-        let parent_block_slot = block_slot - 1;
-        let attestation_slot = block_slot - epoch_length + 1;
-
-        let outcome = validate_attestation_for_block(
-            attestation_slot,
-            block_slot,
-            parent_block_slot,
-            min_attestation_inclusion_delay,
-            epoch_length,
-        );
-        assert_eq!(outcome, Ok(Outcome::Valid));
-    }
-
-    #[test]
-    fn test_inclusion_too_old_saturating_non_zero_attestation_slot() {
-        let min_attestation_inclusion_delay = 10;
-        let epoch_length = 20;
-        let block_slot = epoch_length + 1;
-        let parent_block_slot = block_slot - 1;
-        let attestation_slot = block_slot - min_attestation_inclusion_delay;
-
-        let outcome = validate_attestation_for_block(
-            attestation_slot,
-            block_slot,
-            parent_block_slot,
-            min_attestation_inclusion_delay,
-            epoch_length,
-        );
-        assert_eq!(outcome, Ok(Outcome::Valid));
-    }
-
-    #[test]
-    fn test_inclusion_too_old_saturating_zero_attestation_slot() {
-        let min_attestation_inclusion_delay = 10;
-        let epoch_length = 20;
-        let block_slot = epoch_length + 1;
-        let parent_block_slot = block_slot - 1;
-        let attestation_slot = 0;
-
-        let outcome = validate_attestation_for_block(
-            attestation_slot,
-            block_slot,
-            parent_block_slot,
-            min_attestation_inclusion_delay,
-            epoch_length,
-        );
-        assert_eq!(outcome, Ok(Outcome::Valid));
-    }
-
-    #[test]
-    fn test_inclusion_too_old() {
-        let min_attestation_inclusion_delay = 10;
-        let epoch_length = 20;
-        let block_slot = epoch_length * 2;
-        let parent_block_slot = block_slot - 1;
-        let attestation_slot = parent_block_slot - (epoch_length + 2);
-
-        let outcome = validate_attestation_for_block(
-            attestation_slot,
-            block_slot,
-            parent_block_slot,
-            min_attestation_inclusion_delay,
-            epoch_length,
-        );
-        assert_eq!(outcome, Ok(Outcome::Invalid(Invalid::AttestationTooOld)));
-    }
-
-    /*
-     * Invalid::AttestationTooRecent tests.
-     */
-
-    #[test]
-    fn test_inclusion_too_recent_minimal() {
-        let parent_block_slot = 99;
-        let min_attestation_inclusion_delay = 10;
-        let epoch_length = 20;
-        let block_slot = 100;
-        let attestation_slot = block_slot - min_attestation_inclusion_delay;
-
-        let outcome = validate_attestation_for_block(
-            attestation_slot,
-            block_slot,
-            parent_block_slot,
-            min_attestation_inclusion_delay,
-            epoch_length,
-        );
-        assert_eq!(outcome, Ok(Outcome::Valid));
-    }
-
-    #[test]
-    fn test_inclusion_too_recent_maximal() {
-        let parent_block_slot = 99;
-        let min_attestation_inclusion_delay = 10;
-        let epoch_length = 20;
-        let block_slot = 100;
-        let attestation_slot = block_slot - epoch_length;
-
-        let outcome = validate_attestation_for_block(
-            attestation_slot,
-            block_slot,
-            parent_block_slot,
-            min_attestation_inclusion_delay,
-            epoch_length,
-        );
-        assert_eq!(outcome, Ok(Outcome::Valid));
-    }
-
-    #[test]
-    fn test_inclusion_too_recent_insufficient() {
-        let parent_block_slot = 99;
-        let min_attestation_inclusion_delay = 10;
-        let epoch_length = 20;
-        let block_slot = 100;
-        let attestation_slot = block_slot - (min_attestation_inclusion_delay - 1);
-
-        let outcome = validate_attestation_for_block(
-            attestation_slot,
-            block_slot,
-            parent_block_slot,
-            min_attestation_inclusion_delay,
-            epoch_length,
-        );
-        assert_eq!(outcome, Ok(Outcome::Invalid(Invalid::AttestationTooRecent)));
-    }
-
-    #[test]
-    fn test_inclusion_too_recent_first_possible_slot() {
-        let min_attestation_inclusion_delay = 10;
-        let epoch_length = 20;
-        let block_slot = min_attestation_inclusion_delay;
-        let attestation_slot = 0;
-        let parent_block_slot = block_slot - 1;
-
-        let outcome = validate_attestation_for_block(
-            attestation_slot,
-            block_slot,
-            parent_block_slot,
-            min_attestation_inclusion_delay,
-            epoch_length,
-        );
-        assert_eq!(outcome, Ok(Outcome::Valid));
-    }
-
-    #[test]
-    fn test_inclusion_too_recent_saturation_non_zero_slot() {
-        let min_attestation_inclusion_delay = 10;
-        let epoch_length = 20;
-        let block_slot = min_attestation_inclusion_delay - 1;
-        let parent_block_slot = block_slot - 1;
-        let attestation_slot = 0;
-
-        let outcome = validate_attestation_for_block(
-            attestation_slot,
-            block_slot,
-            parent_block_slot,
-            min_attestation_inclusion_delay,
-            epoch_length,
-        );
-        assert_eq!(outcome, Ok(Outcome::Invalid(Invalid::AttestationTooRecent)));
-    }
-
-    #[test]
-    fn test_inclusion_too_recent_saturation_zero_slot() {
-        let min_attestation_inclusion_delay = 10;
-        let epoch_length = 20;
-        let block_slot = min_attestation_inclusion_delay - 1;
-        let parent_block_slot = block_slot - 1;
-        let attestation_slot = 0;
-
-        let outcome = validate_attestation_for_block(
-            attestation_slot,
-            block_slot,
-            parent_block_slot,
-            min_attestation_inclusion_delay,
-            epoch_length,
-        );
-        assert_eq!(outcome, Ok(Outcome::Invalid(Invalid::AttestationTooRecent)));
-    }
-}

eth2/attestation_validation/src/enums.rs

@@ -1,37 +0,0 @@
-/// Reasons why an `AttestationRecord` can be invalid.
-#[derive(PartialEq, Debug)]
-pub enum Invalid {
-    AttestationTooRecent,
-    AttestationTooOld,
-    JustifiedSlotImpermissable,
-    JustifiedBlockNotInChain,
-    JustifiedBlockHashMismatch,
-    UnknownShard,
-    ShardBlockHashMismatch,
-    SignatureInvalid,
-}
-
-/// The outcome of validating the `AttestationRecord`.
-///
-/// Distinct from the `Error` enum as an `Outcome` indicates that validation executed sucessfully
-/// and determined the validity `AttestationRecord`.
-#[derive(PartialEq, Debug)]
-pub enum Outcome {
-    Valid,
-    Invalid(Invalid),
-}
-
-/// Errors that prevent this function from correctly validating the `AttestationRecord`.
-///
-/// Distinct from the `Outcome` enum as `Errors` indicate that validation encountered an unexpected
-/// condition and was unable to perform its duty.
-#[derive(PartialEq, Debug)]
-pub enum Error {
-    BlockHasNoParent,
-    BadValidatorIndex,
-    UnableToLookupBlockAtSlot,
-    OutOfBoundsBitfieldIndex,
-    PublicKeyCorrupt,
-    NoPublicKeyForValidator,
-    DBError(String),
-}

eth2/attestation_validation/src/justified_block.rs

@@ -1,80 +0,0 @@
-use super::db::stores::{BeaconBlockAtSlotError, BeaconBlockStore};
-use super::db::ClientDB;
-use super::types::AttestationData;
-use super::types::Hash256;
-use super::{Error, Invalid, Outcome};
-use std::sync::Arc;
-
-/// Verify that a attestation's `data.justified_block_hash` matches the local hash of the block at the
-/// attestation's `data.justified_slot`.
-///
-/// `chain_tip_block_hash` is the tip of the chain in which the justified block hash should exist
-/// locally. As Lightouse stores multiple chains locally, it is possible to have multiple blocks at
-/// the same slot. `chain_tip_block_hash` serves to restrict the lookup to a single chain, where
-/// each slot may have exactly zero or one blocks.
-pub fn validate_attestation_justified_block_hash<T>(
-    data: &AttestationData,
-    chain_tip_block_hash: &Hash256,
-    block_store: &Arc<BeaconBlockStore<T>>,
-) -> Result<Outcome, Error>
-where
-    T: ClientDB + Sized,
-{
-    /*
-     * The `justified_block_hash` in the attestation must match exactly the hash of the block at
-     * that slot in the local chain.
-     *
-     * This condition also infers that the `justified_slot` specified in attestation must exist
-     * locally.
-     */
-    match block_hash_at_slot(chain_tip_block_hash, data.justified_slot, block_store)? {
-        None => reject!(Invalid::JustifiedBlockNotInChain),
-        Some(local_justified_block_hash) => {
-            verify_or!(
-                data.justified_block_root == local_justified_block_hash,
-                reject!(Invalid::JustifiedBlockHashMismatch)
-            );
-        }
-    };
-    accept!()
-}
-
-/// Returns the hash (or None) of a block at a slot in the chain that is specified by
-/// `chain_tip_hash`.
-///
-/// Given that the database stores multiple chains, it is possible for there to be multiple blocks
-/// at the given slot. `chain_tip_hash` specifies exactly which chain should be used.
-fn block_hash_at_slot<T>(
-    chain_tip_hash: &Hash256,
-    slot: u64,
-    block_store: &Arc<BeaconBlockStore<T>>,
-) -> Result<Option<Hash256>, Error>
-where
-    T: ClientDB + Sized,
-{
-    match block_store.block_at_slot(&chain_tip_hash, slot)? {
-        None => Ok(None),
-        Some((hash_bytes, _)) => Ok(Some(Hash256::from(&hash_bytes[..]))),
-    }
-}
-
-impl From<BeaconBlockAtSlotError> for Error {
-    fn from(e: BeaconBlockAtSlotError) -> Self {
-        match e {
-            BeaconBlockAtSlotError::DBError(s) => Error::DBError(s),
-            _ => Error::UnableToLookupBlockAtSlot,
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    /*
-     * TODO: Implement tests.
-     *
-     * These tests will require the `BeaconBlock` and `BeaconBlockBody` updates, which are not
-     * yet included in the code base. Adding tests now will result in duplicated work.
-     *
-     * https://github.com/sigp/lighthouse/issues/97
-     */
-}

eth2/attestation_validation/src/justified_slot.rs

@@ -1,39 +0,0 @@
-use super::types::{AttestationData, BeaconState};
-use super::{Error, Invalid, Outcome};
-
-/// Verify that an attestation's `data.justified_slot` matches the justified slot known to the
-/// `state`.
-///
-/// In the case that an attestation references a slot _before_ the latest state transition, is
-/// acceptable for the attestation to reference the previous known `justified_slot`. If this were
-/// not the case, all attestations created _prior_ to the last state recalculation would be rejected
-/// if a block was justified in that state recalculation. It is both ideal and likely that blocks
-/// will be justified during a state recalcuation.
-pub fn validate_attestation_justified_slot(
-    data: &AttestationData,
-    state: &BeaconState,
-    epoch_length: u64,
-) -> Result<Outcome, Error> {
-    let permissable_justified_slot = if data.slot >= state.slot - (state.slot % epoch_length) {
-        state.justified_slot
-    } else {
-        state.previous_justified_slot
-    };
-    verify_or!(
-        data.justified_slot == permissable_justified_slot,
-        reject!(Invalid::JustifiedSlotImpermissable)
-    );
-    accept!()
-}
-
-#[cfg(test)]
-mod tests {
-    /*
-     * TODO: Implement tests.
-     *
-     * These tests will require the `BeaconBlock` and `BeaconBlockBody` updates, which are not
-     * yet included in the code base. Adding tests now will result in duplicated work.
-     *
-     * https://github.com/sigp/lighthouse/issues/97
-     */
-}

eth2/attestation_validation/src/lib.rs

@@ -1,22 +0,0 @@
-extern crate bls;
-extern crate db;
-extern crate hashing;
-extern crate ssz;
-extern crate types;
-
-#[macro_use]
-mod macros;
-
-mod block_inclusion;
-mod enums;
-mod justified_block;
-mod justified_slot;
-mod shard_block;
-mod signature;
-
-pub use crate::block_inclusion::validate_attestation_for_block;
-pub use crate::enums::{Error, Invalid, Outcome};
-pub use crate::justified_block::validate_attestation_justified_block_hash;
-pub use crate::justified_slot::validate_attestation_justified_slot;
-pub use crate::shard_block::validate_attestation_data_shard_block_hash;
-pub use crate::signature::validate_attestation_signature;

eth2/attestation_validation/src/macros.rs

@@ -1,19 +0,0 @@
-macro_rules! verify_or {
-    ($condition: expr, $result: expr) => {
-        if !$condition {
-            $result
-        }
-    };
-}
-
-macro_rules! reject {
-    ($result: expr) => {
-        return Ok(Outcome::Invalid($result));
-    };
-}
-
-macro_rules! accept {
-    () => {
-        Ok(Outcome::Valid)
-    };
-}

eth2/attestation_validation/src/shard_block.rs

@@ -1,46 +0,0 @@
-use super::db::ClientDB;
-use super::types::{AttestationData, BeaconState};
-use super::{Error, Invalid, Outcome};
-
-/// Check that an attestation is valid with reference to some state.
-pub fn validate_attestation_data_shard_block_hash<T>(
-    data: &AttestationData,
-    state: &BeaconState,
-) -> Result<Outcome, Error>
-where
-    T: ClientDB + Sized,
-{
-    /*
-     * The `shard_block_hash` in the state's `latest_crosslinks` must match either the
-     * `latest_crosslink_hash` or the `shard_block_hash` on the attestation.
-     *
-     * TODO: figure out the reasoning behind this.
-     */
-    match state.latest_crosslinks.get(data.shard as usize) {
-        None => reject!(Invalid::UnknownShard),
-        Some(crosslink) => {
-            let local_shard_block_hash = crosslink.shard_block_root;
-            let shard_block_hash_is_permissable = {
-                (local_shard_block_hash == data.latest_crosslink_root)
-                    || (local_shard_block_hash == data.shard_block_root)
-            };
-            verify_or!(
-                shard_block_hash_is_permissable,
-                reject!(Invalid::ShardBlockHashMismatch)
-            );
-        }
-    };
-    accept!()
-}
-
-#[cfg(test)]
-mod tests {
-    /*
-     * TODO: Implement tests.
-     *
-     * These tests will require the `BeaconBlock` and `BeaconBlockBody` updates, which are not
-     * yet included in the code base. Adding tests now will result in duplicated work.
-     *
-     * https://github.com/sigp/lighthouse/issues/97
-     */
-}

eth2/attestation_validation/src/signature.rs

@@ -1,151 +0,0 @@
-use super::bls::{AggregatePublicKey, AggregateSignature};
-use super::db::stores::{ValidatorStore, ValidatorStoreError};
-use super::db::ClientDB;
-use super::types::{AttestationData, Bitfield, BitfieldError};
-use super::{Error, Invalid, Outcome};
-
-/// Validate that some signature is correct for some attestation data and known validator set.
-pub fn validate_attestation_signature<T>(
-    attestation_data: &AttestationData,
-    participation_bitfield: &Bitfield,
-    aggregate_signature: &AggregateSignature,
-    attestation_indices: &[usize],
-    validator_store: &ValidatorStore<T>,
-) -> Result<Outcome, Error>
-where
-    T: ClientDB + Sized,
-{
-    let mut agg_pub_key = AggregatePublicKey::new();
-
-    for i in 0..attestation_indices.len() {
-        let voted = participation_bitfield.get(i)?;
-        if voted {
-            // De-reference the attestation index into a canonical ValidatorRecord index.
-            let validator = *attestation_indices.get(i).ok_or(Error::BadValidatorIndex)?;
-            // Load the public key.
-            let pub_key = validator_store
-                .get_public_key_by_index(validator)?
-                .ok_or(Error::NoPublicKeyForValidator)?;
-            // Aggregate the public key.
-            agg_pub_key.add(&pub_key.as_raw());
-        }
-    }
-
-    let signed_message = attestation_data_signing_message(attestation_data);
-    verify_or!(
-        // TODO: ensure "domain" for aggregate signatures is included.
-        // https://github.com/sigp/lighthouse/issues/91
-        aggregate_signature.verify(&signed_message, &agg_pub_key),
-        reject!(Invalid::SignatureInvalid)
-    );
-
-    accept!()
-}
-
-fn attestation_data_signing_message(attestation_data: &AttestationData) -> Vec<u8> {
-    let mut signed_message = attestation_data.canonical_root().to_vec();
-    signed_message.append(&mut vec![0]);
-    signed_message
-}
-
-impl From<ValidatorStoreError> for Error {
-    fn from(error: ValidatorStoreError) -> Self {
-        match error {
-            ValidatorStoreError::DBError(s) => Error::DBError(s),
-            ValidatorStoreError::DecodeError => Error::PublicKeyCorrupt,
-        }
-    }
-}
-
-impl From<BitfieldError> for Error {
-    fn from(_error: BitfieldError) -> Self {
-        Error::OutOfBoundsBitfieldIndex
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::super::bls::{Keypair, Signature};
-    use super::super::db::MemoryDB;
-    use super::*;
-    use std::sync::Arc;
-
-    /*
-     * TODO: Test cases are not comprehensive.
-     * https://github.com/sigp/lighthouse/issues/94
-     */
-
-    #[test]
-    fn test_signature_verification() {
-        let attestation_data = AttestationData::zero();
-        let message = attestation_data_signing_message(&attestation_data);
-        let signing_keypairs = vec![
-            Keypair::random(),
-            Keypair::random(),
-            Keypair::random(),
-            Keypair::random(),
-            Keypair::random(),
-            Keypair::random(),
-        ];
-        let non_signing_keypairs = vec![
-            Keypair::random(),
-            Keypair::random(),
-            Keypair::random(),
-            Keypair::random(),
-            Keypair::random(),
-            Keypair::random(),
-        ];
-        /*
-         * Signing keypairs first, then non-signing
-         */
-        let mut all_keypairs = signing_keypairs.clone();
-        all_keypairs.append(&mut non_signing_keypairs.clone());
-
-        let attestation_indices: Vec<usize> = (0..all_keypairs.len()).collect();
-        let mut bitfield = Bitfield::from_elem(all_keypairs.len(), false);
-        for i in 0..signing_keypairs.len() {
-            bitfield.set(i, true).unwrap();
-        }
-
-        let db = Arc::new(MemoryDB::open());
-        let store = ValidatorStore::new(db);
-
-        for (i, keypair) in all_keypairs.iter().enumerate() {
-            store.put_public_key_by_index(i, &keypair.pk).unwrap();
-        }
-
-        let mut agg_sig = AggregateSignature::new();
-        for keypair in &signing_keypairs {
-            let sig = Signature::new(&message, &keypair.sk);
-            agg_sig.add(&sig);
-        }
-
-        /*
-         * Test using all valid parameters.
-         */
-        let outcome = validate_attestation_signature(
-            &attestation_data,
-            &bitfield,
-            &agg_sig,
-            &attestation_indices,
-            &store,
-        )
-        .unwrap();
-        assert_eq!(outcome, Outcome::Valid);
-
-        /*
-         * Add another validator to the bitfield, run validation will all other
-         * parameters the same and assert that it fails.
-         */
-        bitfield.set(signing_keypairs.len() + 1, true).unwrap();
-        let outcome = validate_attestation_signature(
-            &attestation_data,
-            &bitfield,
-            &agg_sig,
-            &attestation_indices,
-            &store,
-        )
-        .unwrap();
-        assert_eq!(outcome, Outcome::Invalid(Invalid::SignatureInvalid));
-    }
-}

eth2/attester/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "attester"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+[dependencies]
+slot_clock = { path = "../../eth2/utils/slot_clock" }
+ssz = { path = "../../eth2/utils/ssz" }
+types = { path = "../../eth2/types" }

eth2/attester/src/lib.rs

@@ -0,0 +1,250 @@
+pub mod test_utils;
+mod traits;
+
+use slot_clock::SlotClock;
+use std::sync::Arc;
+use types::{AttestationData, FreeAttestation, Signature};
+
+pub use self::traits::{
+    BeaconNode, BeaconNodeError, DutiesReader, DutiesReaderError, PublishOutcome, Signer,
+};
+
+const PHASE_0_CUSTODY_BIT: bool = false;
+
+#[derive(Debug, PartialEq)]
+pub enum PollOutcome {
+    AttestationProduced(u64),
+    AttestationNotRequired(u64),
+    SlashableAttestationNotProduced(u64),
+    BeaconNodeUnableToProduceAttestation(u64),
+    ProducerDutiesUnknown(u64),
+    SlotAlreadyProcessed(u64),
+    SignerRejection(u64),
+    ValidatorIsUnknown(u64),
+}
+
+#[derive(Debug, PartialEq)]
+pub enum Error {
+    SlotClockError,
+    SlotUnknowable,
+    EpochMapPoisoned,
+    SlotClockPoisoned,
+    EpochLengthIsZero,
+    BeaconNodeError(BeaconNodeError),
+}
+
+/// A polling state machine which performs block production duties, based upon some epoch duties
+/// (`EpochDutiesMap`) and a concept of time (`SlotClock`).
+///
+/// Ensures that messages are not slashable.
+///
+/// Relies upon an external service to keep the `EpochDutiesMap` updated.
+pub struct Attester<T: SlotClock, U: BeaconNode, V: DutiesReader, W: Signer> {
+    pub last_processed_slot: Option<u64>,
+    duties: Arc<V>,
+    slot_clock: Arc<T>,
+    beacon_node: Arc<U>,
+    signer: Arc<W>,
+}
+
+impl<T: SlotClock, U: BeaconNode, V: DutiesReader, W: Signer> Attester<T, U, V, W> {
+    /// Returns a new instance where `last_processed_slot == 0`.
+    pub fn new(duties: Arc<V>, slot_clock: Arc<T>, beacon_node: Arc<U>, signer: Arc<W>) -> Self {
+        Self {
+            last_processed_slot: None,
+            duties,
+            slot_clock,
+            beacon_node,
+            signer,
+        }
+    }
+}
+
+impl<T: SlotClock, U: BeaconNode, V: DutiesReader, W: Signer> Attester<T, U, V, W> {
+    /// Poll the `BeaconNode` and produce an attestation if required.
+    pub fn poll(&mut self) -> Result<PollOutcome, Error> {
+        let slot = self
+            .slot_clock
+            .present_slot()
+            .map_err(|_| Error::SlotClockError)?
+            .ok_or(Error::SlotUnknowable)?;
+
+        if !self.is_processed_slot(slot) {
+            self.last_processed_slot = Some(slot);
+
+            let shard = match self.duties.attestation_shard(slot) {
+                Ok(Some(result)) => result,
+                Ok(None) => return Ok(PollOutcome::AttestationNotRequired(slot)),
+                Err(DutiesReaderError::UnknownEpoch) => {
+                    return Ok(PollOutcome::ProducerDutiesUnknown(slot));
+                }
+                Err(DutiesReaderError::UnknownValidator) => {
+                    return Ok(PollOutcome::ValidatorIsUnknown(slot));
+                }
+                Err(DutiesReaderError::EpochLengthIsZero) => return Err(Error::EpochLengthIsZero),
+                Err(DutiesReaderError::Poisoned) => return Err(Error::EpochMapPoisoned),
+            };
+
+            self.produce_attestation(slot, shard)
+        } else {
+            Ok(PollOutcome::SlotAlreadyProcessed(slot))
+        }
+    }
+
+    fn produce_attestation(&mut self, slot: u64, shard: u64) -> Result<PollOutcome, Error> {
+        let attestation_data = match self.beacon_node.produce_attestation_data(slot, shard)? {
+            Some(attestation_data) => attestation_data,
+            None => return Ok(PollOutcome::BeaconNodeUnableToProduceAttestation(slot)),
+        };
+
+        if !self.safe_to_produce(&attestation_data) {
+            return Ok(PollOutcome::SlashableAttestationNotProduced(slot));
+        }
+
+        let signature = match self.sign_attestation_data(&attestation_data) {
+            Some(signature) => signature,
+            None => return Ok(PollOutcome::SignerRejection(slot)),
+        };
+
+        let validator_index = match self.duties.validator_index() {
+            Some(validator_index) => validator_index,
+            None => return Ok(PollOutcome::ValidatorIsUnknown(slot)),
+        };
+
+        let free_attestation = FreeAttestation {
+            data: attestation_data,
+            signature,
+            validator_index,
+        };
+
+        self.beacon_node
+            .publish_attestation_data(free_attestation)?;
+        Ok(PollOutcome::AttestationProduced(slot))
+    }
+
+    fn is_processed_slot(&self, slot: u64) -> bool {
+        match self.last_processed_slot {
+            Some(processed_slot) if slot <= processed_slot => true,
+            _ => false,
+        }
+    }
+
+    /// Consumes a block, returning that block signed by the validators private key.
+    ///
+    /// Important: this function will not check to ensure the block is not slashable. This must be
+    /// done upstream.
+    fn sign_attestation_data(&mut self, attestation_data: &AttestationData) -> Option<Signature> {
+        self.store_produce(attestation_data);
+
+        self.signer
+            .sign_attestation_message(&attestation_data.signable_message(PHASE_0_CUSTODY_BIT)[..])
+    }
+
+    /// Returns `true` if signing some attestation_data is safe (non-slashable).
+    ///
+    /// !!! UNSAFE !!!
+    ///
+    /// Important: this function is presently stubbed-out. It provides ZERO SAFETY.
+    fn safe_to_produce(&self, _attestation_data: &AttestationData) -> bool {
+        // TODO: ensure the producer doesn't produce slashable blocks.
+        // https://github.com/sigp/lighthouse/issues/160
+        true
+    }
+
+    /// Record that a block was produced so that slashable votes may not be made in the future.
+    ///
+    /// !!! UNSAFE !!!
+    ///
+    /// Important: this function is presently stubbed-out. It provides ZERO SAFETY.
+    fn store_produce(&mut self, _block: &AttestationData) {
+        // TODO: record this block production to prevent future slashings.
+        // https://github.com/sigp/lighthouse/issues/160
+    }
+}
+
+impl From<BeaconNodeError> for Error {
+    fn from(e: BeaconNodeError) -> Error {
+        Error::BeaconNodeError(e)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::test_utils::{EpochMap, LocalSigner, SimulatedBeaconNode};
+    use super::*;
+    use slot_clock::TestingSlotClock;
+    use types::{
+        test_utils::{SeedableRng, TestRandom, XorShiftRng},
+        ChainSpec, Keypair,
+    };
+
+    // TODO: implement more thorough testing.
+    // https://github.com/sigp/lighthouse/issues/160
+    //
+    // These tests should serve as a good example for future tests.
+
+    #[test]
+    pub fn polling() {
+        let mut rng = XorShiftRng::from_seed([42; 16]);
+
+        let spec = Arc::new(ChainSpec::foundation());
+        let slot_clock = Arc::new(TestingSlotClock::new(0));
+        let beacon_node = Arc::new(SimulatedBeaconNode::default());
+        let signer = Arc::new(LocalSigner::new(Keypair::random()));
+
+        let mut duties = EpochMap::new(spec.epoch_length);
+        let attest_slot = 100;
+        let attest_epoch = attest_slot / spec.epoch_length;
+        let attest_shard = 12;
+        duties.insert_attestation_shard(attest_slot, attest_shard);
+        duties.set_validator_index(Some(2));
+        let duties = Arc::new(duties);
+
+        let mut attester = Attester::new(
+            duties.clone(),
+            slot_clock.clone(),
+            beacon_node.clone(),
+            signer.clone(),
+        );
+
+        // Configure responses from the BeaconNode.
+        beacon_node.set_next_produce_result(Ok(Some(AttestationData::random_for_test(&mut rng))));
+        beacon_node.set_next_publish_result(Ok(PublishOutcome::ValidAttestation));
+
+        // One slot before attestation slot...
+        slot_clock.set_slot(attest_slot - 1);
+        assert_eq!(
+            attester.poll(),
+            Ok(PollOutcome::AttestationNotRequired(attest_slot - 1))
+        );
+
+        // On the attest slot...
+        slot_clock.set_slot(attest_slot);
+        assert_eq!(
+            attester.poll(),
+            Ok(PollOutcome::AttestationProduced(attest_slot))
+        );
+
+        // Trying the same attest slot again...
+        slot_clock.set_slot(attest_slot);
+        assert_eq!(
+            attester.poll(),
+            Ok(PollOutcome::SlotAlreadyProcessed(attest_slot))
+        );
+
+        // One slot after the attest slot...
+        slot_clock.set_slot(attest_slot + 1);
+        assert_eq!(
+            attester.poll(),
+            Ok(PollOutcome::AttestationNotRequired(attest_slot + 1))
+        );
+
+        // In an epoch without known duties...
+        let slot = (attest_epoch + 1) * spec.epoch_length;
+        slot_clock.set_slot(slot);
+        assert_eq!(
+            attester.poll(),
+            Ok(PollOutcome::ProducerDutiesUnknown(slot))
+        );
+    }
+}

eth2/attester/src/test_utils/epoch_map.rs

@@ -0,0 +1,44 @@
+use crate::{DutiesReader, DutiesReaderError};
+use std::collections::HashMap;
+
+pub struct EpochMap {
+    epoch_length: u64,
+    validator_index: Option<u64>,
+    map: HashMap<u64, (u64, u64)>,
+}
+
+impl EpochMap {
+    pub fn new(epoch_length: u64) -> Self {
+        Self {
+            epoch_length,
+            validator_index: None,
+            map: HashMap::new(),
+        }
+    }
+
+    pub fn insert_attestation_shard(&mut self, slot: u64, shard: u64) {
+        let epoch = slot / self.epoch_length;
+
+        self.map.insert(epoch, (slot, shard));
+    }
+
+    pub fn set_validator_index(&mut self, index: Option<u64>) {
+        self.validator_index = index;
+    }
+}
+
+impl DutiesReader for EpochMap {
+    fn attestation_shard(&self, slot: u64) -> Result<Option<u64>, DutiesReaderError> {
+        let epoch = slot / self.epoch_length;
+
+        match self.map.get(&epoch) {
+            Some((attest_slot, attest_shard)) if *attest_slot == slot => Ok(Some(*attest_shard)),
+            Some((attest_slot, _attest_shard)) if *attest_slot != slot => Ok(None),
+            _ => Err(DutiesReaderError::UnknownEpoch),
+        }
+    }
+
+    fn validator_index(&self) -> Option<u64> {
+        self.validator_index
+    }
+}

eth2/attester/src/test_utils/local_signer.rs

@@ -0,0 +1,31 @@
+use crate::traits::Signer;
+use std::sync::RwLock;
+use types::{Keypair, Signature};
+
+/// A test-only struct used to simulate a Beacon Node.
+pub struct LocalSigner {
+    keypair: Keypair,
+    should_sign: RwLock<bool>,
+}
+
+impl LocalSigner {
+    /// Produce a new LocalSigner with signing enabled by default.
+    pub fn new(keypair: Keypair) -> Self {
+        Self {
+            keypair,
+            should_sign: RwLock::new(true),
+        }
+    }
+
+    /// If set to `false`, the service will refuse to sign all messages. Otherwise, all messages
+    /// will be signed.
+    pub fn enable_signing(&self, enabled: bool) {
+        *self.should_sign.write().unwrap() = enabled;
+    }
+}
+
+impl Signer for LocalSigner {
+    fn sign_attestation_message(&self, message: &[u8]) -> Option<Signature> {
+        Some(Signature::new(message, &self.keypair.sk))
+    }
+}

eth2/attester/src/test_utils/mod.rs

@@ -0,0 +1,7 @@
+mod epoch_map;
+mod local_signer;
+mod simulated_beacon_node;
+
+pub use self::epoch_map::EpochMap;
+pub use self::local_signer::LocalSigner;
+pub use self::simulated_beacon_node::SimulatedBeaconNode;

eth2/attester/src/test_utils/simulated_beacon_node.rs

@@ -0,0 +1,44 @@
+use crate::traits::{BeaconNode, BeaconNodeError, PublishOutcome};
+use std::sync::RwLock;
+use types::{AttestationData, FreeAttestation};
+
+type ProduceResult = Result<Option<AttestationData>, BeaconNodeError>;
+type PublishResult = Result<PublishOutcome, BeaconNodeError>;
+
+/// A test-only struct used to simulate a Beacon Node.
+#[derive(Default)]
+pub struct SimulatedBeaconNode {
+    pub produce_input: RwLock<Option<(u64, u64)>>,
+    pub produce_result: RwLock<Option<ProduceResult>>,
+
+    pub publish_input: RwLock<Option<FreeAttestation>>,
+    pub publish_result: RwLock<Option<PublishResult>>,
+}
+
+impl SimulatedBeaconNode {
+    pub fn set_next_produce_result(&self, result: ProduceResult) {
+        *self.produce_result.write().unwrap() = Some(result);
+    }
+
+    pub fn set_next_publish_result(&self, result: PublishResult) {
+        *self.publish_result.write().unwrap() = Some(result);
+    }
+}
+
+impl BeaconNode for SimulatedBeaconNode {
+    fn produce_attestation_data(&self, slot: u64, shard: u64) -> ProduceResult {
+        *self.produce_input.write().unwrap() = Some((slot, shard));
+        match *self.produce_result.read().unwrap() {
+            Some(ref r) => r.clone(),
+            None => panic!("TestBeaconNode: produce_result == None"),
+        }
+    }
+
+    fn publish_attestation_data(&self, free_attestation: FreeAttestation) -> PublishResult {
+        *self.publish_input.write().unwrap() = Some(free_attestation.clone());
+        match *self.publish_result.read().unwrap() {
+            Some(ref r) => r.clone(),
+            None => panic!("TestBeaconNode: publish_result == None"),
+        }
+    }
+}

eth2/attester/src/traits.rs

@@ -0,0 +1,49 @@
+use types::{AttestationData, FreeAttestation, Signature};
+
+#[derive(Debug, PartialEq, Clone)]
+pub enum BeaconNodeError {
+    RemoteFailure(String),
+    DecodeFailure,
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub enum PublishOutcome {
+    ValidAttestation,
+    InvalidAttestation(String),
+}
+
+/// Defines the methods required to produce and publish blocks on a Beacon Node.
+pub trait BeaconNode: Send + Sync {
+    fn produce_attestation_data(
+        &self,
+        slot: u64,
+        shard: u64,
+    ) -> Result<Option<AttestationData>, BeaconNodeError>;
+
+    fn publish_attestation_data(
+        &self,
+        free_attestation: FreeAttestation,
+    ) -> Result<PublishOutcome, BeaconNodeError>;
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub enum DutiesReaderError {
+    UnknownValidator,
+    UnknownEpoch,
+    EpochLengthIsZero,
+    Poisoned,
+}
+
+/// Informs a validator of their duties (e.g., block production).
+pub trait DutiesReader: Send + Sync {
+    /// Returns `Some(shard)` if this slot is an attestation slot. Otherwise, returns `None.`
+    fn attestation_shard(&self, slot: u64) -> Result<Option<u64>, DutiesReaderError>;
+
+    /// Returns `Some(shard)` if this slot is an attestation slot. Otherwise, returns `None.`
+    fn validator_index(&self) -> Option<u64>;
+}
+
+/// Signs message using an internally-maintained private key.
+pub trait Signer {
+    fn sign_attestation_message(&self, message: &[u8]) -> Option<Signature>;
+}

eth2/block_producer/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "block_producer"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+[dependencies]
+slot_clock = { path = "../../eth2/utils/slot_clock" }
+ssz = { path = "../../eth2/utils/ssz" }
+types = { path = "../../eth2/types" }

eth2/block_producer/src/lib.rs

@@ -1,16 +1,14 @@
-mod grpc;
+pub mod test_utils;
-mod service;
-#[cfg(test)]
-mod test_node;
 mod traits;
 
-use self::traits::{BeaconNode, BeaconNodeError};
-use super::EpochDutiesMap;
 use slot_clock::SlotClock;
-use std::sync::{Arc, RwLock};
+use ssz::ssz_encode;
-use types::{BeaconBlock, ChainSpec};
+use std::sync::Arc;
+use types::{BeaconBlock, ChainSpec, PublicKey};
 
-pub use self::service::BlockProducerService;
+pub use self::traits::{
+    BeaconNode, BeaconNodeError, DutiesReader, DutiesReaderError, PublishOutcome, Signer,
+};
 
 #[derive(Debug, PartialEq)]
 pub enum PollOutcome {
@@ -26,6 +24,10 @@ pub enum PollOutcome {
     SlotAlreadyProcessed(u64),
     /// The Beacon Node was unable to produce a block at that slot.
     BeaconNodeUnableToProduceBlock(u64),
+    /// The signer failed to sign the message.
+    SignerRejection(u64),
+    /// The public key for this validator is not an active validator.
+    ValidatorIsUnknown(u64),
 }
 
 #[derive(Debug, PartialEq)]
@@ -44,61 +46,65 @@ pub enum Error {
 /// Ensures that messages are not slashable.
 ///
 /// Relies upon an external service to keep the `EpochDutiesMap` updated.
-pub struct BlockProducer<T: SlotClock, U: BeaconNode> {
+pub struct BlockProducer<T: SlotClock, U: BeaconNode, V: DutiesReader, W: Signer> {
-    pub last_processed_slot: u64,
+    pub last_processed_slot: Option<u64>,
+    pubkey: PublicKey,
     spec: Arc<ChainSpec>,
-    epoch_map: Arc<RwLock<EpochDutiesMap>>,
+    epoch_map: Arc<V>,
-    slot_clock: Arc<RwLock<T>>,
+    slot_clock: Arc<T>,
     beacon_node: Arc<U>,
+    signer: Arc<W>,
 }
 
-impl<T: SlotClock, U: BeaconNode> BlockProducer<T, U> {
+impl<T: SlotClock, U: BeaconNode, V: DutiesReader, W: Signer> BlockProducer<T, U, V, W> {
     /// Returns a new instance where `last_processed_slot == 0`.
     pub fn new(
         spec: Arc<ChainSpec>,
-        epoch_map: Arc<RwLock<EpochDutiesMap>>,
+        pubkey: PublicKey,
-        slot_clock: Arc<RwLock<T>>,
+        epoch_map: Arc<V>,
+        slot_clock: Arc<T>,
         beacon_node: Arc<U>,
+        signer: Arc<W>,
     ) -> Self {
         Self {
-            last_processed_slot: 0,
+            last_processed_slot: None,
+            pubkey,
             spec,
             epoch_map,
             slot_clock,
             beacon_node,
+            signer,
         }
     }
 }
 
-impl<T: SlotClock, U: BeaconNode> BlockProducer<T, U> {
+impl<T: SlotClock, U: BeaconNode, V: DutiesReader, W: Signer> BlockProducer<T, U, V, W> {
     /// "Poll" to see if the validator is required to take any action.
     ///
     /// The slot clock will be read and any new actions undertaken.
     pub fn poll(&mut self) -> Result<PollOutcome, Error> {
         let slot = self
             .slot_clock
-            .read()
-            .map_err(|_| Error::SlotClockPoisoned)?
             .present_slot()
             .map_err(|_| Error::SlotClockError)?
             .ok_or(Error::SlotUnknowable)?;
 
-        let epoch = slot
-            .checked_div(self.spec.epoch_length)
-            .ok_or(Error::EpochLengthIsZero)?;
-
         // If this is a new slot.
-        if slot > self.last_processed_slot {
+        if !self.is_processed_slot(slot) {
-            let is_block_production_slot = {
+            let is_block_production_slot = match self.epoch_map.is_block_production_slot(slot) {
-                let epoch_map = self.epoch_map.read().map_err(|_| Error::EpochMapPoisoned)?;
+                Ok(result) => result,
-                match epoch_map.get(&epoch) {
+                Err(DutiesReaderError::UnknownEpoch) => {
-                    None => return Ok(PollOutcome::ProducerDutiesUnknown(slot)),
+                    return Ok(PollOutcome::ProducerDutiesUnknown(slot));
-                    Some(duties) => duties.is_block_production_slot(slot),
                 }
+                Err(DutiesReaderError::UnknownValidator) => {
+                    return Ok(PollOutcome::ValidatorIsUnknown(slot));
+                }
+                Err(DutiesReaderError::EpochLengthIsZero) => return Err(Error::EpochLengthIsZero),
+                Err(DutiesReaderError::Poisoned) => return Err(Error::EpochMapPoisoned),
             };
 
             if is_block_production_slot {
-                self.last_processed_slot = slot;
+                self.last_processed_slot = Some(slot);
 
                 self.produce_block(slot)
             } else {
@@ -109,6 +115,13 @@ impl<T: SlotClock, U: BeaconNode> BlockProducer<T, U> {
         }
     }
 
+    fn is_processed_slot(&self, slot: u64) -> bool {
+        match self.last_processed_slot {
+            Some(processed_slot) if processed_slot >= slot => true,
+            _ => false,
+        }
+    }
+
     /// Produce a block at some slot.
     ///
     /// Assumes that a block is required at this slot (does not check the duties).
@@ -120,11 +133,29 @@ impl<T: SlotClock, U: BeaconNode> BlockProducer<T, U> {
     /// The slash-protection code is not yet implemented. There is zero protection against
     /// slashing.
     fn produce_block(&mut self, slot: u64) -> Result<PollOutcome, Error> {
-        if let Some(block) = self.beacon_node.produce_beacon_block(slot)? {
+        let randao_reveal = {
+            let producer_nonce = self.beacon_node.proposer_nonce(&self.pubkey)?;
+
+            // TODO: add domain, etc to this message.
+            let message = ssz_encode(&producer_nonce);
+
+            match self.signer.sign_randao_reveal(&message) {
+                None => return Ok(PollOutcome::SignerRejection(slot)),
+                Some(signature) => signature,
+            }
+        };
+
+        if let Some(block) = self
+            .beacon_node
+            .produce_beacon_block(slot, &randao_reveal)?
+        {
             if self.safe_to_produce(&block) {
-                let block = self.sign_block(block);
+                if let Some(block) = self.sign_block(block) {
                     self.beacon_node.publish_beacon_block(block)?;
                     Ok(PollOutcome::BlockProduced(slot))
+                } else {
+                    Ok(PollOutcome::SignerRejection(slot))
+                }
             } else {
                 Ok(PollOutcome::SlashableBlockNotProduced(slot))
             }
@@ -137,11 +168,19 @@ impl<T: SlotClock, U: BeaconNode> BlockProducer<T, U> {
     ///
     /// Important: this function will not check to ensure the block is not slashable. This must be
     /// done upstream.
-    fn sign_block(&mut self, block: BeaconBlock) -> BeaconBlock {
+    fn sign_block(&mut self, mut block: BeaconBlock) -> Option<BeaconBlock> {
-        // TODO: sign the block
-        // https://github.com/sigp/lighthouse/issues/160
         self.store_produce(&block);
-        block
+
+        match self
+            .signer
+            .sign_block_proposal(&block.proposal_root(&self.spec)[..])
+        {
+            None => None,
+            Some(signature) => {
+                block.signature = signature;
+                Some(block)
+            }
+        }
     }
 
     /// Returns `true` if signing a block is safe (non-slashable).
@@ -174,11 +213,13 @@ impl From<BeaconNodeError> for Error {
 
 #[cfg(test)]
 mod tests {
-    use super::test_node::TestBeaconNode;
+    use super::test_utils::{EpochMap, LocalSigner, SimulatedBeaconNode};
     use super::*;
-    use crate::duties::EpochDuties;
     use slot_clock::TestingSlotClock;
-    use types::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use types::{
+        test_utils::{SeedableRng, TestRandom, XorShiftRng},
+        Keypair,
+    };
 
     // TODO: implement more thorough testing.
     // https://github.com/sigp/lighthouse/issues/160
@@ -190,53 +231,54 @@ mod tests {
         let mut rng = XorShiftRng::from_seed([42; 16]);
 
         let spec = Arc::new(ChainSpec::foundation());
-        let epoch_map = Arc::new(RwLock::new(EpochDutiesMap::new()));
+        let slot_clock = Arc::new(TestingSlotClock::new(0));
-        let slot_clock = Arc::new(RwLock::new(TestingSlotClock::new(0)));
+        let beacon_node = Arc::new(SimulatedBeaconNode::default());
-        let beacon_node = Arc::new(TestBeaconNode::default());
+        let signer = Arc::new(LocalSigner::new(Keypair::random()));
+
+        let mut epoch_map = EpochMap::new(spec.epoch_length);
+        let produce_slot = 100;
+        let produce_epoch = produce_slot / spec.epoch_length;
+        epoch_map.map.insert(produce_epoch, produce_slot);
+        let epoch_map = Arc::new(epoch_map);
+        let keypair = Keypair::random();
 
         let mut block_producer = BlockProducer::new(
             spec.clone(),
+            keypair.pk.clone(),
             epoch_map.clone(),
             slot_clock.clone(),
             beacon_node.clone(),
+            signer.clone(),
         );
 
         // Configure responses from the BeaconNode.
         beacon_node.set_next_produce_result(Ok(Some(BeaconBlock::random_for_test(&mut rng))));
-        beacon_node.set_next_publish_result(Ok(true));
+        beacon_node.set_next_publish_result(Ok(PublishOutcome::ValidBlock));
-
+        beacon_node.set_next_nonce_result(Ok(0));
-        // Setup some valid duties for the validator
-        let produce_slot = 100;
-        let duties = EpochDuties {
-            block_production_slot: Some(produce_slot),
-            ..std::default::Default::default()
-        };
-        let produce_epoch = produce_slot / spec.epoch_length;
-        epoch_map.write().unwrap().insert(produce_epoch, duties);
 
         // One slot before production slot...
-        slot_clock.write().unwrap().set_slot(produce_slot - 1);
+        slot_clock.set_slot(produce_slot - 1);
         assert_eq!(
             block_producer.poll(),
             Ok(PollOutcome::BlockProductionNotRequired(produce_slot - 1))
         );
 
         // On the produce slot...
-        slot_clock.write().unwrap().set_slot(produce_slot);
+        slot_clock.set_slot(produce_slot);
         assert_eq!(
             block_producer.poll(),
             Ok(PollOutcome::BlockProduced(produce_slot))
         );
 
         // Trying the same produce slot again...
-        slot_clock.write().unwrap().set_slot(produce_slot);
+        slot_clock.set_slot(produce_slot);
         assert_eq!(
             block_producer.poll(),
             Ok(PollOutcome::SlotAlreadyProcessed(produce_slot))
         );
 
         // One slot after the produce slot...
-        slot_clock.write().unwrap().set_slot(produce_slot + 1);
+        slot_clock.set_slot(produce_slot + 1);
         assert_eq!(
             block_producer.poll(),
             Ok(PollOutcome::BlockProductionNotRequired(produce_slot + 1))
@@ -244,7 +286,7 @@ mod tests {
 
         // In an epoch without known duties...
         let slot = (produce_epoch + 1) * spec.epoch_length;
-        slot_clock.write().unwrap().set_slot(slot);
+        slot_clock.set_slot(slot);
         assert_eq!(
             block_producer.poll(),
             Ok(PollOutcome::ProducerDutiesUnknown(slot))

eth2/block_producer/src/test_utils/epoch_map.rs

@@ -0,0 +1,27 @@
+use crate::{DutiesReader, DutiesReaderError};
+use std::collections::HashMap;
+
+pub struct EpochMap {
+    epoch_length: u64,
+    pub map: HashMap<u64, u64>,
+}
+
+impl EpochMap {
+    pub fn new(epoch_length: u64) -> Self {
+        Self {
+            epoch_length,
+            map: HashMap::new(),
+        }
+    }
+}
+
+impl DutiesReader for EpochMap {
+    fn is_block_production_slot(&self, slot: u64) -> Result<bool, DutiesReaderError> {
+        let epoch = slot / self.epoch_length;
+        match self.map.get(&epoch) {
+            Some(s) if *s == slot => Ok(true),
+            Some(s) if *s != slot => Ok(false),
+            _ => Err(DutiesReaderError::UnknownEpoch),
+        }
+    }
+}

eth2/block_producer/src/test_utils/local_signer.rs

@@ -0,0 +1,35 @@
+use crate::traits::Signer;
+use std::sync::RwLock;
+use types::{Keypair, Signature};
+
+/// A test-only struct used to simulate a Beacon Node.
+pub struct LocalSigner {
+    keypair: Keypair,
+    should_sign: RwLock<bool>,
+}
+
+impl LocalSigner {
+    /// Produce a new LocalSigner with signing enabled by default.
+    pub fn new(keypair: Keypair) -> Self {
+        Self {
+            keypair,
+            should_sign: RwLock::new(true),
+        }
+    }
+
+    /// If set to `false`, the service will refuse to sign all messages. Otherwise, all messages
+    /// will be signed.
+    pub fn enable_signing(&self, enabled: bool) {
+        *self.should_sign.write().unwrap() = enabled;
+    }
+}
+
+impl Signer for LocalSigner {
+    fn sign_block_proposal(&self, message: &[u8]) -> Option<Signature> {
+        Some(Signature::new(message, &self.keypair.sk))
+    }
+
+    fn sign_randao_reveal(&self, message: &[u8]) -> Option<Signature> {
+        Some(Signature::new(message, &self.keypair.sk))
+    }
+}

eth2/block_producer/src/test_utils/mod.rs

@@ -0,0 +1,7 @@
+mod epoch_map;
+mod local_signer;
+mod simulated_beacon_node;
+
+pub use self::epoch_map::EpochMap;
+pub use self::local_signer::LocalSigner;
+pub use self::simulated_beacon_node::SimulatedBeaconNode;

eth2/block_producer/src/test_utils/simulated_beacon_node.rs

@@ -0,0 +1,65 @@
+use crate::traits::{BeaconNode, BeaconNodeError, PublishOutcome};
+use std::sync::RwLock;
+use types::{BeaconBlock, PublicKey, Signature};
+
+type NonceResult = Result<u64, BeaconNodeError>;
+type ProduceResult = Result<Option<BeaconBlock>, BeaconNodeError>;
+type PublishResult = Result<PublishOutcome, BeaconNodeError>;
+
+/// A test-only struct used to simulate a Beacon Node.
+#[derive(Default)]
+pub struct SimulatedBeaconNode {
+    pub nonce_input: RwLock<Option<PublicKey>>,
+    pub nonce_result: RwLock<Option<NonceResult>>,
+
+    pub produce_input: RwLock<Option<(u64, Signature)>>,
+    pub produce_result: RwLock<Option<ProduceResult>>,
+
+    pub publish_input: RwLock<Option<BeaconBlock>>,
+    pub publish_result: RwLock<Option<PublishResult>>,
+}
+
+impl SimulatedBeaconNode {
+    /// Set the result to be returned when `produce_beacon_block` is called.
+    pub fn set_next_nonce_result(&self, result: NonceResult) {
+        *self.nonce_result.write().unwrap() = Some(result);
+    }
+
+    /// Set the result to be returned when `produce_beacon_block` is called.
+    pub fn set_next_produce_result(&self, result: ProduceResult) {
+        *self.produce_result.write().unwrap() = Some(result);
+    }
+
+    /// Set the result to be returned when `publish_beacon_block` is called.
+    pub fn set_next_publish_result(&self, result: PublishResult) {
+        *self.publish_result.write().unwrap() = Some(result);
+    }
+}
+
+impl BeaconNode for SimulatedBeaconNode {
+    fn proposer_nonce(&self, pubkey: &PublicKey) -> NonceResult {
+        *self.nonce_input.write().unwrap() = Some(pubkey.clone());
+        match *self.nonce_result.read().unwrap() {
+            Some(ref r) => r.clone(),
+            None => panic!("SimulatedBeaconNode: nonce_result == None"),
+        }
+    }
+
+    /// Returns the value specified by the `set_next_produce_result`.
+    fn produce_beacon_block(&self, slot: u64, randao_reveal: &Signature) -> ProduceResult {
+        *self.produce_input.write().unwrap() = Some((slot, randao_reveal.clone()));
+        match *self.produce_result.read().unwrap() {
+            Some(ref r) => r.clone(),
+            None => panic!("SimulatedBeaconNode: produce_result == None"),
+        }
+    }
+
+    /// Returns the value specified by the `set_next_publish_result`.
+    fn publish_beacon_block(&self, block: BeaconBlock) -> PublishResult {
+        *self.publish_input.write().unwrap() = Some(block);
+        match *self.publish_result.read().unwrap() {
+            Some(ref r) => r.clone(),
+            None => panic!("SimulatedBeaconNode: publish_result == None"),
+        }
+    }
+}

eth2/block_producer/src/traits.rs

@@ -0,0 +1,52 @@
+use types::{BeaconBlock, PublicKey, Signature};
+
+#[derive(Debug, PartialEq, Clone)]
+pub enum BeaconNodeError {
+    RemoteFailure(String),
+    DecodeFailure,
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub enum PublishOutcome {
+    ValidBlock,
+    InvalidBlock(String),
+}
+
+/// Defines the methods required to produce and publish blocks on a Beacon Node.
+pub trait BeaconNode: Send + Sync {
+    /// Requests the proposer nonce (presently named `proposer_slots`).
+    fn proposer_nonce(&self, pubkey: &PublicKey) -> Result<u64, BeaconNodeError>;
+
+    /// Request that the node produces a block.
+    ///
+    /// Returns Ok(None) if the Beacon Node is unable to produce at the given slot.
+    fn produce_beacon_block(
+        &self,
+        slot: u64,
+        randao_reveal: &Signature,
+    ) -> Result<Option<BeaconBlock>, BeaconNodeError>;
+
+    /// Request that the node publishes a block.
+    ///
+    /// Returns `true` if the publish was sucessful.
+    fn publish_beacon_block(&self, block: BeaconBlock) -> Result<PublishOutcome, BeaconNodeError>;
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub enum DutiesReaderError {
+    UnknownValidator,
+    UnknownEpoch,
+    EpochLengthIsZero,
+    Poisoned,
+}
+
+/// Informs a validator of their duties (e.g., block production).
+pub trait DutiesReader: Send + Sync {
+    fn is_block_production_slot(&self, slot: u64) -> Result<bool, DutiesReaderError>;
+}
+
+/// Signs message using an internally-maintained private key.
+pub trait Signer {
+    fn sign_block_proposal(&self, message: &[u8]) -> Option<Signature>;
+    fn sign_randao_reveal(&self, message: &[u8]) -> Option<Signature>;
+}

eth2/genesis/Cargo.toml

@@ -9,4 +9,3 @@ bls = { path = "../utils/bls"  }
 ssz = { path = "../utils/ssz" }
 types = { path = "../types"  }
 validator_induction = { path = "../validator_induction"  }
-validator_shuffling = { path = "../validator_shuffling"  }

eth2/genesis/src/beacon_block.rs

@@ -31,12 +31,13 @@ mod tests {
     use bls::Signature;
 
     #[test]
-    fn test_genesis() {
+    fn test_state_root() {
         let spec = ChainSpec::foundation();
         let state_root = Hash256::from("cats".as_bytes());
 
-        // This only checks that the function runs without panic.
+        let block = genesis_beacon_block(state_root, &spec);
-        genesis_beacon_block(state_root, &spec);
+
+        assert_eq!(block.state_root, state_root);
     }
 
     #[test]

eth2/genesis/src/beacon_state.rs

@@ -1,30 +1,12 @@
 use types::{BeaconState, ChainSpec, Crosslink, Fork};
-use validator_shuffling::{shard_and_committees_for_cycle, ValidatorAssignmentError};
-
-#[derive(Debug, PartialEq)]
-pub enum Error {
-    NoValidators,
-    ValidationAssignmentError(ValidatorAssignmentError),
-    NotImplemented,
-}
-
-pub fn genesis_beacon_state(spec: &ChainSpec) -> Result<BeaconState, Error> {
-    /*
-     * Assign the validators to shards, using all zeros as the seed.
-     */
-    let _shard_and_committee_for_slots = {
-        let mut a = shard_and_committees_for_cycle(&[0; 32], &spec.initial_validators, 0, &spec)?;
-        let mut b = a.clone();
-        a.append(&mut b);
-        a
-    };
 
+pub fn genesis_beacon_state(spec: &ChainSpec) -> BeaconState {
     let initial_crosslink = Crosslink {
         slot: spec.genesis_slot,
         shard_block_root: spec.zero_hash,
     };
 
-    Ok(BeaconState {
+    BeaconState {
         /*
          * Misc
          */
@@ -55,8 +37,8 @@ pub fn genesis_beacon_state(spec: &ChainSpec) -> Result<BeaconState, Error> {
         current_epoch_start_shard: spec.genesis_start_shard,
         previous_epoch_calculation_slot: spec.genesis_slot,
         current_epoch_calculation_slot: spec.genesis_slot,
-        previous_epoch_randao_mix: spec.zero_hash,
+        previous_epoch_seed: spec.zero_hash,
-        current_epoch_randao_mix: spec.zero_hash,
+        current_epoch_seed: spec.zero_hash,
         /*
          * Custody challenges
          */
@@ -73,7 +55,7 @@ pub fn genesis_beacon_state(spec: &ChainSpec) -> Result<BeaconState, Error> {
          */
         latest_crosslinks: vec![initial_crosslink; spec.shard_count as usize],
         latest_block_roots: vec![spec.zero_hash; spec.latest_block_roots_length as usize],
-        latest_penalized_exit_balances: vec![0; spec.latest_penalized_exit_length as usize],
+        latest_penalized_balances: vec![0; spec.latest_penalized_exit_length as usize],
         latest_attestations: vec![],
         batched_block_roots: vec![],
         /*
@@ -81,12 +63,6 @@ pub fn genesis_beacon_state(spec: &ChainSpec) -> Result<BeaconState, Error> {
          */
         latest_eth1_data: spec.intial_eth1_data.clone(),
         eth1_data_votes: vec![],
-    })
-}
-
-impl From<ValidatorAssignmentError> for Error {
-    fn from(e: ValidatorAssignmentError) -> Error {
-        Error::ValidationAssignmentError(e)
     }
 }
 
@@ -99,7 +75,7 @@ mod tests {
     fn test_genesis_state() {
         let spec = ChainSpec::foundation();
 
-        let state = genesis_beacon_state(&spec).unwrap();
+        let state = genesis_beacon_state(&spec);
 
         assert_eq!(
             state.validator_registry.len(),
@@ -111,7 +87,7 @@ mod tests {
     fn test_genesis_state_misc() {
         let spec = ChainSpec::foundation();
 
-        let state = genesis_beacon_state(&spec).unwrap();
+        let state = genesis_beacon_state(&spec);
 
         assert_eq!(state.slot, 0);
         assert_eq!(state.genesis_time, spec.genesis_time);
@@ -124,7 +100,7 @@ mod tests {
     fn test_genesis_state_validators() {
         let spec = ChainSpec::foundation();
 
-        let state = genesis_beacon_state(&spec).unwrap();
+        let state = genesis_beacon_state(&spec);
 
         assert_eq!(state.validator_registry, spec.initial_validators);
         assert_eq!(state.validator_balances, spec.initial_balances);
@@ -137,7 +113,7 @@ mod tests {
     fn test_genesis_state_randomness_committees() {
         let spec = ChainSpec::foundation();
 
-        let state = genesis_beacon_state(&spec).unwrap();
+        let state = genesis_beacon_state(&spec);
 
         // Array of size 8,192 each being zero_hash
         assert_eq!(state.latest_randao_mixes.len(), 8_192);
@@ -166,7 +142,7 @@ mod tests {
     fn test_genesis_state_finanilty() {
         let spec = ChainSpec::foundation();
 
-        let state = genesis_beacon_state(&spec).unwrap();
+        let state = genesis_beacon_state(&spec);
 
         assert_eq!(state.previous_justified_slot, 0);
         assert_eq!(state.justified_slot, 0);
@@ -178,7 +154,7 @@ mod tests {
     fn test_genesis_state_recent_state() {
         let spec = ChainSpec::foundation();
 
-        let state = genesis_beacon_state(&spec).unwrap();
+        let state = genesis_beacon_state(&spec);
 
         // Test latest_crosslinks
         assert_eq!(state.latest_crosslinks.len(), 1_024);
@@ -193,9 +169,9 @@ mod tests {
             assert_eq!(*block, Hash256::zero());
         }
 
-        // Test latest_penalized_exit_balances
+        // Test latest_penalized_balances
-        assert_eq!(state.latest_penalized_exit_balances.len(), 8_192);
+        assert_eq!(state.latest_penalized_balances.len(), 8_192);
-        for item in state.latest_penalized_exit_balances.iter() {
+        for item in state.latest_penalized_balances.iter() {
             assert!(*item == 0);
         }
 
@@ -210,7 +186,7 @@ mod tests {
     fn test_genesis_state_deposit_root() {
         let spec = ChainSpec::foundation();
 
-        let state = genesis_beacon_state(&spec).unwrap();
+        let state = genesis_beacon_state(&spec);
 
         assert_eq!(&state.latest_eth1_data, &spec.intial_eth1_data);
         assert!(state.eth1_data_votes.is_empty());

eth2/genesis/src/lib.rs

@@ -2,4 +2,4 @@ mod beacon_block;
 mod beacon_state;
 
 pub use crate::beacon_block::genesis_beacon_block;
-pub use crate::beacon_state::{genesis_beacon_state, Error as GenesisError};
+pub use crate::beacon_state::genesis_beacon_state;

eth2/types/Cargo.toml

@@ -9,5 +9,13 @@ bls = { path = "../utils/bls" }
 boolean-bitfield = { path = "../utils/boolean-bitfield" }
 ethereum-types = "0.4.0"
 hashing = { path = "../utils/hashing" }
+honey-badger-split =  { path = "../utils/honey-badger-split" }
+integer-sqrt = "0.1"
+log = "0.4"
+rayon = "1.0"
 rand = "0.5.5"
+serde = "1.0"
+serde_derive = "1.0"
+serde_json = "1.0"
 ssz = { path = "../utils/ssz" }
+vec_shuffle = { path = "../utils/vec_shuffle" }

eth2/types/src/attestation.rs

@@ -1,10 +1,11 @@
-use super::bls::AggregateSignature;
+use super::{AttestationData, Bitfield, Hash256};
-use super::ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
-use super::{AttestationData, Bitfield};
 use crate::test_utils::TestRandom;
+use bls::AggregateSignature;
 use rand::RngCore;
+use serde_derive::Serialize;
+use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize)]
 pub struct Attestation {
     pub data: AttestationData,
     pub aggregation_bitfield: Bitfield,
@@ -12,6 +13,16 @@ pub struct Attestation {
     pub aggregate_signature: AggregateSignature,
 }
 
+impl Attestation {
+    pub fn canonical_root(&self) -> Hash256 {
+        Hash256::from(&self.hash_tree_root()[..])
+    }
+
+    pub fn signable_message(&self, custody_bit: bool) -> Vec<u8> {
+        self.data.signable_message(custody_bit)
+    }
+}
+
 impl Encodable for Attestation {
     fn ssz_append(&self, s: &mut SszStream) {
         s.append(&self.data);
@@ -73,9 +84,9 @@ impl<T: RngCore> TestRandom<T> for Attestation {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::*;
     use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/types/src/attestation_data.rs

@@ -1,7 +1,8 @@
-use super::ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
+use super::{AttestationDataAndCustodyBit, Hash256};
-use super::Hash256;
 use crate::test_utils::TestRandom;
 use rand::RngCore;
+use serde_derive::Serialize;
+use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 
 pub const SSZ_ATTESTION_DATA_LENGTH: usize = {
     8 +             // slot
@@ -14,7 +15,7 @@ pub const SSZ_ATTESTION_DATA_LENGTH: usize = {
     32 // justified_block_root
 };
 
-#[derive(Debug, Clone, PartialEq, Default)]
+#[derive(Debug, Clone, PartialEq, Default, Serialize, Hash)]
 pub struct AttestationData {
     pub slot: u64,
     pub shard: u64,
@@ -26,6 +27,8 @@ pub struct AttestationData {
     pub justified_block_root: Hash256,
 }
 
+impl Eq for AttestationData {}
+
 impl AttestationData {
     pub fn zero() -> Self {
         Self {
@@ -40,10 +43,16 @@ impl AttestationData {
         }
     }
 
-    // TODO: Implement this as a merkle root, once tree_ssz is implemented.
-    // https://github.com/sigp/lighthouse/issues/92
     pub fn canonical_root(&self) -> Hash256 {
-        Hash256::zero()
+        Hash256::from(&self.hash_tree_root()[..])
+    }
+
+    pub fn signable_message(&self, custody_bit: bool) -> Vec<u8> {
+        let attestation_data_and_custody_bit = AttestationDataAndCustodyBit {
+            data: self.clone(),
+            custody_bit,
+        };
+        attestation_data_and_custody_bit.hash_tree_root()
     }
 }
 
@@ -117,9 +126,9 @@ impl<T: RngCore> TestRandom<T> for AttestationData {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::*;
     use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/types/src/attestation_data_and_custody_bit.rs

@@ -1,9 +1,10 @@
-use super::ssz::{Decodable, DecodeError, Encodable, hash, TreeHash, SszStream};
-use rand::RngCore;
-use crate::test_utils::TestRandom;
 use super::AttestationData;
+use crate::test_utils::TestRandom;
+use rand::RngCore;
+use serde_derive::Serialize;
+use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 
-#[derive(Debug, Clone, PartialEq, Default)]
+#[derive(Debug, Clone, PartialEq, Default, Serialize)]
 pub struct AttestationDataAndCustodyBit {
     pub data: AttestationData,
     pub custody_bit: bool,
@@ -12,19 +13,16 @@ pub struct AttestationDataAndCustodyBit {
 impl Encodable for AttestationDataAndCustodyBit {
     fn ssz_append(&self, s: &mut SszStream) {
         s.append(&self.data);
-        s.append(&self.custody_bit);
+        // TODO: deal with bools
     }
 }
 
 impl Decodable for AttestationDataAndCustodyBit {
     fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
         let (data, i) = <_>::ssz_decode(bytes, i)?;
-        let (custody_bit, i) = <_>::ssz_decode(bytes, i)?;
+        let custody_bit = false;
 
-        let attestation_data_and_custody_bit = AttestationDataAndCustodyBit {
+        let attestation_data_and_custody_bit = AttestationDataAndCustodyBit { data, custody_bit };
-            data,
-            custody_bit,
-        };
 
         Ok((attestation_data_and_custody_bit, i))
     }
@@ -32,7 +30,7 @@ impl Decodable for AttestationDataAndCustodyBit {
 
 impl TreeHash for AttestationDataAndCustodyBit {
     fn hash_tree_root(&self) -> Vec<u8> {
-        let result: Vec<u8> = vec![];
+        let mut result: Vec<u8> = vec![];
         result.append(&mut self.data.hash_tree_root());
         // TODO: add bool ssz
         // result.append(custody_bit.hash_tree_root());
@@ -44,16 +42,17 @@ impl<T: RngCore> TestRandom<T> for AttestationDataAndCustodyBit {
     fn random_for_test(rng: &mut T) -> Self {
         Self {
             data: <_>::random_for_test(rng),
-            custody_bit: <_>::random_for_test(rng),
+            // TODO: deal with bools
+            custody_bit: false,
         }
     }
 }
 
 #[cfg(test)]
 mod test {
-    use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
     use super::*;
-    use super::super::ssz::ssz_encode;
+    use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/types/src/beacon_block.rs

@@ -1,11 +1,11 @@
-use super::ssz::{hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash};
+use super::{BeaconBlockBody, ChainSpec, Eth1Data, Hash256, ProposalSignedData};
-use super::{BeaconBlockBody, Eth1Data, Hash256};
 use crate::test_utils::TestRandom;
 use bls::Signature;
-use hashing::canonical_hash;
 use rand::RngCore;
+use serde_derive::Serialize;
+use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 
-#[derive(Debug, PartialEq, Clone)]
+#[derive(Debug, PartialEq, Clone, Serialize)]
 pub struct BeaconBlock {
     pub slot: u64,
     pub parent_root: Hash256,
@@ -18,9 +18,22 @@ pub struct BeaconBlock {
 
 impl BeaconBlock {
     pub fn canonical_root(&self) -> Hash256 {
-        // TODO: implement tree hashing.
+        Hash256::from(&self.hash_tree_root()[..])
-        // https://github.com/sigp/lighthouse/issues/70
+    }
-        Hash256::from(&canonical_hash(&ssz_encode(self))[..])
+
+    pub fn proposal_root(&self, spec: &ChainSpec) -> Hash256 {
+        let block_without_signature_root = {
+            let mut block_without_signature = self.clone();
+            block_without_signature.signature = spec.empty_signature.clone();
+            block_without_signature.canonical_root()
+        };
+
+        let proposal = ProposalSignedData {
+            slot: self.slot,
+            shard: spec.beacon_chain_shard_number,
+            block_root: block_without_signature_root,
+        };
+        Hash256::from_slice(&proposal.hash_tree_root()[..])
     }
 }
 
@@ -91,9 +104,9 @@ impl<T: RngCore> TestRandom<T> for BeaconBlock {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::*;
     use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/types/src/beacon_block_body.rs

@@ -1,7 +1,8 @@
-use super::ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 use super::{Attestation, CasperSlashing, Deposit, Exit, ProposerSlashing};
 use crate::test_utils::TestRandom;
 use rand::RngCore;
+use serde_derive::Serialize;
+use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 
 // The following types are just dummy classes as they will not be defined until
 // Phase 1 (Sharding phase)
@@ -9,7 +10,7 @@ type CustodyReseed = usize;
 type CustodyChallenge = usize;
 type CustodyResponse = usize;
 
-#[derive(Debug, PartialEq, Clone, Default)]
+#[derive(Debug, PartialEq, Clone, Default, Serialize)]
 pub struct BeaconBlockBody {
     pub proposer_slashings: Vec<ProposerSlashing>,
     pub casper_slashings: Vec<CasperSlashing>,
@@ -93,9 +94,9 @@ impl<T: RngCore> TestRandom<T> for BeaconBlockBody {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::*;
     use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/types/src/casper_slashing.rs

@@ -1,9 +1,10 @@
-use super::ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 use super::SlashableVoteData;
 use crate::test_utils::TestRandom;
 use rand::RngCore;
+use serde_derive::Serialize;
+use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 
-#[derive(Debug, PartialEq, Clone)]
+#[derive(Debug, PartialEq, Clone, Serialize)]
 pub struct CasperSlashing {
     pub slashable_vote_data_1: SlashableVoteData,
     pub slashable_vote_data_2: SlashableVoteData,
@@ -51,9 +52,9 @@ impl<T: RngCore> TestRandom<T> for CasperSlashing {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::*;
     use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/types/src/crosslink.rs

@@ -1,9 +1,10 @@
-use super::ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 use super::Hash256;
 use crate::test_utils::TestRandom;
 use rand::RngCore;
+use serde_derive::Serialize;
+use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, Serialize)]
 pub struct Crosslink {
     pub slot: u64,
     pub shard_block_root: Hash256,
@@ -61,9 +62,9 @@ impl<T: RngCore> TestRandom<T> for Crosslink {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::*;
     use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/types/src/deposit.rs

@@ -1,9 +1,10 @@
-use super::ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 use super::{DepositData, Hash256};
 use crate::test_utils::TestRandom;
 use rand::RngCore;
+use serde_derive::Serialize;
+use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 
-#[derive(Debug, PartialEq, Clone)]
+#[derive(Debug, PartialEq, Clone, Serialize)]
 pub struct Deposit {
     pub merkle_branch: Vec<Hash256>,
     pub merkle_tree_index: u64,
@@ -57,9 +58,9 @@ impl<T: RngCore> TestRandom<T> for Deposit {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::*;
     use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/types/src/deposit_data.rs

@@ -1,9 +1,10 @@
-use super::ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 use super::DepositInput;
 use crate::test_utils::TestRandom;
 use rand::RngCore;
+use serde_derive::Serialize;
+use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 
-#[derive(Debug, PartialEq, Clone)]
+#[derive(Debug, PartialEq, Clone, Serialize)]
 pub struct DepositData {
     pub amount: u64,
     pub timestamp: u64,
@@ -57,9 +58,9 @@ impl<T: RngCore> TestRandom<T> for DepositData {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::*;
     use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/types/src/deposit_input.rs

@@ -1,10 +1,11 @@
-use super::ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 use super::Hash256;
 use crate::test_utils::TestRandom;
 use bls::{PublicKey, Signature};
 use rand::RngCore;
+use serde_derive::Serialize;
+use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 
-#[derive(Debug, PartialEq, Clone)]
+#[derive(Debug, PartialEq, Clone, Serialize)]
 pub struct DepositInput {
     pub pubkey: PublicKey,
     pub withdrawal_credentials: Hash256,
@@ -58,9 +59,9 @@ impl<T: RngCore> TestRandom<T> for DepositInput {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::*;
     use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/types/src/eth1_data.rs

@@ -1,10 +1,11 @@
-use super::ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 use super::Hash256;
 use crate::test_utils::TestRandom;
 use rand::RngCore;
+use serde_derive::Serialize;
+use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 
 // Note: this is refer to as DepositRootVote in specs
-#[derive(Debug, PartialEq, Clone, Default)]
+#[derive(Debug, PartialEq, Clone, Default, Serialize)]
 pub struct Eth1Data {
     pub deposit_root: Hash256,
     pub block_hash: Hash256,
@@ -52,9 +53,9 @@ impl<T: RngCore> TestRandom<T> for Eth1Data {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::*;
     use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/types/src/eth1_data_vote.rs

@@ -1,10 +1,11 @@
-use super::ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 use super::Eth1Data;
 use crate::test_utils::TestRandom;
 use rand::RngCore;
+use serde_derive::Serialize;
+use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 
 // Note: this is refer to as DepositRootVote in specs
-#[derive(Debug, PartialEq, Clone, Default)]
+#[derive(Debug, PartialEq, Clone, Default, Serialize)]
 pub struct Eth1DataVote {
     pub eth1_data: Eth1Data,
     pub vote_count: u64,
@@ -52,9 +53,9 @@ impl<T: RngCore> TestRandom<T> for Eth1DataVote {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::*;
     use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/types/src/exit.rs

@@ -1,9 +1,10 @@
-use super::ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 use crate::test_utils::TestRandom;
 use bls::Signature;
 use rand::RngCore;
+use serde_derive::Serialize;
+use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 
-#[derive(Debug, PartialEq, Clone)]
+#[derive(Debug, PartialEq, Clone, Serialize)]
 pub struct Exit {
     pub slot: u64,
     pub validator_index: u32,
@@ -57,9 +58,9 @@ impl<T: RngCore> TestRandom<T> for Exit {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::*;
     use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/types/src/fork.rs

@@ -1,8 +1,9 @@
-use super::ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 use crate::test_utils::TestRandom;
 use rand::RngCore;
+use serde_derive::Serialize;
+use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 
-#[derive(Debug, Clone, PartialEq, Default)]
+#[derive(Debug, Clone, PartialEq, Default, Serialize)]
 pub struct Fork {
     pub pre_fork_version: u64,
     pub post_fork_version: u64,
@@ -56,9 +57,9 @@ impl<T: RngCore> TestRandom<T> for Fork {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::*;
     use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/types/src/free_attestation.rs

@@ -0,0 +1,12 @@
+/// Note: this object does not actually exist in the spec.
+///
+/// We use it for managing attestations that have not been aggregated.
+use super::{AttestationData, Signature};
+use serde_derive::Serialize;
+
+#[derive(Debug, Clone, PartialEq, Serialize)]
+pub struct FreeAttestation {
+    pub data: AttestationData,
+    pub signature: Signature,
+    pub validator_index: u64,
+}

eth2/types/src/lib.rs

@@ -1,12 +1,8 @@
-extern crate bls;
-extern crate boolean_bitfield;
-extern crate ethereum_types;
-extern crate ssz;
-
 pub mod test_utils;
 
 pub mod attestation;
 pub mod attestation_data;
+pub mod attestation_data_and_custody_bit;
 pub mod beacon_block;
 pub mod beacon_block_body;
 pub mod beacon_state;
@@ -19,6 +15,7 @@ pub mod eth1_data;
 pub mod eth1_data_vote;
 pub mod exit;
 pub mod fork;
+pub mod free_attestation;
 pub mod pending_attestation;
 pub mod proposal_signed_data;
 pub mod proposer_slashing;
@@ -33,11 +30,12 @@ pub mod validator_registry_delta_block;
 
 pub mod readers;
 
-use self::ethereum_types::{H160, H256, U256};
+use ethereum_types::{H160, H256, U256};
 use std::collections::HashMap;
 
 pub use crate::attestation::Attestation;
 pub use crate::attestation_data::AttestationData;
+pub use crate::attestation_data_and_custody_bit::AttestationDataAndCustodyBit;
 pub use crate::beacon_block::BeaconBlock;
 pub use crate::beacon_block_body::BeaconBlockBody;
 pub use crate::beacon_state::BeaconState;
@@ -50,6 +48,7 @@ pub use crate::eth1_data::Eth1Data;
 pub use crate::eth1_data_vote::Eth1DataVote;
 pub use crate::exit::Exit;
 pub use crate::fork::Fork;
+pub use crate::free_attestation::FreeAttestation;
 pub use crate::pending_attestation::PendingAttestation;
 pub use crate::proposal_signed_data::ProposalSignedData;
 pub use crate::proposer_slashing::ProposerSlashing;
@@ -72,4 +71,4 @@ pub type AttesterMap = HashMap<(u64, u64), Vec<usize>>;
 /// Maps a slot to a block proposer.
 pub type ProposerMap = HashMap<u64, usize>;
 
-pub use bls::{AggregatePublicKey, AggregateSignature, PublicKey, Signature};
+pub use bls::{AggregatePublicKey, AggregateSignature, Keypair, PublicKey, Signature};

eth2/types/src/pending_attestation.rs

@@ -1,9 +1,10 @@
-use super::ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 use super::{AttestationData, Bitfield};
 use crate::test_utils::TestRandom;
 use rand::RngCore;
+use serde_derive::Serialize;
+use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize)]
 pub struct PendingAttestation {
     pub data: AttestationData,
     pub aggregation_bitfield: Bitfield,
@@ -63,9 +64,9 @@ impl<T: RngCore> TestRandom<T> for PendingAttestation {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::*;
     use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/types/src/proposal_signed_data.rs

@@ -1,9 +1,10 @@
-use super::ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 use super::Hash256;
 use crate::test_utils::TestRandom;
 use rand::RngCore;
+use serde_derive::Serialize;
+use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 
-#[derive(Debug, PartialEq, Clone, Default)]
+#[derive(Debug, PartialEq, Clone, Default, Serialize)]
 pub struct ProposalSignedData {
     pub slot: u64,
     pub shard: u64,
@@ -57,9 +58,9 @@ impl<T: RngCore> TestRandom<T> for ProposalSignedData {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::*;
     use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/types/src/proposer_slashing.rs

@@ -1,10 +1,11 @@
-use super::ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 use super::ProposalSignedData;
 use crate::test_utils::TestRandom;
 use bls::Signature;
 use rand::RngCore;
+use serde_derive::Serialize;
+use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 
-#[derive(Debug, PartialEq, Clone)]
+#[derive(Debug, PartialEq, Clone, Serialize)]
 pub struct ProposerSlashing {
     pub proposer_index: u32,
     pub proposal_data_1: ProposalSignedData,
@@ -70,9 +71,9 @@ impl<T: RngCore> TestRandom<T> for ProposerSlashing {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::*;
     use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/types/src/readers/block_reader.rs

@@ -1,3 +1,4 @@
+use super::state_reader::BeaconStateReader;
 use crate::{BeaconBlock, Hash256};
 use std::fmt::Debug;
 

eth2/types/src/shard_committee.rs

@@ -1,8 +1,9 @@
-use super::ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 use crate::test_utils::TestRandom;
 use rand::RngCore;
+use serde_derive::Serialize;
+use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, Serialize)]
 pub struct ShardCommittee {
     pub shard: u64,
     pub committee: Vec<usize>,
@@ -44,9 +45,9 @@ impl<T: RngCore> TestRandom<T> for ShardCommittee {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::*;
     use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/types/src/shard_reassignment_record.rs

@@ -1,8 +1,9 @@
-use super::ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 use crate::test_utils::TestRandom;
 use rand::RngCore;
+use serde_derive::Serialize;
+use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 
-#[derive(Debug, PartialEq, Clone)]
+#[derive(Debug, PartialEq, Clone, Serialize)]
 pub struct ShardReassignmentRecord {
     pub validator_index: u64,
     pub shard: u64,
@@ -56,9 +57,9 @@ impl<T: RngCore> TestRandom<T> for ShardReassignmentRecord {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::*;
     use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/types/src/slashable_vote_data.rs

@@ -1,10 +1,11 @@
-use super::ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 use super::AttestationData;
 use crate::test_utils::TestRandom;
 use bls::AggregateSignature;
 use rand::RngCore;
+use serde_derive::Serialize;
+use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 
-#[derive(Debug, PartialEq, Clone)]
+#[derive(Debug, PartialEq, Clone, Serialize)]
 pub struct SlashableVoteData {
     pub custody_bit_0_indices: Vec<u32>,
     pub custody_bit_1_indices: Vec<u32>,
@@ -64,9 +65,9 @@ impl<T: RngCore> TestRandom<T> for SlashableVoteData {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::*;
     use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/types/src/spec/mod.rs

@@ -3,7 +3,7 @@ mod foundation;
 use crate::{Address, Eth1Data, Hash256, Validator};
 use bls::Signature;
 
-#[derive(PartialEq, Debug)]
+#[derive(PartialEq, Debug, Clone)]
 pub struct ChainSpec {
     /*
      * Misc

eth2/types/src/special_record.rs

@@ -1,9 +1,10 @@
+use serde_derive::Serialize;
 use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 
 /// The value of the "type" field of SpecialRecord.
 ///
 /// Note: this value must serialize to a u8 and therefore must not be greater than 255.
-#[derive(Debug, PartialEq, Clone, Copy)]
+#[derive(Debug, PartialEq, Clone, Copy, Serialize)]
 pub enum SpecialRecordKind {
     Logout = 0,
     CasperSlashing = 1,

eth2/types/src/validator.rs

@@ -1,13 +1,13 @@
-use super::bls::PublicKey;
 use super::Hash256;
-use crate::test_utils::TestRandom;
+use crate::{test_utils::TestRandom, PublicKey};
 use rand::RngCore;
+use serde_derive::Serialize;
 use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 
 const STATUS_FLAG_INITIATED_EXIT: u8 = 1;
 const STATUS_FLAG_WITHDRAWABLE: u8 = 2;
 
-#[derive(Debug, PartialEq, Clone, Copy)]
+#[derive(Debug, PartialEq, Clone, Copy, Serialize)]
 pub enum StatusFlags {
     InitiatedExit,
     Withdrawable,
@@ -43,7 +43,7 @@ fn status_flag_from_byte(flag: u8) -> Result<Option<StatusFlags>, StatusFlagsDec
     }
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize)]
 pub struct Validator {
     pub pubkey: PublicKey,
     pub withdrawal_credentials: Hash256,
@@ -180,9 +180,9 @@ impl<T: RngCore> TestRandom<T> for Validator {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::*;
     use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/types/src/validator_registry_delta_block.rs

@@ -2,10 +2,11 @@ use super::Hash256;
 use crate::test_utils::TestRandom;
 use bls::PublicKey;
 use rand::RngCore;
+use serde_derive::Serialize;
 use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 
 // The information gathered from the PoW chain validator registration function.
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize)]
 pub struct ValidatorRegistryDeltaBlock {
     pub latest_registry_delta_root: Hash256,
     pub validator_index: u32,
@@ -84,9 +85,9 @@ impl<T: RngCore> TestRandom<T> for ValidatorRegistryDeltaBlock {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::*;
     use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/utils/bls/Cargo.toml

@@ -8,4 +8,5 @@ edition = "2018"
 bls-aggregates = { git = "https://github.com/sigp/signature-schemes", tag = "v0.3.0" }
 hashing = { path = "../hashing" }
 hex = "0.3"
+serde = "1.0"
 ssz = { path = "../ssz" }

eth2/utils/bls/src/aggregate_signature.rs

@@ -1,6 +1,9 @@
-use super::ssz::{decode_ssz_list, hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 use super::{AggregatePublicKey, Signature};
 use bls_aggregates::AggregateSignature as RawAggregateSignature;
+use serde::ser::{Serialize, Serializer};
+use ssz::{
+    decode_ssz_list, hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash,
+};
 
 /// A BLS aggregate signature.
 ///
@@ -44,6 +47,15 @@ impl Decodable for AggregateSignature {
     }
 }
 
+impl Serialize for AggregateSignature {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_bytes(&ssz_encode(self))
+    }
+}
+
 impl TreeHash for AggregateSignature {
     fn hash_tree_root(&self) -> Vec<u8> {
         hash(&self.0.as_bytes())
@@ -52,9 +64,9 @@ impl TreeHash for AggregateSignature {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::super::{Keypair, Signature};
     use super::*;
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/utils/bls/src/lib.rs

@@ -18,7 +18,7 @@ pub use self::bls_aggregates::AggregatePublicKey;
 
 pub const BLS_AGG_SIG_BYTE_SIZE: usize = 97;
 
-use hashing::canonical_hash;
+use hashing::hash;
 use ssz::ssz_encode;
 use std::default::Default;
 
@@ -30,13 +30,23 @@ fn extend_if_needed(hash: &mut Vec<u8>) {
 /// For some signature and public key, ensure that the signature message was the public key and it
 /// was signed by the secret key that corresponds to that public key.
 pub fn verify_proof_of_possession(sig: &Signature, pubkey: &PublicKey) -> bool {
-    let mut hash = canonical_hash(&ssz_encode(pubkey));
+    let mut hash = hash(&ssz_encode(pubkey));
     extend_if_needed(&mut hash);
     sig.verify_hashed(&hash, &pubkey)
 }
 
 pub fn create_proof_of_possession(keypair: &Keypair) -> Signature {
-    let mut hash = canonical_hash(&ssz_encode(&keypair.pk));
+    let mut hash = hash(&ssz_encode(&keypair.pk));
     extend_if_needed(&mut hash);
     Signature::new_hashed(&hash, &keypair.sk)
 }
+
+pub fn bls_verify_aggregate(
+    pubkey: &AggregatePublicKey,
+    message: &[u8],
+    signature: &AggregateSignature,
+    _domain: u64,
+) -> bool {
+    // TODO: add domain
+    signature.verify(message, pubkey)
+}

eth2/utils/bls/src/public_key.rs

@@ -1,6 +1,7 @@
 use super::SecretKey;
 use bls_aggregates::PublicKey as RawPublicKey;
 use hex::encode as hex_encode;
+use serde::ser::{Serialize, Serializer};
 use ssz::{
     decode_ssz_list, hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash,
 };
@@ -55,6 +56,15 @@ impl Decodable for PublicKey {
     }
 }
 
+impl Serialize for PublicKey {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_bytes(&ssz_encode(self))
+    }
+}
+
 impl TreeHash for PublicKey {
     fn hash_tree_root(&self) -> Vec<u8> {
         hash(&self.0.as_bytes())
@@ -75,8 +85,8 @@ impl Hash for PublicKey {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::*;
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/utils/bls/src/secret_key.rs

@@ -48,8 +48,8 @@ impl TreeHash for SecretKey {
 
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::*;
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/utils/bls/src/signature.rs

@@ -1,6 +1,9 @@
-use super::ssz::{decode_ssz_list, hash, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 use super::{PublicKey, SecretKey};
 use bls_aggregates::Signature as RawSignature;
+use serde::ser::{Serialize, Serializer};
+use ssz::{
+    decode_ssz_list, hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash,
+};
 
 /// A single BLS signature.
 ///
@@ -63,11 +66,20 @@ impl TreeHash for Signature {
     }
 }
 
+impl Serialize for Signature {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_bytes(&ssz_encode(self))
+    }
+}
+
 #[cfg(test)]
 mod tests {
-    use super::super::ssz::ssz_encode;
     use super::super::Keypair;
     use super::*;
+    use ssz::ssz_encode;
 
     #[test]
     pub fn test_ssz_round_trip() {

eth2/utils/boolean-bitfield/Cargo.toml

@@ -7,3 +7,5 @@ edition = "2018"
 [dependencies]
 ssz = { path = "../ssz" }
 bit-vec = "0.5.0"
+serde = "1.0"
+serde_derive = "1.0"

eth2/utils/boolean-bitfield/src/lib.rs

@@ -3,6 +3,7 @@ extern crate ssz;
 
 use bit_vec::BitVec;
 
+use serde::ser::{Serialize, Serializer};
 use std::cmp;
 use std::default;
 
@@ -113,6 +114,28 @@ impl cmp::PartialEq for BooleanBitfield {
     }
 }
 
+/// Create a new bitfield that is a union of two other bitfields.
+///
+/// For example `union(0101, 1000) == 1101`
+impl std::ops::BitAnd for BooleanBitfield {
+    type Output = Self;
+
+    fn bitand(self, other: Self) -> Self {
+        let (biggest, smallest) = if self.len() > other.len() {
+            (&self, &other)
+        } else {
+            (&other, &self)
+        };
+        let mut new = biggest.clone();
+        for i in 0..smallest.len() {
+            if let Ok(true) = smallest.get(i) {
+                new.set(i, true);
+            }
+        }
+        new
+    }
+}
+
 impl ssz::Encodable for BooleanBitfield {
     // ssz_append encodes Self according to the `ssz` spec.
     fn ssz_append(&self, s: &mut ssz::SszStream) {
@@ -149,6 +172,15 @@ impl ssz::Decodable for BooleanBitfield {
     }
 }
 
+impl Serialize for BooleanBitfield {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_bytes(&ssz::ssz_encode(self))
+    }
+}
+
 impl ssz::TreeHash for BooleanBitfield {
     fn hash_tree_root(&self) -> Vec<u8> {
         self.to_bytes().hash_tree_root()
@@ -365,4 +397,12 @@ mod tests {
         let (decoded, _) = BooleanBitfield::ssz_decode(&ssz, 0).unwrap();
         assert_eq!(original, decoded);
     }
+
+    #[test]
+    fn test_bitand() {
+        let a = BooleanBitfield::from_bytes(&vec![2, 8, 1][..]);
+        let b = BooleanBitfield::from_bytes(&vec![4, 8, 16][..]);
+        let c = BooleanBitfield::from_bytes(&vec![6, 8, 17][..]);
+        assert_eq!(c, a & b);
+    }
 }

eth2/utils/hashing/src/lib.rs

@@ -1,8 +1,6 @@
-extern crate tiny_keccak;
-
 use tiny_keccak::Keccak;
 
-pub fn canonical_hash(input: &[u8]) -> Vec<u8> {
+pub fn hash(input: &[u8]) -> Vec<u8> {
     let mut keccak = Keccak::new_keccak256();
     keccak.update(input);
     let mut result = vec![0; 32];
@@ -19,7 +17,7 @@ mod tests {
     fn test_hashing() {
         let input: Vec<u8> = From::from("hello");
 
-        let output = canonical_hash(input.as_ref());
+        let output = hash(input.as_ref());
         let expected = &[
             0x1c, 0x8a, 0xff, 0x95, 0x06, 0x85, 0xc2, 0xed, 0x4b, 0xc3, 0x17, 0x4f, 0x34, 0x72,
             0x28, 0x7b, 0x56, 0xd9, 0x51, 0x7b, 0x9c, 0x94, 0x81, 0x27, 0x31, 0x9a, 0x09, 0xa7,

eth2/utils/slot_clock/src/system_time_slot_clock.rs

@@ -3,13 +3,14 @@ use std::time::{Duration, SystemTime};
 
 pub use std::time::SystemTimeError;
 
-#[derive(Debug)]
+#[derive(Debug, PartialEq)]
 pub enum Error {
     SlotDurationIsZero,
-    SystemTimeError(SystemTimeError),
+    SystemTimeError(String),
 }
 
 /// Determines the present slot based upon the present system time.
+#[derive(Clone)]
 pub struct SystemTimeSlotClock {
     genesis_seconds: u64,
     slot_duration_seconds: u64,
@@ -51,7 +52,7 @@ impl SlotClock for SystemTimeSlotClock {
 
 impl From<SystemTimeError> for Error {
     fn from(e: SystemTimeError) -> Error {
-        Error::SystemTimeError(e)
+        Error::SystemTimeError(format!("{:?}", e))
     }
 }
 

eth2/utils/slot_clock/src/testing_slot_clock.rs

@@ -1,11 +1,12 @@
 use super::SlotClock;
+use std::sync::RwLock;
 
 #[derive(Debug, PartialEq)]
 pub enum Error {}
 
 /// Determines the present slot based upon the present system time.
 pub struct TestingSlotClock {
-    slot: u64,
+    slot: RwLock<u64>,
 }
 
 impl TestingSlotClock {
@@ -13,11 +14,13 @@ impl TestingSlotClock {
     ///
     /// Returns an Error if `slot_duration_seconds == 0`.
     pub fn new(slot: u64) -> TestingSlotClock {
-        TestingSlotClock { slot }
+        TestingSlotClock {
+            slot: RwLock::new(slot),
+        }
     }
 
-    pub fn set_slot(&mut self, slot: u64) {
+    pub fn set_slot(&self, slot: u64) {
-        self.slot = slot;
+        *self.slot.write().expect("TestingSlotClock poisoned.") = slot;
     }
 }
 
@@ -25,7 +28,8 @@ impl SlotClock for TestingSlotClock {
     type Error = Error;
 
     fn present_slot(&self) -> Result<Option<u64>, Error> {
-        Ok(Some(self.slot))
+        let slot = *self.slot.read().expect("TestingSlotClock poisoned.");
+        Ok(Some(slot))
     }
 }
 
@@ -35,7 +39,7 @@ mod tests {
 
     #[test]
     fn test_slot_now() {
-        let mut clock = TestingSlotClock::new(10);
+        let clock = TestingSlotClock::new(10);
         assert_eq!(clock.present_slot(), Ok(Some(10)));
         clock.set_slot(123);
         assert_eq!(clock.present_slot(), Ok(Some(123)));

eth2/utils/ssz/src/impl_tree_hash.rs

@@ -1,5 +1,6 @@
 use super::ethereum_types::{Address, H256};
-use super::{hash, merkle_hash, ssz_encode, TreeHash};
+use super::{merkle_hash, ssz_encode, TreeHash};
+use hashing::hash;
 
 impl TreeHash for u8 {
     fn hash_tree_root(&self) -> Vec<u8> {

eth2/utils/ssz/src/lib.rs

@@ -20,7 +20,9 @@ mod impl_tree_hash;
 
 pub use crate::decode::{decode_ssz, decode_ssz_list, Decodable, DecodeError};
 pub use crate::encode::{Encodable, SszStream};
-pub use crate::tree_hash::{hash, merkle_hash, TreeHash};
+pub use crate::tree_hash::{merkle_hash, TreeHash};
+
+pub use hashing::hash;
 
 pub const LENGTH_BYTES: usize = 4;
 pub const MAX_LIST_SIZE: usize = 1 << (4 * 8);

eth2/utils/ssz/src/tree_hash.rs

@@ -1,4 +1,4 @@
-use hashing::canonical_hash;
+use hashing::hash;
 
 const SSZ_CHUNK_SIZE: usize = 128;
 const HASHSIZE: usize = 32;
@@ -65,10 +65,6 @@ fn list_to_blob(list: &mut Vec<Vec<u8>>) -> (usize, Vec<u8>) {
     (chunk_size, data)
 }
 
-pub fn hash(data: &[u8]) -> Vec<u8> {
-    canonical_hash(data)
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;