Merge branch 'master' into validator-record-update

beacon_chain/attestation_validation/src/signature.rs

@@ -27,7 +27,7 @@ where
                 .get_public_key_by_index(validator)?
                 .ok_or(Error::NoPublicKeyForValidator)?;
             // Aggregate the public key.
-            agg_pub_key.add(&pub_key);
+            agg_pub_key.add(&pub_key.as_raw());
         }
     }
 
@@ -129,7 +129,8 @@ mod tests {
             &agg_sig,
             &attestation_indices,
             &store,
-        ).unwrap();
+        )
+        .unwrap();
         assert_eq!(outcome, Outcome::Valid);
 
         /*
@@ -143,7 +144,8 @@ mod tests {
             &agg_sig,
             &attestation_indices,
             &store,
-        ).unwrap();
+        )
+        .unwrap();
         assert_eq!(outcome, Outcome::Invalid(Invalid::SignatureInvalid));
     }
 }

beacon_chain/chain/src/maps.rs

@@ -45,7 +45,7 @@ mod tests {
     use super::*;
 
     fn sac_generator(
-        shard_count: u16,
+        shard_count: u64,
         slot_count: usize,
         sac_per_slot: usize,
         committee_size: usize,

beacon_chain/types/src/beacon_state.rs

@@ -6,6 +6,9 @@ use super::shard_committee::ShardCommittee;
 use super::shard_reassignment_record::ShardReassignmentRecord;
 use super::validator_record::ValidatorRecord;
 use super::Hash256;
+use crate::test_utils::TestRandom;
+use rand::RngCore;
+use ssz::{Decodable, DecodeError, Encodable, SszStream};
 
 #[derive(Debug, PartialEq, Clone, Default)]
 pub struct BeaconState {
@@ -52,3 +55,114 @@ impl BeaconState {
         Hash256::zero()
     }
 }
+
+impl Encodable for BeaconState {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append(&self.slot);
+        s.append(&self.genesis_time);
+        s.append(&self.fork_data);
+        s.append(&self.validator_registry);
+        s.append(&self.validator_registry_latest_change_slot);
+        s.append(&self.validator_registry_exit_count);
+        s.append(&self.validator_registry_delta_chain_tip);
+        s.append(&self.randao_mix);
+        s.append(&self.next_seed);
+        s.append(&self.shard_committees_at_slots);
+        s.append(&self.persistent_committees);
+        s.append(&self.persistent_committee_reassignments);
+        s.append(&self.previous_justified_slot);
+        s.append(&self.justified_slot);
+        s.append(&self.justification_bitfield);
+        s.append(&self.finalized_slot);
+        s.append(&self.latest_crosslinks);
+        s.append(&self.latest_block_roots);
+        s.append(&self.latest_penalized_exit_balances);
+        s.append(&self.latest_attestations);
+        s.append(&self.processed_pow_receipt_root);
+        s.append(&self.candidate_pow_receipt_roots);
+    }
+}
+
+impl Decodable for BeaconState {
+    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
+        let (slot, i) = <_>::ssz_decode(bytes, i)?;
+        let (genesis_time, i) = <_>::ssz_decode(bytes, i)?;
+        let (fork_data, i) = <_>::ssz_decode(bytes, i)?;
+        let (validator_registry, i) = <_>::ssz_decode(bytes, i)?;
+        let (validator_registry_latest_change_slot, i) = <_>::ssz_decode(bytes, i)?;
+        let (validator_registry_exit_count, i) = <_>::ssz_decode(bytes, i)?;
+        let (validator_registry_delta_chain_tip, i) = <_>::ssz_decode(bytes, i)?;
+        let (randao_mix, i) = <_>::ssz_decode(bytes, i)?;
+        let (next_seed, i) = <_>::ssz_decode(bytes, i)?;
+        let (shard_committees_at_slots, i) = <_>::ssz_decode(bytes, i)?;
+        let (persistent_committees, i) = <_>::ssz_decode(bytes, i)?;
+        let (persistent_committee_reassignments, i) = <_>::ssz_decode(bytes, i)?;
+        let (previous_justified_slot, i) = <_>::ssz_decode(bytes, i)?;
+        let (justified_slot, i) = <_>::ssz_decode(bytes, i)?;
+        let (justification_bitfield, i) = <_>::ssz_decode(bytes, i)?;
+        let (finalized_slot, i) = <_>::ssz_decode(bytes, i)?;
+        let (latest_crosslinks, i) = <_>::ssz_decode(bytes, i)?;
+        let (latest_block_roots, i) = <_>::ssz_decode(bytes, i)?;
+        let (latest_penalized_exit_balances, i) = <_>::ssz_decode(bytes, i)?;
+        let (latest_attestations, i) = <_>::ssz_decode(bytes, i)?;
+        let (processed_pow_receipt_root, i) = <_>::ssz_decode(bytes, i)?;
+        let (candidate_pow_receipt_roots, i) = <_>::ssz_decode(bytes, i)?;
+
+        Ok((
+            Self {
+                slot,
+                genesis_time,
+                fork_data,
+                validator_registry,
+                validator_registry_latest_change_slot,
+                validator_registry_exit_count,
+                validator_registry_delta_chain_tip,
+                randao_mix,
+                next_seed,
+                shard_committees_at_slots,
+                persistent_committees,
+                persistent_committee_reassignments,
+                previous_justified_slot,
+                justified_slot,
+                justification_bitfield,
+                finalized_slot,
+                latest_crosslinks,
+                latest_block_roots,
+                latest_penalized_exit_balances,
+                latest_attestations,
+                processed_pow_receipt_root,
+                candidate_pow_receipt_roots,
+            },
+            i,
+        ))
+    }
+}
+
+impl<T: RngCore> TestRandom<T> for BeaconState {
+    fn random_for_test(rng: &mut T) -> Self {
+        Self {
+            slot: <_>::random_for_test(rng),
+            genesis_time: <_>::random_for_test(rng),
+            fork_data: <_>::random_for_test(rng),
+            validator_registry: <_>::random_for_test(rng),
+            validator_registry_latest_change_slot: <_>::random_for_test(rng),
+            validator_registry_exit_count: <_>::random_for_test(rng),
+            validator_registry_delta_chain_tip: <_>::random_for_test(rng),
+            randao_mix: <_>::random_for_test(rng),
+            next_seed: <_>::random_for_test(rng),
+            shard_committees_at_slots: <_>::random_for_test(rng),
+            persistent_committees: <_>::random_for_test(rng),
+            persistent_committee_reassignments: <_>::random_for_test(rng),
+            previous_justified_slot: <_>::random_for_test(rng),
+            justified_slot: <_>::random_for_test(rng),
+            justification_bitfield: <_>::random_for_test(rng),
+            finalized_slot: <_>::random_for_test(rng),
+            latest_crosslinks: <_>::random_for_test(rng),
+            latest_block_roots: <_>::random_for_test(rng),
+            latest_penalized_exit_balances: <_>::random_for_test(rng),
+            latest_attestations: <_>::random_for_test(rng),
+            processed_pow_receipt_root: <_>::random_for_test(rng),
+            candidate_pow_receipt_roots: <_>::random_for_test(rng),
+        }
+    }
+}

beacon_chain/types/src/candidate_pow_receipt_root_record.rs

@@ -1,7 +1,60 @@
+use super::ssz::{Decodable, DecodeError, Encodable, SszStream};
 use super::Hash256;
+use crate::test_utils::TestRandom;
+use rand::RngCore;
 
 #[derive(Debug, PartialEq, Clone)]
 pub struct CandidatePoWReceiptRootRecord {
     pub candidate_pow_receipt_root: Hash256,
     pub votes: u64,
 }
+
+impl Encodable for CandidatePoWReceiptRootRecord {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append(&self.candidate_pow_receipt_root);
+        s.append(&self.votes);
+    }
+}
+
+impl Decodable for CandidatePoWReceiptRootRecord {
+    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
+        let (candidate_pow_receipt_root, i) = <_>::ssz_decode(bytes, i)?;
+        let (votes, i) = <_>::ssz_decode(bytes, i)?;
+
+        Ok((
+            Self {
+                candidate_pow_receipt_root,
+                votes,
+            },
+            i,
+        ))
+    }
+}
+
+impl<T: RngCore> TestRandom<T> for CandidatePoWReceiptRootRecord {
+    fn random_for_test(rng: &mut T) -> Self {
+        Self {
+            candidate_pow_receipt_root: <_>::random_for_test(rng),
+            votes: <_>::random_for_test(rng),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::ssz::ssz_encode;
+    use super::*;
+    use crate::test_utils::TestRandom;
+    use rand::{prng::XorShiftRng, SeedableRng};
+
+    #[test]
+    pub fn test_ssz_round_trip() {
+        let mut rng = XorShiftRng::from_seed([42; 16]);
+        let original = CandidatePoWReceiptRootRecord::random_for_test(&mut rng);
+
+        let bytes = ssz_encode(&original);
+        let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap();
+
+        assert_eq!(original, decoded);
+    }
+}

beacon_chain/types/src/common/maps.rs

@@ -1,7 +0,0 @@
-use std::collections::HashMap;
-
-/// Maps a (slot, shard_id) to attestation_indices.
-pub type AttesterMap = HashMap<(u64, u16), Vec<usize>>;
-
-/// Maps a slot to a block proposer.
-pub type ProposerMap = HashMap<u64, usize>;

beacon_chain/types/src/common/mod.rs

@@ -1,9 +0,0 @@
-mod delegation;
-mod shuffling;
-
-pub mod maps;
-pub mod attestation_parent_hashes;
-
-use super::utils;
-use super::utils::types::Hash256;
-pub use self::shuffling::shuffle;

beacon_chain/types/src/crosslink_record.rs

@@ -1,4 +1,7 @@
+use super::ssz::{Decodable, DecodeError, Encodable, SszStream};
 use super::Hash256;
+use crate::test_utils::TestRandom;
+use rand::RngCore;
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct CrosslinkRecord {
@@ -15,3 +18,53 @@ impl CrosslinkRecord {
         }
     }
 }
+
+impl Encodable for CrosslinkRecord {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append(&self.slot);
+        s.append(&self.shard_block_root);
+    }
+}
+
+impl Decodable for CrosslinkRecord {
+    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
+        let (slot, i) = <_>::ssz_decode(bytes, i)?;
+        let (shard_block_root, i) = <_>::ssz_decode(bytes, i)?;
+
+        Ok((
+            Self {
+                slot,
+                shard_block_root,
+            },
+            i,
+        ))
+    }
+}
+
+impl<T: RngCore> TestRandom<T> for CrosslinkRecord {
+    fn random_for_test(rng: &mut T) -> Self {
+        Self {
+            slot: <_>::random_for_test(rng),
+            shard_block_root: <_>::random_for_test(rng),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::ssz::ssz_encode;
+    use super::*;
+    use crate::test_utils::TestRandom;
+    use rand::{prng::XorShiftRng, SeedableRng};
+
+    #[test]
+    pub fn test_ssz_round_trip() {
+        let mut rng = XorShiftRng::from_seed([42; 16]);
+        let original = CrosslinkRecord::random_for_test(&mut rng);
+
+        let bytes = ssz_encode(&original);
+        let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap();
+
+        assert_eq!(original, decoded);
+    }
+}

beacon_chain/types/src/deposit_input.rs

@@ -1,4 +1,4 @@
-use super::ssz::{decode_ssz_list, Decodable, DecodeError, Encodable, SszStream};
+use super::ssz::{Decodable, DecodeError, Encodable, SszStream};
 use super::Hash256;
 use crate::test_utils::TestRandom;
 use bls::{PublicKey, Signature};
@@ -15,7 +15,7 @@ pub struct DepositInput {
 
 impl Encodable for DepositInput {
     fn ssz_append(&self, s: &mut SszStream) {
-        s.append_vec(&self.pubkey.as_bytes());
+        s.append(&self.pubkey);
         s.append(&self.withdrawal_credentials);
         s.append(&self.randao_commitment);
         s.append(&self.poc_commitment);
@@ -25,8 +25,7 @@ impl Encodable for DepositInput {
 
 impl Decodable for DepositInput {
     fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
-        let (pubkey_bytes, i) = decode_ssz_list(bytes, i)?;
+        let (pubkey, i) = <_>::ssz_decode(bytes, i)?;
-        let pubkey = PublicKey::from_bytes(&pubkey_bytes).map_err(|_| DecodeError::TooShort)?;
         let (withdrawal_credentials, i) = <_>::ssz_decode(bytes, i)?;
         let (randao_commitment, i) = <_>::ssz_decode(bytes, i)?;
         let (poc_commitment, i) = <_>::ssz_decode(bytes, i)?;

beacon_chain/types/src/fork_data.rs

@@ -1,6 +1,64 @@
-#[derive(Debug, Clone, PartialEq, Default)]
+use super::ssz::{Decodable, DecodeError, Encodable, SszStream};
+use crate::test_utils::TestRandom;
+use rand::RngCore;
+
+#[derive(Debug, Clone, PartialEq)]
 pub struct ForkData {
     pub pre_fork_version: u64,
     pub post_fork_version: u64,
     pub fork_slot: u64,
 }
+
+impl Encodable for ForkData {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append(&self.pre_fork_version);
+        s.append(&self.post_fork_version);
+        s.append(&self.fork_slot);
+    }
+}
+
+impl Decodable for ForkData {
+    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
+        let (pre_fork_version, i) = <_>::ssz_decode(bytes, i)?;
+        let (post_fork_version, i) = <_>::ssz_decode(bytes, i)?;
+        let (fork_slot, i) = <_>::ssz_decode(bytes, i)?;
+
+        Ok((
+            Self {
+                pre_fork_version,
+                post_fork_version,
+                fork_slot,
+            },
+            i,
+        ))
+    }
+}
+
+impl<T: RngCore> TestRandom<T> for ForkData {
+    fn random_for_test(rng: &mut T) -> Self {
+        Self {
+            pre_fork_version: <_>::random_for_test(rng),
+            post_fork_version: <_>::random_for_test(rng),
+            fork_slot: <_>::random_for_test(rng),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::ssz::ssz_encode;
+    use super::*;
+    use crate::test_utils::TestRandom;
+    use rand::{prng::XorShiftRng, SeedableRng};
+
+    #[test]
+    pub fn test_ssz_round_trip() {
+        let mut rng = XorShiftRng::from_seed([42; 16]);
+        let original = ForkData::random_for_test(&mut rng);
+
+        let bytes = ssz_encode(&original);
+        let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap();
+
+        assert_eq!(original, decoded);
+    }
+}

beacon_chain/types/src/lib.rs

@@ -63,7 +63,7 @@ pub type Bitfield = boolean_bitfield::BooleanBitfield;
 pub type BitfieldError = boolean_bitfield::Error;
 
 /// Maps a (slot, shard_id) to attestation_indices.
-pub type AttesterMap = HashMap<(u64, u16), Vec<usize>>;
+pub type AttesterMap = HashMap<(u64, u64), Vec<usize>>;
 
 /// Maps a slot to a block proposer.
 pub type ProposerMap = HashMap<u64, usize>;

beacon_chain/types/src/pending_attestation_record.rs

@@ -1,4 +1,7 @@
+use super::ssz::{Decodable, DecodeError, Encodable, SszStream};
 use super::{AttestationData, Bitfield};
+use crate::test_utils::TestRandom;
+use rand::RngCore;
 
 #[derive(Debug, Clone, PartialEq)]
 pub struct PendingAttestationRecord {
@@ -7,3 +10,61 @@ pub struct PendingAttestationRecord {
     pub custody_bitfield: Bitfield,
     pub slot_included: u64,
 }
+
+impl Encodable for PendingAttestationRecord {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append(&self.data);
+        s.append(&self.participation_bitfield);
+        s.append(&self.custody_bitfield);
+        s.append(&self.slot_included);
+    }
+}
+
+impl Decodable for PendingAttestationRecord {
+    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
+        let (data, i) = <_>::ssz_decode(bytes, i)?;
+        let (participation_bitfield, i) = <_>::ssz_decode(bytes, i)?;
+        let (custody_bitfield, i) = <_>::ssz_decode(bytes, i)?;
+        let (slot_included, i) = <_>::ssz_decode(bytes, i)?;
+
+        Ok((
+            Self {
+                data,
+                participation_bitfield,
+                custody_bitfield,
+                slot_included,
+            },
+            i,
+        ))
+    }
+}
+
+impl<T: RngCore> TestRandom<T> for PendingAttestationRecord {
+    fn random_for_test(rng: &mut T) -> Self {
+        Self {
+            data: <_>::random_for_test(rng),
+            participation_bitfield: <_>::random_for_test(rng),
+            custody_bitfield: <_>::random_for_test(rng),
+            slot_included: <_>::random_for_test(rng),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::ssz::ssz_encode;
+    use super::*;
+    use crate::test_utils::TestRandom;
+    use rand::{prng::XorShiftRng, SeedableRng};
+
+    #[test]
+    pub fn test_ssz_round_trip() {
+        let mut rng = XorShiftRng::from_seed([42; 16]);
+        let original = PendingAttestationRecord::random_for_test(&mut rng);
+
+        let bytes = ssz_encode(&original);
+        let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap();
+
+        assert_eq!(original, decoded);
+    }
+}

beacon_chain/types/src/shard_committee.rs

@@ -1,28 +1,53 @@
+use super::ssz::{Decodable, DecodeError, Encodable, SszStream};
+use crate::test_utils::TestRandom;
+use rand::RngCore;
+
 #[derive(Clone, Debug, PartialEq)]
 pub struct ShardCommittee {
-    pub shard: u16,
+    pub shard: u64,
     pub committee: Vec<usize>,
 }
 
-impl ShardCommittee {
+impl Encodable for ShardCommittee {
-    /// Returns a new instance where the `shard_id` is zero and the
+    fn ssz_append(&self, s: &mut SszStream) {
-    /// committee is an empty vector.
+        s.append(&self.shard);
-    pub fn zero() -> Self {
+        s.append(&self.committee);
+    }
+}
+
+impl Decodable for ShardCommittee {
+    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
+        let (shard, i) = <_>::ssz_decode(bytes, i)?;
+        let (committee, i) = <_>::ssz_decode(bytes, i)?;
+
+        Ok((Self { shard, committee }, i))
+    }
+}
+
+impl<T: RngCore> TestRandom<T> for ShardCommittee {
+    fn random_for_test(rng: &mut T) -> Self {
         Self {
-            shard: 0,
+            shard: <_>::random_for_test(rng),
-            committee: vec![],
+            committee: <_>::random_for_test(rng),
         }
     }
 }
 
 #[cfg(test)]
 mod tests {
+    use super::super::ssz::ssz_encode;
     use super::*;
+    use crate::test_utils::TestRandom;
+    use rand::{prng::XorShiftRng, SeedableRng};
 
     #[test]
-    fn test_shard_and_committee_zero() {
+    pub fn test_ssz_round_trip() {
-        let s = ShardCommittee::zero();
+        let mut rng = XorShiftRng::from_seed([42; 16]);
-        assert_eq!(s.shard, 0);
+        let original = ShardCommittee::random_for_test(&mut rng);
-        assert_eq!(s.committee.len(), 0);
+
+        let bytes = ssz_encode(&original);
+        let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap();
+
+        assert_eq!(original, decoded);
     }
 }

beacon_chain/types/src/shard_reassignment_record.rs

@@ -1,6 +1,64 @@
+use super::ssz::{Decodable, DecodeError, Encodable, SszStream};
+use crate::test_utils::TestRandom;
+use rand::RngCore;
+
 #[derive(Debug, PartialEq, Clone)]
 pub struct ShardReassignmentRecord {
     pub validator_index: u64,
     pub shard: u64,
     pub slot: u64,
 }
+
+impl Encodable for ShardReassignmentRecord {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append(&self.validator_index);
+        s.append(&self.shard);
+        s.append(&self.slot);
+    }
+}
+
+impl Decodable for ShardReassignmentRecord {
+    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
+        let (validator_index, i) = <_>::ssz_decode(bytes, i)?;
+        let (shard, i) = <_>::ssz_decode(bytes, i)?;
+        let (slot, i) = <_>::ssz_decode(bytes, i)?;
+
+        Ok((
+            Self {
+                validator_index,
+                shard,
+                slot,
+            },
+            i,
+        ))
+    }
+}
+
+impl<T: RngCore> TestRandom<T> for ShardReassignmentRecord {
+    fn random_for_test(rng: &mut T) -> Self {
+        Self {
+            validator_index: <_>::random_for_test(rng),
+            shard: <_>::random_for_test(rng),
+            slot: <_>::random_for_test(rng),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::ssz::ssz_encode;
+    use super::*;
+    use crate::test_utils::TestRandom;
+    use rand::{prng::XorShiftRng, SeedableRng};
+
+    #[test]
+    pub fn test_ssz_round_trip() {
+        let mut rng = XorShiftRng::from_seed([42; 16]);
+        let original = ShardReassignmentRecord::random_for_test(&mut rng);
+
+        let bytes = ssz_encode(&original);
+        let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap();
+
+        assert_eq!(original, decoded);
+    }
+}

beacon_chain/types/src/test_utils/address.rs

@@ -0,0 +1,11 @@
+use super::TestRandom;
+use crate::Address;
+use rand::RngCore;
+
+impl<T: RngCore> TestRandom<T> for Address {
+    fn random_for_test(rng: &mut T) -> Self {
+        let mut key_bytes = vec![0; 20];
+        rng.fill_bytes(&mut key_bytes);
+        Address::from(&key_bytes[..])
+    }
+}

beacon_chain/types/src/test_utils/mod.rs

@@ -2,6 +2,7 @@ use rand::RngCore;
 
 pub use rand::{prng::XorShiftRng, SeedableRng};
 
+pub mod address;
 pub mod aggregate_signature;
 pub mod bitfield;
 pub mod hash256;
@@ -27,6 +28,12 @@ impl<T: RngCore> TestRandom<T> for u32 {
     }
 }
 
+impl<T: RngCore> TestRandom<T> for usize {
+    fn random_for_test(rng: &mut T) -> Self {
+        rng.next_u32() as usize
+    }
+}
+
 impl<T: RngCore, U> TestRandom<T> for Vec<U>
 where U: TestRandom<T>
 {

beacon_chain/types/src/validator_record.rs

@@ -1,5 +1,8 @@
-use super::bls::{Keypair, PublicKey};
+use super::bls::PublicKey;
-use super::{Hash256};
+use super::{Address, Hash256};
+use crate::test_utils::TestRandom;
+use rand::RngCore;
+use ssz::{Decodable, DecodeError, Encodable, SszStream};
 use std::convert;
 
 #[derive(Debug, PartialEq, Clone)]
@@ -41,47 +44,136 @@ pub struct ValidatorRecord {
 }
 
 impl ValidatorRecord {
-    /// Generates a new instance where the keypair is generated using
-    /// `rand::thread_rng` entropy and all other fields are set to zero.
-    ///
-    /// Returns the new instance and new keypair.
-    pub fn zero_with_thread_rand_keypair() -> (Self, Keypair) {
-        let keypair = Keypair::random();
-        let s = Self {
-            pubkey: keypair.pk.clone(),
-            withdrawal_credentials: Hash256::zero(),
-            randao_commitment: Hash256::zero(),
-            randao_layers: 0,
-            status: From::from(0),
-            latest_status_change_slot: 0,
-            exit_count: 0,
-            poc_commitment: Hash256::zero(),
-            last_poc_change_slot: 0,
-            second_last_poc_slot: 0
-        };
-        (s, keypair)
-    }
-
     pub fn status_is(&self, status: ValidatorStatus) -> bool {
         self.status == status
     }
 }
 
+impl Encodable for ValidatorStatus {
+    fn ssz_append(&self, s: &mut SszStream) {
+        let byte: u8 = match self {
+            ValidatorStatus::PendingActivation => 0,
+            ValidatorStatus::Active => 1,
+            ValidatorStatus::PendingExit => 2,
+            ValidatorStatus::PendingWithdraw => 3,
+            ValidatorStatus::Withdrawn => 5,
+            ValidatorStatus::Penalized => 127,
+        };
+        s.append(&byte);
+    }
+}
+
+impl Decodable for ValidatorStatus {
+    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
+        let (byte, i) = u8::ssz_decode(bytes, i)?;
+        let status = match byte {
+            0 => ValidatorStatus::PendingActivation,
+            1 => ValidatorStatus::Active,
+            2 => ValidatorStatus::PendingExit,
+            3 => ValidatorStatus::PendingWithdraw,
+            5 => ValidatorStatus::Withdrawn,
+            127 => ValidatorStatus::Penalized,
+            _ => return Err(DecodeError::Invalid),
+        };
+        Ok((status, i))
+    }
+}
+
+impl<T: RngCore> TestRandom<T> for ValidatorStatus {
+    fn random_for_test(rng: &mut T) -> Self {
+        let options = vec![
+            ValidatorStatus::PendingActivation,
+            ValidatorStatus::Active,
+            ValidatorStatus::PendingExit,
+            ValidatorStatus::PendingWithdraw,
+            ValidatorStatus::Withdrawn,
+            ValidatorStatus::Penalized,
+        ];
+        options[(rng.next_u32() as usize) % options.len()].clone()
+    }
+}
+
+impl Encodable for ValidatorRecord {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append(&self.pubkey);
+        s.append(&self.withdrawal_shard);
+        s.append(&self.withdrawal_address);
+        s.append(&self.randao_commitment);
+        s.append(&self.randao_last_change);
+        s.append(&self.balance);
+        s.append(&self.status);
+        s.append(&self.exit_slot);
+    }
+}
+
+impl Decodable for ValidatorRecord {
+    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
+        let (pubkey, i) = <_>::ssz_decode(bytes, i)?;
+        let (withdrawal_shard, i) = <_>::ssz_decode(bytes, i)?;
+        let (withdrawal_address, i) = <_>::ssz_decode(bytes, i)?;
+        let (randao_commitment, i) = <_>::ssz_decode(bytes, i)?;
+        let (randao_last_change, i) = <_>::ssz_decode(bytes, i)?;
+        let (balance, i) = <_>::ssz_decode(bytes, i)?;
+        let (status, i) = <_>::ssz_decode(bytes, i)?;
+        let (exit_slot, i) = <_>::ssz_decode(bytes, i)?;
+
+        Ok((
+            Self {
+                pubkey,
+                withdrawal_shard,
+                withdrawal_address,
+                randao_commitment,
+                randao_last_change,
+                balance,
+                status,
+                exit_slot,
+            },
+            i,
+        ))
+    }
+}
+
+impl<T: RngCore> TestRandom<T> for ValidatorRecord {
+    fn random_for_test(rng: &mut T) -> Self {
+        Self {
+            pubkey: <_>::random_for_test(rng),
+            withdrawal_shard: <_>::random_for_test(rng),
+            withdrawal_address: <_>::random_for_test(rng),
+            randao_commitment: <_>::random_for_test(rng),
+            randao_last_change: <_>::random_for_test(rng),
+            balance: <_>::random_for_test(rng),
+            status: <_>::random_for_test(rng),
+            exit_slot: <_>::random_for_test(rng),
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
+    use super::super::ssz::ssz_encode;
     use super::*;
+    use crate::test_utils::TestRandom;
+    use rand::{prng::XorShiftRng, SeedableRng};
 
     #[test]
-    fn test_validator_record_zero_rand_keypair() {
+    pub fn test_ssz_round_trip() {
-        let (v, _kp) = ValidatorRecord::zero_with_thread_rand_keypair();
+        let mut rng = XorShiftRng::from_seed([42; 16]);
-        assert!(v.withdrawal_credentials.is_zero());
+        let original = ValidatorRecord::random_for_test(&mut rng);
-        assert!(v.randao_commitment.is_zero());
+
-        assert_eq!(v.randao_layers, 0);
+        let bytes = ssz_encode(&original);
-        assert_eq!(v.status, From::from(0));
+        let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap();
-        assert_eq!(v.latest_status_change_slot, 0);
+
-        assert_eq!(v.exit_count, 0);
+        assert_eq!(original, decoded);
-        assert!(v.poc_commitment.is_zero());
+    }
-        assert_eq!(v.last_poc_change_slot, 0);
+
-        assert_eq!(v.second_last_poc_slot, 0);
+    #[test]
+    pub fn test_validator_status_ssz_round_trip() {
+        let mut rng = XorShiftRng::from_seed([42; 16]);
+        let original = ValidatorStatus::random_for_test(&mut rng);
+
+        let bytes = ssz_encode(&original);
+        let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap();
+
+        assert_eq!(original, decoded);
     }
 }

beacon_chain/types/src/validator_registration.rs

@@ -1,5 +1,5 @@
 use super::{Address, Hash256};
-use bls::{create_proof_of_possession, Keypair, PublicKey, Signature};
+use bls::{PublicKey, Signature};
 
 /// The information gathered from the PoW chain validator registration function.
 #[derive(Debug, Clone, PartialEq)]
@@ -10,17 +10,3 @@ pub struct ValidatorRegistration {
     pub randao_commitment: Hash256,
     pub proof_of_possession: Signature,
 }
-
-impl ValidatorRegistration {
-    pub fn random() -> Self {
-        let keypair = Keypair::random();
-
-        Self {
-            pubkey: keypair.pk.clone(),
-            withdrawal_shard: 0,
-            withdrawal_address: Address::random(),
-            randao_commitment: Hash256::random(),
-            proof_of_possession: create_proof_of_possession(&keypair),
-        }
-    }
-}

beacon_chain/utils/bls/src/keypair.rs

@@ -0,0 +1,16 @@
+use super::{PublicKey, SecretKey};
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct Keypair {
+    pub sk: SecretKey,
+    pub pk: PublicKey,
+}
+
+impl Keypair {
+    /// Instantiate a Keypair using SecretKey::random().
+    pub fn random() -> Self {
+        let sk = SecretKey::random();
+        let pk = PublicKey::from_secret_key(&sk);
+        Keypair { sk, pk }
+    }
+}

beacon_chain/utils/bls/src/lib.rs

@@ -3,20 +3,24 @@ extern crate hashing;
 extern crate ssz;
 
 mod aggregate_signature;
+mod keypair;
+mod public_key;
+mod secret_key;
 mod signature;
 
 pub use crate::aggregate_signature::AggregateSignature;
+pub use crate::keypair::Keypair;
+pub use crate::public_key::PublicKey;
+pub use crate::secret_key::SecretKey;
 pub use crate::signature::Signature;
 
 pub use self::bls_aggregates::AggregatePublicKey;
-pub use self::bls_aggregates::Keypair;
-pub use self::bls_aggregates::PublicKey;
-pub use self::bls_aggregates::SecretKey;
 
 pub const BLS_AGG_SIG_BYTE_SIZE: usize = 97;
 
 use hashing::canonical_hash;
 use std::default::Default;
+use ssz::ssz_encode;
 
 fn extend_if_needed(hash: &mut Vec<u8>) {
     // NOTE: bls_aggregates crate demands 48 bytes, this may be removed as we get closer to production
@@ -26,13 +30,13 @@ fn extend_if_needed(hash: &mut Vec<u8>) {
 /// For some signature and public key, ensure that the signature message was the public key and it
 /// was signed by the secret key that corresponds to that public key.
 pub fn verify_proof_of_possession(sig: &Signature, pubkey: &PublicKey) -> bool {
-    let mut hash = canonical_hash(&pubkey.as_bytes());
+    let mut hash = canonical_hash(&ssz_encode(pubkey));
     extend_if_needed(&mut hash);
     sig.verify_hashed(&hash, &pubkey)
 }
 
 pub fn create_proof_of_possession(keypair: &Keypair) -> Signature {
-    let mut hash = canonical_hash(&keypair.pk.as_bytes());
+    let mut hash = canonical_hash(&ssz_encode(&keypair.pk));
     extend_if_needed(&mut hash);
     Signature::new_hashed(&hash, &keypair.sk)
 }

beacon_chain/utils/bls/src/public_key.rs

@@ -0,0 +1,52 @@
+use super::SecretKey;
+use bls_aggregates::PublicKey as RawPublicKey;
+use ssz::{decode_ssz_list, Decodable, DecodeError, Encodable, SszStream};
+
+/// A single BLS signature.
+///
+/// This struct is a wrapper upon a base type and provides helper functions (e.g., SSZ
+/// serialization).
+#[derive(Debug, PartialEq, Clone)]
+pub struct PublicKey(RawPublicKey);
+
+impl PublicKey {
+    pub fn from_secret_key(secret_key: &SecretKey) -> Self {
+        PublicKey(RawPublicKey::from_secret_key(secret_key.as_raw()))
+    }
+
+    /// Returns the underlying signature.
+    pub fn as_raw(&self) -> &RawPublicKey {
+        &self.0
+    }
+}
+
+impl Encodable for PublicKey {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append_vec(&self.0.as_bytes());
+    }
+}
+
+impl Decodable for PublicKey {
+    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
+        let (sig_bytes, i) = decode_ssz_list(bytes, i)?;
+        let raw_sig = RawPublicKey::from_bytes(&sig_bytes).map_err(|_| DecodeError::TooShort)?;
+        Ok((PublicKey(raw_sig), i))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::ssz::ssz_encode;
+    use super::*;
+
+    #[test]
+    pub fn test_ssz_round_trip() {
+        let sk = SecretKey::random();
+        let original = PublicKey::from_secret_key(&sk);
+
+        let bytes = ssz_encode(&original);
+        let (decoded, _) = PublicKey::ssz_decode(&bytes, 0).unwrap();
+
+        assert_eq!(original, decoded);
+    }
+}

beacon_chain/utils/bls/src/secret_key.rs

@@ -0,0 +1,59 @@
+use bls_aggregates::{DecodeError as BlsDecodeError, SecretKey as RawSecretKey};
+use ssz::{decode_ssz_list, Decodable, DecodeError, Encodable, SszStream};
+
+/// A single BLS signature.
+///
+/// This struct is a wrapper upon a base type and provides helper functions (e.g., SSZ
+/// serialization).
+#[derive(Debug, PartialEq, Clone)]
+pub struct SecretKey(RawSecretKey);
+
+impl SecretKey {
+    pub fn random() -> Self {
+        SecretKey(RawSecretKey::random())
+    }
+
+    /// Instantiate a SecretKey from existing bytes.
+    ///
+    /// Note: this is _not_ SSZ decoding.
+    pub fn from_bytes(bytes: &[u8]) -> Result<SecretKey, BlsDecodeError> {
+        Ok(SecretKey(RawSecretKey::from_bytes(bytes)?))
+    }
+
+    /// Returns the underlying secret key.
+    pub fn as_raw(&self) -> &RawSecretKey {
+        &self.0
+    }
+}
+
+impl Encodable for SecretKey {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append_vec(&self.0.as_bytes());
+    }
+}
+
+impl Decodable for SecretKey {
+    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
+        let (sig_bytes, i) = decode_ssz_list(bytes, i)?;
+        let raw_sig = RawSecretKey::from_bytes(&sig_bytes).map_err(|_| DecodeError::TooShort)?;
+        Ok((SecretKey(raw_sig), i))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::ssz::ssz_encode;
+    use super::*;
+
+    #[test]
+    pub fn test_ssz_round_trip() {
+        let original =
+            SecretKey::from_bytes("jzjxxgjajfjrmgodszzsgqccmhnyvetcuxobhtynojtpdtbj".as_bytes())
+                .unwrap();
+
+        let bytes = ssz_encode(&original);
+        let (decoded, _) = SecretKey::ssz_decode(&bytes, 0).unwrap();
+
+        assert_eq!(original, decoded);
+    }
+}

beacon_chain/utils/bls/src/signature.rs

@@ -1,5 +1,6 @@
 use super::ssz::{decode_ssz_list, Decodable, DecodeError, Encodable, SszStream};
-use bls_aggregates::{PublicKey, SecretKey, Signature as RawSignature};
+use super::{PublicKey, SecretKey};
+use bls_aggregates::Signature as RawSignature;
 
 /// A single BLS signature.
 ///
@@ -11,23 +12,23 @@ pub struct Signature(RawSignature);
 impl Signature {
     /// Instantiate a new Signature from a message and a SecretKey.
     pub fn new(msg: &[u8], sk: &SecretKey) -> Self {
-        Signature(RawSignature::new(msg, sk))
+        Signature(RawSignature::new(msg, sk.as_raw()))
     }
 
     /// Instantiate a new Signature from a message and a SecretKey, where the message has already
     /// been hashed.
     pub fn new_hashed(msg_hashed: &[u8], sk: &SecretKey) -> Self {
-        Signature(RawSignature::new_hashed(msg_hashed, sk))
+        Signature(RawSignature::new_hashed(msg_hashed, sk.as_raw()))
     }
 
     /// Verify the Signature against a PublicKey.
     pub fn verify(&self, msg: &[u8], pk: &PublicKey) -> bool {
-        self.0.verify(msg, pk)
+        self.0.verify(msg, pk.as_raw())
     }
 
     /// Verify the Signature against a PublicKey, where the message has already been hashed.
     pub fn verify_hashed(&self, msg_hash: &[u8], pk: &PublicKey) -> bool {
-        self.0.verify_hashed(msg_hash, pk)
+        self.0.verify_hashed(msg_hash, pk.as_raw())
     }
 
     /// Returns the underlying signature.

beacon_chain/utils/ssz/src/decode.rs

@@ -4,6 +4,7 @@ use super::LENGTH_BYTES;
 pub enum DecodeError {
     TooShort,
     TooLong,
+    Invalid,
 }
 
 pub trait Decodable: Sized {
@@ -148,7 +149,8 @@ mod tests {
                 0, 0, 0, 16, 0, 0, 0, 10, 0, 0, 0, 10, 0, 0, 0, 10, 0, 0, 0, 10,
             ],
             0,
-        ).unwrap();
+        )
+        .unwrap();
         assert_eq!(decoded.0, v);
         assert_eq!(decoded.1, 20);
 
@@ -160,7 +162,8 @@ mod tests {
                 10, 0, 0, 0, 0, 0, 0, 0, 10,
             ],
             0,
-        ).unwrap();
+        )
+        .unwrap();
         assert_eq!(decoded.0, v);
         assert_eq!(decoded.1, 36);
 
@@ -172,7 +175,8 @@ mod tests {
                 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 15,
             ],
             10,
-        ).unwrap();
+        )
+        .unwrap();
         assert_eq!(decoded.0, v);
         assert_eq!(decoded.1, 46);
 

beacon_chain/utils/ssz/src/impl_decode.rs

@@ -1,5 +1,5 @@
 use super::decode::decode_ssz_list;
-use super::ethereum_types::H256;
+use super::ethereum_types::{Address, H256};
 use super::{Decodable, DecodeError};
 
 macro_rules! impl_decodable_for_uint {
@@ -49,6 +49,16 @@ impl Decodable for H256 {
     }
 }
 
+impl Decodable for Address {
+    fn ssz_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), DecodeError> {
+        if bytes.len() < 20 || bytes.len() - 20 < index {
+            Err(DecodeError::TooShort)
+        } else {
+            Ok((Address::from(&bytes[index..(index + 20)]), index + 20))
+        }
+    }
+}
+
 impl<T> Decodable for Vec<T>
 where
     T: Decodable,

beacon_chain/utils/ssz/src/impl_encode.rs

@@ -1,7 +1,7 @@
 extern crate bytes;
 
 use self::bytes::{BufMut, BytesMut};
-use super::ethereum_types::H256;
+use super::ethereum_types::{Address, H256};
 use super::{Encodable, SszStream};
 
 /*
@@ -52,6 +52,21 @@ impl Encodable for H256 {
     }
 }
 
+impl Encodable for Address {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append_encoded_raw(&self.to_vec());
+    }
+}
+
+impl<T> Encodable for Vec<T>
+where
+    T: Encodable,
+{
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append_vec(&self);
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

beacon_chain/validator_change/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "validator_change"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+[dependencies]
+bytes = "0.4.10"
+hashing = { path = "../utils/hashing" }
+ssz = { path = "../utils/ssz" }
+types = { path = "../types" }

beacon_chain/validator_change/src/lib.rs

@@ -0,0 +1,142 @@
+extern crate bytes;
+extern crate hashing;
+extern crate types;
+
+use bytes::{BufMut, BytesMut};
+use hashing::canonical_hash;
+use ssz::ssz_encode;
+use std::cmp::max;
+use types::{Hash256, ValidatorRecord, ValidatorStatus};
+
+pub enum UpdateValidatorSetError {
+    ArithmeticOverflow,
+}
+
+const VALIDATOR_FLAG_ENTRY: u8 = 0;
+const VALIDATOR_FLAG_EXIT: u8 = 1;
+
+pub fn update_validator_set(
+    validators: &mut Vec<ValidatorRecord>,
+    hash_chain: Hash256,
+    present_slot: u64,
+    deposit_size_gwei: u64,
+    max_validator_churn_quotient: u64,
+) -> Result<(), UpdateValidatorSetError> {
+    /*
+     * Total balance of all active validators.
+     *
+     * Return an error if an overflow occurs.
+     */
+    let total_balance = {
+        let mut bal: u64 = 0;
+        for v in validators.iter() {
+            if v.status_is(ValidatorStatus::Active) {
+                bal = bal
+                    .checked_add(v.balance)
+                    .ok_or(UpdateValidatorSetError::ArithmeticOverflow)?;
+            }
+        }
+        bal
+    };
+
+    /*
+     * Note: this is not the maximum allowable change, it can actually be higher.
+     */
+    let max_allowable_change = {
+        let double_deposit_size = deposit_size_gwei
+            .checked_mul(2)
+            .ok_or(UpdateValidatorSetError::ArithmeticOverflow)?;
+        max(
+            double_deposit_size,
+            total_balance / max_validator_churn_quotient,
+        )
+    };
+
+    let mut hasher = ValidatorChangeHashChain {
+        bytes: hash_chain.to_vec(),
+    };
+    let mut total_changed: u64 = 0;
+    for (i, v) in validators.iter_mut().enumerate() {
+        match v.status {
+            /*
+             * Validator is pending activiation.
+             */
+            ValidatorStatus::PendingActivation => {
+                let new_total_changed = total_changed
+                    .checked_add(deposit_size_gwei)
+                    .ok_or(UpdateValidatorSetError::ArithmeticOverflow)?;
+                /*
+                 * If entering this validator would not exceed the max balance delta,
+                 * activate the validator.
+                 */
+                if new_total_changed <= max_allowable_change {
+                    v.status = ValidatorStatus::Active;
+                    hasher.extend(i, &ssz_encode(&v.pubkey), VALIDATOR_FLAG_ENTRY);
+                    total_changed = new_total_changed;
+                } else {
+                    // Entering the validator would exceed the balance delta.
+                    break;
+                }
+            }
+            /*
+             * Validator is pending exit.
+             */
+            ValidatorStatus::PendingExit => {
+                let new_total_changed = total_changed
+                    .checked_add(v.balance)
+                    .ok_or(UpdateValidatorSetError::ArithmeticOverflow)?;
+                /*
+                 * If exiting this validator would not exceed the max balance delta,
+                 * exit the validator
+                 */
+                if new_total_changed <= max_allowable_change {
+                    v.status = ValidatorStatus::PendingWithdraw;
+                    v.exit_slot = present_slot;
+                    hasher.extend(i, &ssz_encode(&v.pubkey), VALIDATOR_FLAG_EXIT);
+                    total_changed = new_total_changed;
+                } else {
+                    // Exiting the validator would exceed the balance delta.
+                    break;
+                }
+            }
+            _ => (),
+        };
+        if total_changed >= max_allowable_change {
+            break;
+        }
+    }
+    Ok(())
+}
+
+pub struct ValidatorChangeHashChain {
+    bytes: Vec<u8>,
+}
+
+impl ValidatorChangeHashChain {
+    pub fn extend(&mut self, index: usize, pubkey: &Vec<u8>, flag: u8) {
+        let mut message = self.bytes.clone();
+        message.append(&mut serialize_validator_change_record(index, pubkey, flag));
+        self.bytes = canonical_hash(&message);
+    }
+}
+
+fn serialize_validator_change_record(index: usize, pubkey: &Vec<u8>, flag: u8) -> Vec<u8> {
+    let mut buf = BytesMut::with_capacity(68);
+    buf.put_u8(flag);
+    let index_bytes = {
+        let mut buf = BytesMut::with_capacity(8);
+        buf.put_u64_be(index as u64);
+        buf.take()[8 - 3..8].to_vec()
+    };
+    buf.put(index_bytes);
+    buf.put(pubkey);
+    buf.take().to_vec()
+}
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn it_works() {
+        assert_eq!(2 + 2, 4);
+    }
+}

beacon_chain/validator_shuffling/src/shuffle.rs

@@ -95,7 +95,7 @@ fn generate_cycle(
                 .honey_badger_split(committees_per_slot)
                 .enumerate()
                 .map(|(j, shard_indices)| ShardCommittee {
-                    shard: ((shard_start + j) % shard_count) as u16,
+                    shard: ((shard_start + j) % shard_count) as u64,
                     committee: shard_indices.to_vec(),
                 })
                 .collect()

docs/onboarding.md

@@ -1,36 +1,286 @@
-# Learn how to contribute to ETH 2.0!
+# Contributing to Lighthouse
 
-Lighthouse is an Ethereum 2.0 client built in Rust.
+Lighthouse is an open-source Ethereum Serenity client built in
+[Rust](https://www.rust-lang.org/).
 
-If you are interested in contributing to the Ethereum ecosystem, and you want to learn Rust, Lighthouse is a great project to work on.
+Lighthouse welcomes all contributions with open arms. If you are interested in
+contributing to the Ethereum ecosystem, and you want to learn Rust, Lighthouse
+is a great project to work on.
 
-Initially this doc will contain reading material to help get you started in Rust and Ethereum. Eventually it will have guides specific to Lighthouse.
+This documentation aims to provide a smooth on-boarding for all who wish to
+help contribute to Lighthouse. Whether it is helping with the mountain of
+documentation, writing extra tests or developing components, all help is
+appreciated and your contributions will help not only the community but all
+the contributors.
 
-## Learn Rust
+If you have any additional questions, please feel free to jump on the
+[gitter](https://gitter.im/sigp/lighthouse) and have a chat with all of us.
 
-* [The Rust Programming Language](https://doc.rust-lang.org/book/2018-edition/index.html)
+## Ideology
-* [Learning Rust With Entirely Too Many Linked Lists](http://cglab.ca/~abeinges/blah/too-many-lists/book/)
-* [Rustlings](https://github.com/rustlings/rustlings) - Small exercises to get you used to reading and writing Rust code.
-* [Rust Exercism](https://exercism.io/tracks/rust) - 88 exercises to learn Rust.
 
-## Learn Ethereum
+### Never Panic
+
+Lighthouse will be the gateway interacting with the Proof-of-Stake system
+employed by Ethereum. This requires the validation and proposal of blocks
+and extremely timely responses. As part of this, Lighthouse aims to ensure
+the most uptime as possible, meaning minimising the amount of
+exceptions and gracefully handling any issues.
+
+Rust's `panic` provides the ability to throw an exception and exit, this
+will terminate the running processes. Thus, Lighthouse aims to use `panic`
+as little as possible to minimise the possible termination cases.
+
+### Security First Mindset
+
+Lighthouse aims to provide a safe, secure Serenity client for the Ethereum
+ecosystem. At each step of development, the aim is to have a security-first
+mindset and always ensure you are following the safe, secure mindset. When
+contributing to any part of the Lighthouse client, through any development,
+always ensure you understand each aspect thoroughly and cover all potential
+security considerations of your code.
+
+### Functions aren't completed until they are tested
+
+As part of the Security First mindset, we want to aim to cover as many distinct
+cases. A function being developed is not considered "completed" until tests
+exist for that function. The tests not only help show the correctness of the
+function, but also provide a way for new developers to understand how the
+function is to be called and how it works.
+
+## Understanding Serenity
+
+Ethereum's Serenity is based on a Proof-of-Stake based sharded beacon chain.
+
+(*If you don't know what that is, don't `panic`, that's what this documentation
+is for!* :smile:)
+
+### Ethereum
+
+Ethereum is an open blockchain protocol, allowing for the building and use of
+decentralized applications that run on blockchain technology. The blockchain can
+be seen as a decentralized, distributed ledger of transactions.
+
+General Ethereum Introduction:
 
-#### General Ethereum Resources
 * [What is Ethereum](http://ethdocs.org/en/latest/introduction/what-is-ethereum.html)
 * [Ethereum Introduction](https://github.com/ethereum/wiki/wiki/Ethereum-introduction)
 
-#### Ethereum 2.0
-* [Ethereum 2.0 Spec - Casper and Sharding](https://github.com/ethereum/eth2.0-specs/blob/master/specs/beacon-chain.md)
 
-#### Sharding
+### Proof-of-Work and the current state of Ethereum.
 
-* [How to Scale Ethereum: Sharding Explained](https://medium.com/prysmatic-labs/how-to-scale-ethereum-sharding-explained-ba2e283b7fce)
+Currently, Ethereum is based on the Proof-of-Work model, a Sybil resilient
+mechanism to allow nodes to propose blocks to the network. Although it provides
+properties that allow the blockchain to operate in an open, public
+(permissionless) network, it faces it's challenges and as a result impacts
+the operation of the blockchain.
 
-#### Casper
+The main goals to advance Ethereum is to (1) increase the scalability and
+overall transaction processing power of the Ethereum world computer and (2)
+find a suitable replacement for Proof-of-Work that still provides the necessary
+properties that we need.
 
+* [Proof-of-Work in Cryptocurrencies: an accessible introduction](https://blog.sigmaprime.io/what-is-proof-of-work.html)
+
+### Serenity
+
+As part of the original Ethereum roadmap
+[\[1\]](https://blog.ethereum.org/2015/03/03/ethereum-launch-process/)
+[\[2\]](http://ethdocs.org/en/latest/introduction/the-homestead-release.html),
+the Proof-of-Stake integration falls under **Release Step 4:*Serenity***. With
+this, a number of changes are to be made to the current Ethereum protocol to
+incorporate some of the new Proof-of-Stake mechanisms as well as improve on
+some of the hindrances faced by the current Proof-of-Work chain.
+
+To now advance the current Ethereum, the decision is made to move to a sharded
+Beacon chain structure where multiple shard-chains will be operating and
+interacting with a central beacon chain.
+
+(Be mindful, the specifications change occasionally, so check these to keep up
+to date)
+
+* Current Specifications:
+  * [Danny Ryan's "State of the Spec"](https://notes.ethereum.org/s/BJEZWNoyE) (A nice summary of the current specifications)
+  * [Ethereum Serenity - Phase 0: Beacon Chain Spec](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md)
+  * [Ethereum Serenity - Phase 1: Sharded Data Chains](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/1_shard-data-chains.md)
+  * [Beacon Chain - Vitalik Buterin and Justin Drake explain](https://www.youtube.com/watch?v=GAywmwGToUI)
+* Understanding Sharding:
+  * [Prysmatic Labs: Sharding Explained](https://medium.com/prysmatic-labs/how-to-scale-ethereum-sharding-explained-ba2e283b7fce)
+* Other relevant resources
   * [Proof of Stake - Casper FFG](https://www.youtube.com/watch?v=uQ3IqLDf-oo)
-* [Beacon Casper Chain](https://www.youtube.com/watch?v=GAywmwGToUI)
 
-### TODO
+## Development Onboarding
-- add reading material as we discover.
+
-- start developing guides specific to lighthouse.
+If you would like to contribute and develop Lighthouse, there are only a few
+things to go through (and then you're on your way!).
+
+### Understanding Rust
+
+Rust is an extremely powerful, low-level programming language that provides
+freedom and performance to create powerful projects. The [Rust 
+Book](https://doc.rust-lang.org/stable/book/) provides insight into the Rust
+language and some of the coding style to follow (As well as acting as a great
+introduction and tutorial for the language.)
+
+Rust has a steep learning curve, but there are many resources to help you!
+
+* [Rust Book](https://doc.rust-lang.org/stable/book/)
+* [Rust by example](https://doc.rust-lang.org/stable/rust-by-example/)
+* [Learning Rust With Entirely Too Many Linked Lists](http://cglab.ca/~abeinges/blah/too-many-lists/book/)
+* [Rustlings](https://github.com/rustlings/rustlings)
+* [Rust Exercism](https://exercism.io/tracks/rust)
+* [Learn X in Y minutes - Rust](https://learnxinyminutes.com/docs/rust/)
+
+
+#### Getting Started and installing Rust
+
+We recommend installing Rust using [**rustup**](https://rustup.rs/). Rustup
+allows you to easily install versions of rust.
+
+**Linux/Unix/Mac:**
+
+```
+$ curl https://sh.rustup.rs -sSf | sh
+```
+
+**Windows (You need a bit more):**
+
+* Install the Visual Studio 2015 with C++ support
+* Install Rustup using: https://static.rust-lang.org/rustup/dist/x86_64-pc-windows-msvc/rustup-init.exe
+* You can then use the ``VS2015 x64 Native Tools Command Prompt`` and run:
+
+```
+rustup default stable-x86-64-pc-windows-msvc
+```
+
+#### Getting ready with Cargo
+
+[Cargo](https://doc.rust-lang.org/cargo/) is the package manager for Rust, and
+allows to extend to a number of packages and external libraries. It's also extremely
+handy for handling dependencies and helping to modularise your project better.
+
+*Note: If you've installed rust through rustup, you should have ``cargo``
+installed.*
+
+
+#### Rust Terminology
+
+When developing rust, you'll come across some terminology that differs to
+other programming languages you may have used.
+
+* **Trait**: A trait is a collection of methods defined for a type, they can be
+implemented for any data type.
+* **Struct**: A custom data type that lets us name and package together
+multiple related values that make a meaninguful group.
+* **Crate**: A crate is synonymous with a *library* or *package* in other
+languages. They can produce an executable or library depending on the
+project.
+* **Module**: A collection of items: functions, structs, traits, and even other
+modules. Modules allow you to hierarchically split code into logical units
+and manage visibility.
+* **Attribute**: Metadaata applied to some module, crate or item.
+* **Macros**: Macros are powerful meta-programming statements that get expanded
+into source code that gets compiled with the rest of the code (Unlike `C`
+macros that are pre-processed, Rust macros form an Abstract Syntax Tree).
+
+
+Other good appendix resources:
+
+* [Keywords](https://doc.rust-lang.org/book/appendix-01-keywords.html)
+* [Operators/Symbols](https://doc.rust-lang.org/book/appendix-02-operators.html)
+* [Traits](https://doc.rust-lang.org/book/appendix-03-derivable-traits.html)
+
+
+### Understanding the Git Workflow
+
+Lighthouse utilises git as the primary open-source development tool. To help
+with your contributions, it is great to understand the processes used to ensure
+everything remains in sync and there's as little conflict as possible when
+working on similar files.
+
+Lighthouse uses the **feature branch** workflow, where each issue, or each
+feature, is developed on its own branch and then merged in via a pull-request.
+
+* [Feature Branch Tutorial](https://www.atlassian.com/git/tutorials/comparing-workflows/feature-branch-workflow)
+
+## Code Conventions/Styleguide and Ethos
+
+### Ethos
+
+**Pull Requests**
+
+Pull requests should be reviewed by **at least** one "*core developer*"
+(someone with write-access to the repo). This should ensure bugs are caught and
+the code is kept in a consistent state that follows all conventions and style.
+
+All discussion (whether in PRs or Issues or in the Gitter) should be respectful
+and intellectual. Have fun, but always respect the limits of other people.
+
+**Testing**
+
+Generally, tests can be self-contained in the same file. Integration tests
+should be added into the ``tests/`` directory in the crate's **root**.
+
+Large line-count tests should be in a separate file.
+
+### Rust StyleGuide
+
+Lighthouse adheres to Rust code conventions as outlined in the [**Rust
+Styleguide**](https://github.com/rust-dev-tools/fmt-rfcs/blob/master/guide/guide.md).
+
+Ensure you use [Clippy](https://github.com/rust-lang/rust-clippy) to lint and
+check your code.
+
+| Code Aspect         | Guideline Format               |
+|:--------------------|:-------------------------------|
+| Types               | ``UpperCamelCase``             |
+| Enums/Enum Variants | ``UpperCamelCase``             |
+| Struct Fields       | ``snake_case``                 |
+| Function / Method   | ``snake_case``                 |
+| Macro Names         | ``snake_case``                 |
+| Constants           | ``SCREAMING_SNAKE_CASE``       |
+| Forbidden name      | Trialing Underscore: ``name_`` |
+
+Other general rust docs:
+
+* [Rust Other Style Advice](https://github.com/rust-dev-tools/fmt-rfcs/blob/master/guide/advice.md)
+* [Cargo.toml Conventions](https://github.com/rust-dev-tools/fmt-rfcs/blob/master/guide/cargo.md)
+
+### TODOs
+
+All `TODO` statements should be accompanied by a GitHub issue.
+
+```rust
+pub fn my_function(&mut self, _something &[u8]) -> Result<String, Error> {
+  // TODO: something_here
+  // https://github.com/sigp/lighthouse/issues/XX
+}
+```
+
+### Comments
+
+**General Comments**
+
+* Prefer line (``//``) comments to block comments (``/* ... */``)
+* Comments can appear on the line prior to the item or after a trailing space.
+```rust
+// Comment for this struct
+struct Lighthouse {}
+
+fn make_blockchain() {} // A comment on the same line after a space
+```
+
+**Doc Comments**
+
+* The ``///`` is used to generate comments for Docs. 
+* The comments should come before attributes.
+
+```rust
+/// Stores the core configuration for this Lighthouse instance.
+/// This struct is general, other components may implement more
+/// specialized config structs.
+#[derive(Clone)]
+pub struct LighthouseConfig {
+    pub data_dir: PathBuf,
+    pub p2p_listen_port: u16,
+}
+```

lighthouse/db/src/stores/validator_store.rs

@@ -4,6 +4,7 @@ use self::bytes::{BufMut, BytesMut};
 use super::bls::PublicKey;
 use super::VALIDATOR_DB_COLUMN as DB_COLUMN;
 use super::{ClientDB, DBError};
+use ssz::{ssz_encode, Decodable};
 use std::sync::Arc;
 
 #[derive(Debug, PartialEq)]
@@ -54,7 +55,7 @@ impl<T: ClientDB> ValidatorStore<T> {
         public_key: &PublicKey,
     ) -> Result<(), ValidatorStoreError> {
         let key = self.get_db_key_for_index(&KeyPrefixes::PublicKey, index);
-        let val = public_key.as_bytes();
+        let val = ssz_encode(public_key);
         self.db
             .put(DB_COLUMN, &key[..], &val[..])
             .map_err(ValidatorStoreError::from)
@@ -68,8 +69,8 @@ impl<T: ClientDB> ValidatorStore<T> {
         let val = self.db.get(DB_COLUMN, &key[..])?;
         match val {
             None => Ok(None),
-            Some(val) => match PublicKey::from_bytes(&val) {
+            Some(val) => match PublicKey::ssz_decode(&val, 0) {
-                Ok(key) => Ok(Some(key)),
+                Ok((key, _)) => Ok(Some(key)),
                 Err(_) => Err(ValidatorStoreError::DecodeError),
             },
         }
@@ -87,7 +88,10 @@ mod tests {
         let db = Arc::new(MemoryDB::open());
         let store = ValidatorStore::new(db.clone());
 
-        assert_eq!(store.prefix_bytes(&KeyPrefixes::PublicKey), b"pubkey".to_vec());
+        assert_eq!(
+            store.prefix_bytes(&KeyPrefixes::PublicKey),
+            b"pubkey".to_vec()
+        );
     }
 
     #[test]
@@ -98,7 +102,10 @@ mod tests {
         let mut buf = BytesMut::with_capacity(6 + 8);
         buf.put(b"pubkey".to_vec());
         buf.put_u64_be(42);
-        assert_eq!(store.get_db_key_for_index(&KeyPrefixes::PublicKey, 42), buf.take().to_vec())
+        assert_eq!(
+            store.get_db_key_for_index(&KeyPrefixes::PublicKey, 42),
+            buf.take().to_vec()
+        )
     }
 
     #[test]
@@ -110,12 +117,15 @@ mod tests {
         let public_key = Keypair::random().pk;
 
         store.put_public_key_by_index(index, &public_key).unwrap();
-        let public_key_at_index = db.get(
+        let public_key_at_index = db
+            .get(
                 DB_COLUMN,
-            &store.get_db_key_for_index(&KeyPrefixes::PublicKey, index)[..]
+                &store.get_db_key_for_index(&KeyPrefixes::PublicKey, index)[..],
-        ).unwrap().unwrap();
+            )
+            .unwrap()
+            .unwrap();
 
-        assert_eq!(public_key_at_index, public_key.as_bytes());
+        assert_eq!(public_key_at_index, ssz_encode(&public_key));
     }
 
     #[test]
@@ -129,8 +139,9 @@ mod tests {
         db.put(
             DB_COLUMN,
             &store.get_db_key_for_index(&KeyPrefixes::PublicKey, index)[..],
-            &public_key.as_bytes()[..]
+            &ssz_encode(&public_key)[..],
-        ).unwrap();
+        )
+        .unwrap();
 
         let public_key_at_index = store.get_public_key_by_index(index).unwrap().unwrap();
         assert_eq!(public_key_at_index, public_key);
@@ -146,8 +157,9 @@ mod tests {
         db.put(
             DB_COLUMN,
             &store.get_db_key_for_index(&KeyPrefixes::PublicKey, 3)[..],
-            &public_key.as_bytes()[..]
+            &ssz_encode(&public_key)[..],
-        ).unwrap();
+        )
+        .unwrap();
 
         let public_key_at_index = store.get_public_key_by_index(4).unwrap();
         assert_eq!(public_key_at_index, None);
@@ -195,11 +207,9 @@ mod tests {
         /*
          * Check that an index that wasn't stored returns None.
          */
-        assert!(
+        assert!(store
-            store
             .get_public_key_by_index(keys.len() + 1)
             .unwrap()
-                .is_none()
+            .is_none());
-        );
     }
 }