Merge pull request #63 from sigp/genesis

[BLOCKED] Add genesis and "chain" concept

Cargo.toml

@@ -32,16 +32,20 @@ name = "lighthouse"
 
 [workspace]
 members = [
+	"beacon_chain/chain",
 	"beacon_chain/types",
-	"beacon_chain/transition",
+	"beacon_chain/utils/active-validators",
 	"beacon_chain/utils/bls",
 	"beacon_chain/utils/boolean-bitfield",
 	"beacon_chain/utils/hashing",
 	"beacon_chain/utils/honey-badger-split",
-	"beacon_chain/utils/shuffling",
+	"beacon_chain/utils/slot-clock",
 	"beacon_chain/utils/ssz",
 	"beacon_chain/utils/ssz_helpers",
+	"beacon_chain/utils/vec_shuffle",
 	"beacon_chain/validation",
+	"beacon_chain/validator_change",
 	"beacon_chain/validator_induction",
+	"beacon_chain/validator_shuffling",
 	"lighthouse/db",
 ]

beacon_chain/chain/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "chain"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+
+[dependencies]
+bls = { path = "../utils/bls" }
+db = { path = "../../lighthouse/db" }
+ssz_helpers = { path = "../utils/ssz_helpers" }
+types = { path = "../types" }
+validation = { path = "../validation" }
+validator_induction = { path = "../validator_induction" }
+validator_shuffling = { path = "../validator_shuffling" }

beacon_chain/chain/src/block_preprocessing.rs

@@ -0,0 +1,132 @@
+extern crate ssz_helpers;
+extern crate validation;
+
+use db::{
+    ClientDB,
+};
+use db::stores::{
+    BeaconBlockAtSlotError,
+};
+use self::validation::block_validation::{
+    BeaconBlockValidationContext,
+    SszBeaconBlockValidationError,
+};
+use super::{
+    BeaconChain,
+    BeaconChainError,
+};
+use self::ssz_helpers::ssz_beacon_block::{
+    SszBeaconBlock,
+    SszBeaconBlockError,
+};
+use std::sync::Arc;
+use types::{
+    BeaconBlock,
+    Hash256,
+};
+
+pub use self::validation::block_validation::BeaconBlockStatus;
+
+pub enum BeaconChainBlockError {
+    UnknownCrystallizedState,
+    UnknownActiveState,
+    UnknownAttesterProposerMaps,
+    NoParentHash,
+    UnknownJustifiedBlock,
+    BlockAlreadyKnown,
+    BlockSlotLookupError(BeaconBlockAtSlotError),
+    BadSsz(SszBeaconBlockError),
+    BlockValidationError(SszBeaconBlockValidationError),
+    DBError(String),
+}
+
+impl From<BeaconBlockAtSlotError> for BeaconChainBlockError {
+    fn from(e: BeaconBlockAtSlotError) -> BeaconChainBlockError {
+        BeaconChainBlockError::BlockSlotLookupError(e)
+    }
+}
+
+impl From<SszBeaconBlockValidationError> for BeaconChainBlockError {
+    fn from(e: SszBeaconBlockValidationError) -> BeaconChainBlockError {
+        BeaconChainBlockError::BlockValidationError(e)
+    }
+}
+
+pub type BlockStatusTriple = (BeaconBlockStatus, Hash256, BeaconBlock);
+
+impl<T> BeaconChain<T>
+    where T: ClientDB + Sized
+{
+    fn block_preprocessing(&self, ssz: &[u8], present_slot: u64)
+        -> Result<BlockStatusTriple, BeaconChainBlockError>
+    {
+        /*
+         * Generate a SszBlock to read directly from the serialized SSZ.
+         */
+        let block = SszBeaconBlock::from_slice(ssz)?;
+        let block_hash = Hash256::from(&block.block_hash()[..]);
+
+        /*
+         * Load the crystallized state for this block from our caches.
+         *
+         * Fail if the crystallized state is unknown.
+         */
+        let cry_state_root = Hash256::from(block.cry_state_root());
+        let cry_state = self.crystallized_states.get(&cry_state_root)
+            .ok_or(BeaconChainBlockError::UnknownCrystallizedState)?;
+
+        /*
+         * Load the active state for this block from our caches.
+         *
+         * Fail if the active state is unknown.
+         */
+        let act_state_root = Hash256::from(block.act_state_root());
+        let act_state = self.active_states.get(&act_state_root)
+            .ok_or(BeaconChainBlockError::UnknownActiveState)?;
+
+        /*
+         * Learn the last justified slot from the crystallized state and load
+         * the hash of this block from the database
+         */
+        let last_justified_slot = cry_state.last_justified_slot;
+        let parent_block_hash = block.parent_hash()
+            .ok_or(BeaconChainBlockError::NoParentHash)?;
+        let (last_justified_block_hash, _) = self.store.block.block_at_slot(
+            &parent_block_hash, last_justified_slot)?
+            .ok_or(BeaconChainBlockError::UnknownJustifiedBlock)?;
+
+        /*
+         * Load the attester and proposer maps for the crystallized state.
+         */
+        let (attester_map, proposer_map) = self.attester_proposer_maps.get(&cry_state_root)
+            .ok_or(BeaconChainBlockError::UnknownAttesterProposerMaps)?;
+
+        /*
+         * Build a block validation context to test the block against.
+         */
+        let validation_context = BeaconBlockValidationContext {
+            present_slot,
+            cycle_length: self.config.cycle_length,
+            last_justified_slot: cry_state.last_justified_slot,
+            last_justified_block_hash: Hash256::from(&last_justified_block_hash[..]),
+            last_finalized_slot: self.last_finalized_slot,
+            recent_block_hashes: Arc::new(act_state.recent_block_hashes.clone()),
+            proposer_map: proposer_map.clone(),
+            attester_map: attester_map.clone(),
+            block_store: self.store.block.clone(),
+            validator_store: self.store.validator.clone(),
+            pow_store: self.store.pow_chain.clone(),
+        };
+        let (block_status, deserialized_block) = validation_context.validate_ssz_block(&block_hash, &block)?;
+        match deserialized_block {
+            Some(b) => Ok((block_status, block_hash, b)),
+            None => Err(BeaconChainBlockError::BlockAlreadyKnown)
+        }
+    }
+}
+
+impl From<SszBeaconBlockError> for BeaconChainBlockError {
+    fn from(e: SszBeaconBlockError) -> BeaconChainBlockError {
+        BeaconChainBlockError::BadSsz(e)
+    }
+}

beacon_chain/chain/src/genesis.rs

@@ -0,0 +1,204 @@
+use types::{
+    CrosslinkRecord,
+    Hash256,
+    ValidatorRegistration,
+    ValidatorStatus,
+};
+use super::{
+    ActiveState,
+    CrystallizedState,
+    BeaconChainError,
+    ChainConfig,
+};
+use validator_induction::ValidatorInductor;
+use validator_shuffling::{
+    shard_and_committees_for_cycle,
+    ValidatorAssignmentError,
+};
+
+pub const INITIAL_FORK_VERSION: u32 = 0;
+
+impl From<ValidatorAssignmentError> for BeaconChainError {
+    fn from(_: ValidatorAssignmentError) -> BeaconChainError {
+        BeaconChainError::InvalidGenesis
+    }
+}
+
+/// Initialize a new ChainHead with genesis parameters.
+///
+/// Used when syncing a chain from scratch.
+pub fn genesis_states(config: &ChainConfig)
+    -> Result<(ActiveState, CrystallizedState), ValidatorAssignmentError>
+{
+    /*
+     * Parse the ValidatorRegistrations into ValidatorRecords and induct them.
+     *
+     * Ignore any records which fail proof-of-possession or are invalid.
+     */
+    let validators = {
+        let mut inductor = ValidatorInductor::new(0, config.shard_count, vec![]);
+        for registration in &config.initial_validators {
+            let _ = inductor.induct(&registration, ValidatorStatus::Active);
+        };
+        inductor.to_vec()
+    };
+
+    /*
+     * Assign the validators to shards, using all zeros as the seed.
+     *
+     * Crystallizedstate stores two cycles, so we simply repeat the same assignment twice.
+     */
+    let shard_and_committee_for_slots = {
+        let mut a = shard_and_committees_for_cycle(&vec![0; 32], &validators, 0, &config)?;
+        let mut b = a.clone();
+        a.append(&mut b);
+        a
+    };
+
+    /*
+     * Set all the crosslink records to reference zero hashes.
+     */
+    let crosslinks = {
+        let mut c = vec![];
+        for _ in 0..config.shard_count {
+            c.push(CrosslinkRecord {
+                recently_changed: false,
+                slot: 0,
+                hash: Hash256::zero(),
+            });
+        }
+        c
+    };
+
+    /*
+     * Initialize a genesis `Crystallizedstate`
+     */
+    let crystallized_state = CrystallizedState {
+        validator_set_change_slot: 0,
+        validators: validators.to_vec(),
+        crosslinks,
+        last_state_recalculation_slot: 0,
+        last_finalized_slot: 0,
+        last_justified_slot: 0,
+        justified_streak: 0,
+        shard_and_committee_for_slots,
+        deposits_penalized_in_period: vec![],
+        validator_set_delta_hash_chain: Hash256::zero(),
+        pre_fork_version: INITIAL_FORK_VERSION,
+        post_fork_version: INITIAL_FORK_VERSION,
+        fork_slot_number: 0,
+    };
+
+    /*
+     * Set all recent block hashes to zero.
+     */
+    let recent_block_hashes = vec![Hash256::zero(); config.cycle_length as usize];
+
+    /*
+     * Create an active state.
+     */
+    let active_state = ActiveState {
+        pending_attestations: vec![],
+        pending_specials: vec![],
+        recent_block_hashes,
+        randao_mix: Hash256::zero(),
+    };
+
+    Ok((active_state, crystallized_state))
+}
+
+
+#[cfg(test)]
+mod tests {
+    extern crate validator_induction;
+    extern crate bls;
+
+    use super::*;
+    use self::bls::{
+        create_proof_of_possession,
+        Keypair,
+    };
+    use types::{
+        Hash256,
+        Address,
+    };
+
+    #[test]
+    fn test_genesis_no_validators() {
+        let config = ChainConfig::standard();
+        let (act, cry) = genesis_states(&config).unwrap();
+
+        assert_eq!(cry.validator_set_change_slot, 0);
+        assert_eq!(cry.validators.len(), 0);
+        assert_eq!(cry.crosslinks.len(), config.shard_count as usize);
+        for cl in cry.crosslinks {
+            assert_eq!(cl.recently_changed, false);
+            assert_eq!(cl.slot, 0);
+            assert_eq!(cl.hash, Hash256::zero());
+        }
+        assert_eq!(cry.last_state_recalculation_slot, 0);
+        assert_eq!(cry.last_finalized_slot, 0);
+        assert_eq!(cry.last_justified_slot, 0);
+        assert_eq!(cry.justified_streak, 0);
+        assert_eq!(cry.shard_and_committee_for_slots.len(), (config.cycle_length as usize) * 2);
+        assert_eq!(cry.deposits_penalized_in_period.len(), 0);
+        assert_eq!(cry.validator_set_delta_hash_chain, Hash256::zero());
+        assert_eq!(cry.pre_fork_version, INITIAL_FORK_VERSION);
+        assert_eq!(cry.post_fork_version, INITIAL_FORK_VERSION);
+        assert_eq!(cry.fork_slot_number, 0);
+
+        assert_eq!(act.pending_attestations.len(), 0);
+        assert_eq!(act.pending_specials.len(), 0);
+        assert_eq!(act.recent_block_hashes, vec![Hash256::zero(); config.cycle_length as usize]);
+        assert_eq!(act.randao_mix, Hash256::zero());
+    }
+
+    fn random_registration() -> ValidatorRegistration {
+        let keypair = Keypair::random();
+        ValidatorRegistration {
+            pubkey: keypair.pk.clone(),
+            withdrawal_shard: 0,
+            withdrawal_address: Address::random(),
+            randao_commitment: Hash256::random(),
+            proof_of_possession: create_proof_of_possession(&keypair)
+        }
+    }
+
+    #[test]
+    fn test_genesis_valid_validators() {
+        let mut config = ChainConfig::standard();
+        let validator_count = 5;
+
+        for _ in 0..validator_count {
+            config.initial_validators.push(random_registration());
+        }
+
+        let (_, cry) = genesis_states(&config).unwrap();
+
+        assert_eq!(cry.validators.len(), validator_count);
+    }
+
+    #[test]
+    fn test_genesis_invalid_validators() {
+        let mut config = ChainConfig::standard();
+        let good_validator_count = 5;
+
+        for _ in 0..good_validator_count {
+            config.initial_validators.push(random_registration());
+        }
+
+        let mut bad_v = random_registration();
+        let bad_kp = Keypair::random();
+        bad_v.proof_of_possession =  create_proof_of_possession(&bad_kp);
+        config.initial_validators.push(bad_v);
+
+        let mut bad_v = random_registration();
+        bad_v.withdrawal_shard =  config.shard_count + 1;
+        config.initial_validators.push(bad_v);
+
+        let (_, cry) = genesis_states(&config).unwrap();
+
+        assert!(config.initial_validators.len() != good_validator_count, "test is invalid");
+        assert_eq!(cry.validators.len(), good_validator_count);
+    }
+}

beacon_chain/chain/src/lib.rs

@@ -0,0 +1,139 @@
+extern crate db;
+extern crate types;
+extern crate validator_induction;
+extern crate validator_shuffling;
+
+mod stores;
+mod block_preprocessing;
+mod maps;
+mod genesis;
+
+use db::ClientDB;
+use genesis::genesis_states;
+use maps::{
+    generate_attester_and_proposer_maps,
+    AttesterAndProposerMapError,
+};
+use std::collections::HashMap;
+use std::sync::Arc;
+use stores::BeaconChainStore;
+use types::{
+    ActiveState,
+    AttesterMap,
+    ChainConfig,
+    CrystallizedState,
+    Hash256,
+    ProposerMap,
+};
+
+#[derive(Debug, PartialEq)]
+pub enum BeaconChainError {
+    InvalidGenesis,
+    InsufficientValidators,
+    UnableToGenerateMaps(AttesterAndProposerMapError),
+    DBError(String),
+}
+
+impl From<AttesterAndProposerMapError> for BeaconChainError {
+    fn from(e: AttesterAndProposerMapError) -> BeaconChainError {
+        BeaconChainError::UnableToGenerateMaps(e)
+    }
+}
+
+pub struct BeaconChain<T: ClientDB + Sized> {
+    /// The last slot which has been finalized, this is common to all forks.
+    pub last_finalized_slot: u64,
+    /// The hash of the head of the canonical chain.
+    pub canonical_latest_block_hash: Hash256,
+    /// A vec of hashes of heads of fork (non-canonical) chains.
+    pub fork_latest_block_hashes: Vec<Hash256>,
+    /// A map where the value is an active state the the key is its hash.
+    pub active_states: HashMap<Hash256, ActiveState>,
+    /// A map where the value is crystallized state the the key is its hash.
+    pub crystallized_states: HashMap<Hash256, CrystallizedState>,
+    /// A map of crystallized state to a proposer and attester map.
+    pub attester_proposer_maps: HashMap<Hash256, (Arc<AttesterMap>, Arc<ProposerMap>)>,
+    /// A collection of database stores used by the chain.
+    pub store: BeaconChainStore<T>,
+    /// The chain configuration.
+    pub config: ChainConfig,
+}
+
+impl<T> BeaconChain<T>
+    where T: ClientDB + Sized
+{
+    pub fn new(store: BeaconChainStore<T>, config: ChainConfig)
+        -> Result<Self, BeaconChainError>
+    {
+        if config.initial_validators.is_empty() {
+            return Err(BeaconChainError::InsufficientValidators);
+        }
+
+        let (active_state, crystallized_state) = genesis_states(&config)?;
+
+        let canonical_latest_block_hash = Hash256::zero();
+        let fork_latest_block_hashes = vec![];
+        let mut active_states = HashMap::new();
+        let mut crystallized_states = HashMap::new();
+        let mut attester_proposer_maps = HashMap::new();
+
+        let (attester_map, proposer_map) = generate_attester_and_proposer_maps(
+            &crystallized_state.shard_and_committee_for_slots, 0)?;
+
+        active_states.insert(canonical_latest_block_hash, active_state);
+        crystallized_states.insert(canonical_latest_block_hash, crystallized_state);
+        attester_proposer_maps.insert(
+            canonical_latest_block_hash,
+            (Arc::new(attester_map), Arc::new(proposer_map)));
+
+        Ok(Self{
+            last_finalized_slot: 0,
+            canonical_latest_block_hash,
+            fork_latest_block_hashes,
+            active_states,
+            crystallized_states,
+            attester_proposer_maps,
+            store,
+            config,
+        })
+    }
+}
+
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+    use super::*;
+    use types::ValidatorRegistration;
+    use db::MemoryDB;
+    use db::stores::*;
+
+    #[test]
+    fn test_new_chain() {
+        let mut config = ChainConfig::standard();
+        config.cycle_length = 4;
+        config.shard_count = 4;
+        let db = Arc::new(MemoryDB::open());
+        let store = BeaconChainStore {
+            block: Arc::new(BeaconBlockStore::new(db.clone())),
+            pow_chain: Arc::new(PoWChainStore::new(db.clone())),
+            validator: Arc::new(ValidatorStore::new(db.clone())),
+        };
+
+        for _ in 0..config.cycle_length * 2 {
+            config.initial_validators.push(ValidatorRegistration::random())
+        }
+
+        let chain = BeaconChain::new(store, config.clone()).unwrap();
+        let (act, cry) = genesis_states(&config).unwrap();
+
+        assert_eq!(chain.last_finalized_slot, 0);
+        assert_eq!(chain.canonical_latest_block_hash, Hash256::zero());
+
+        let stored_act = chain.active_states.get(&Hash256::zero()).unwrap();
+        assert_eq!(act, *stored_act);
+
+        let stored_cry = chain.crystallized_states.get(&Hash256::zero()).unwrap();
+        assert_eq!(cry, *stored_cry);
+    }
+}

beacon_chain/chain/src/maps.rs

@@ -0,0 +1,124 @@
+use types::{
+    AttesterMap,
+    ProposerMap,
+    ShardAndCommittee,
+};
+
+#[derive(Debug, PartialEq)]
+pub enum AttesterAndProposerMapError {
+    NoShardAndCommitteeForSlot,
+    NoAvailableProposer,
+}
+
+/// Generate a map of `(slot, shard) |--> committee`.
+///
+/// The attester map is used to optimise the lookup of a committee.
+pub fn generate_attester_and_proposer_maps(
+    shard_and_committee_for_slots: &Vec<Vec<ShardAndCommittee>>,
+    start_slot: u64)
+    -> Result<(AttesterMap, ProposerMap), AttesterAndProposerMapError>
+{
+    let mut attester_map = AttesterMap::new();
+    let mut proposer_map = ProposerMap::new();
+    for (i, slot) in shard_and_committee_for_slots.iter().enumerate() {
+        /*
+         * Store the proposer for the block.
+         */
+        let slot_number = (i as u64).saturating_add(start_slot);
+        let first_committee = &slot.get(0)
+            .ok_or(AttesterAndProposerMapError::NoShardAndCommitteeForSlot)?
+            .committee;
+        let proposer_index = (slot_number as usize).checked_rem(first_committee.len())
+            .ok_or(AttesterAndProposerMapError::NoAvailableProposer)?;
+        proposer_map.insert(slot_number, first_committee[proposer_index]);
+
+        /*
+         * Loop through the shards and extend the attester map.
+         */
+        for shard_and_committee in slot {
+            let committee = shard_and_committee.committee.clone();
+            attester_map.insert((slot_number, shard_and_committee.shard), committee);
+        }
+    };
+    Ok((attester_map, proposer_map))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn sac_generator(shard_count: u16,
+                     slot_count: usize,
+                     sac_per_slot: usize,
+                     committee_size: usize)
+        -> Vec<Vec<ShardAndCommittee>>
+    {
+        let mut shard = 0;
+        let mut validator = 0;
+        let mut cycle = vec![];
+
+        for _ in 0..slot_count {
+            let mut slot: Vec<ShardAndCommittee> = vec![];
+            for _ in 0..sac_per_slot {
+                let mut sac = ShardAndCommittee {
+                    shard: shard % shard_count,
+                    committee: vec![],
+                };
+                for _ in 0..committee_size {
+                    sac.committee.push(validator);
+                    validator += 1;
+                }
+                slot.push(sac);
+                shard += 1;
+            }
+            cycle.push(slot);
+        }
+        cycle
+    }
+
+    #[test]
+    fn test_attester_proposer_maps_empty_slots() {
+        let sac = sac_generator(4, 4, 0, 1);
+        let result = generate_attester_and_proposer_maps(&sac, 0);
+        assert_eq!(result, Err(AttesterAndProposerMapError::NoShardAndCommitteeForSlot));
+    }
+
+    #[test]
+    fn test_attester_proposer_maps_empty_committees() {
+        let sac = sac_generator(4, 4, 1, 0);
+        let result = generate_attester_and_proposer_maps(&sac, 0);
+        assert_eq!(result, Err(AttesterAndProposerMapError::NoAvailableProposer));
+    }
+
+    #[test]
+    fn test_attester_proposer_maps_scenario_a() {
+        let sac = sac_generator(4, 4, 1, 1);
+        let (a, p) = generate_attester_and_proposer_maps(&sac, 0).unwrap();
+
+        assert_eq!(*p.get(&0).unwrap(), 0);
+        assert_eq!(*p.get(&1).unwrap(), 1);
+        assert_eq!(*p.get(&2).unwrap(), 2);
+        assert_eq!(*p.get(&3).unwrap(), 3);
+
+        assert_eq!(*a.get(&(0, 0)).unwrap(), vec![0]);
+        assert_eq!(*a.get(&(1, 1)).unwrap(), vec![1]);
+        assert_eq!(*a.get(&(2, 2)).unwrap(), vec![2]);
+        assert_eq!(*a.get(&(3, 3)).unwrap(), vec![3]);
+    }
+
+    #[test]
+    fn test_attester_proposer_maps_scenario_b() {
+        let sac = sac_generator(4, 4, 1, 4);
+        let (a, p) = generate_attester_and_proposer_maps(&sac, 0).unwrap();
+
+        assert_eq!(*p.get(&0).unwrap(), 0);
+        assert_eq!(*p.get(&1).unwrap(), 5);
+        assert_eq!(*p.get(&2).unwrap(), 10);
+        assert_eq!(*p.get(&3).unwrap(), 15);
+
+        assert_eq!(*a.get(&(0, 0)).unwrap(), vec![0, 1, 2, 3]);
+        assert_eq!(*a.get(&(1, 1)).unwrap(), vec![4, 5, 6, 7]);
+        assert_eq!(*a.get(&(2, 2)).unwrap(), vec![8, 9, 10, 11]);
+        assert_eq!(*a.get(&(3, 3)).unwrap(), vec![12, 13, 14, 15]);
+    }
+}

beacon_chain/chain/src/stores.rs

@@ -0,0 +1,15 @@
+use db::{
+    ClientDB,
+};
+use db::stores::{
+    BeaconBlockStore,
+    PoWChainStore,
+    ValidatorStore,
+};
+use std::sync::Arc;
+
+pub struct BeaconChainStore<T: ClientDB + Sized> {
+    pub block: Arc<BeaconBlockStore<T>>,
+    pub pow_chain: Arc<PoWChainStore<T>>,
+    pub validator: Arc<ValidatorStore<T>>,
+}

beacon_chain/transition/Cargo.toml

@@ -1,9 +0,0 @@
-[package]
-name = "transition"
-version = "0.1.0"
-authors = ["Age Manning <Age@AgeManning.com>"]
-
-[dependencies]
-honey-badger-split = { path = "../utils/honey-badger-split" }
-types = { path = "../types" }
-shuffling = { path = "../utils/shuffling" }

beacon_chain/transition/src/delegation/mod.rs

@@ -1,6 +0,0 @@
-use super::honey_badger_split;
-use super::types;
-use super::TransitionError;
-use super::shuffling::shuffle;
-
-pub mod validator;

beacon_chain/transition/src/lib.rs

@@ -1,10 +0,0 @@
-extern crate honey_badger_split;
-extern crate types;
-extern crate shuffling;
-
-pub mod delegation;
-
-#[derive(Debug)]
-pub enum TransitionError {
-	InvalidInput(String),
-}

beacon_chain/types/src/active_state.rs

@@ -1,30 +1,13 @@
 use super::Hash256;
-use super::attestation_record::AttestationRecord;
+use super::{
+    AttestationRecord,
+    SpecialRecord,
+};
 
+#[derive(Debug, PartialEq)]
 pub struct ActiveState {
     pub pending_attestations: Vec<AttestationRecord>,
+    pub pending_specials: Vec<SpecialRecord>,
     pub recent_block_hashes: Vec<Hash256>,
-}
-
-impl ActiveState {
-    /// Returns a new instance where all fields are empty vectors.
-    pub fn zero() -> Self {
-        Self {
-            pending_attestations: vec![],
-            recent_block_hashes: vec![],
-        }
-    }
-}
-
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_act_state_zero() {
-        let a = ActiveState::zero();
-        assert_eq!(a.pending_attestations.len(), 0);
-        assert_eq!(a.recent_block_hashes.len(), 0);
-    }
+    pub randao_mix: Hash256,
 }

beacon_chain/types/src/chain_config.rs

@@ -1,22 +1,33 @@
+use super::ValidatorRegistration;
+
+#[derive(Debug, Clone, PartialEq)]
 pub struct ChainConfig {
     pub cycle_length: u8,
+    pub deposit_size_gwei: u64,
     pub shard_count: u16,
     pub min_committee_size: u64,
+    pub max_validator_churn_quotient: u64,
     pub genesis_time: u64,
+    pub slot_duration_millis: u64,
+    pub initial_validators: Vec<ValidatorRegistration>,
 }
 
 /*
  * Presently this is just some arbitrary time in Sept 2018.
  */
-const GENESIS_TIME: u64 = 1_537_488_655;
+const TEST_GENESIS_TIME: u64 = 1_537_488_655;
 
 impl ChainConfig {
     pub fn standard() -> Self {
         Self {
             cycle_length: 64,
+            deposit_size_gwei: 32 * (10^9),
             shard_count: 1024,
             min_committee_size: 128,
-            genesis_time: GENESIS_TIME,   // arbitrary
+            max_validator_churn_quotient: 32,
+            genesis_time: TEST_GENESIS_TIME,
+            slot_duration_millis: 16 * 1000,
+            initial_validators: vec![],
         }
     }
 
@@ -25,7 +36,7 @@ impl ChainConfig {
 
 	    // shard_count / cycle_length > 0 otherwise validator delegation
 	    // will fail.
-	    if self.shard_count / self.cycle_length as u16 == 0  {
+	    if self.shard_count / u16::from(self.cycle_length) == 0  {
 		    return false;
 	    }
 
@@ -38,9 +49,13 @@ impl ChainConfig {
     pub fn super_fast_tests() -> Self {
         Self {
             cycle_length: 2,
+            deposit_size_gwei: 32 * (10^9),
             shard_count: 2,
             min_committee_size: 2,
-            genesis_time: GENESIS_TIME,   // arbitrary
+            max_validator_churn_quotient: 32,
+            genesis_time: TEST_GENESIS_TIME,   // arbitrary
+            slot_duration_millis: 16 * 1000,
+            initial_validators: vec![],
         }
     }
 }

beacon_chain/types/src/crosslink_record.rs

@@ -1,8 +1,9 @@
 use super::Hash256;
 
-#[derive(Clone)]
+#[derive(Clone, Debug, PartialEq)]
 pub struct CrosslinkRecord {
-    pub dynasty: u64,
+    pub recently_changed: bool,
+    pub slot: u64,
     pub hash: Hash256,
 }
 
@@ -10,7 +11,8 @@ impl CrosslinkRecord {
     /// Generates a new instance where `dynasty` and `hash` are both zero.
     pub fn zero() -> Self {
         Self {
-            dynasty: 0,
+            recently_changed: false,
+            slot: 0,
             hash: Hash256::zero(),
         }
     }
@@ -23,7 +25,8 @@ mod tests {
     #[test]
     fn test_crosslink_record_zero() {
         let c = CrosslinkRecord::zero();
-        assert_eq!(c.dynasty, 0);
+        assert_eq!(c.recently_changed, false);
+        assert_eq!(c.slot, 0);
         assert!(c.hash.is_zero());
     }
 }

beacon_chain/types/src/crystallized_state.rs

@@ -1,65 +1,22 @@
 use super::validator_record::ValidatorRecord;
 use super::crosslink_record::CrosslinkRecord;
 use super::shard_and_committee::ShardAndCommittee;
-use super::ethereum_types::U256;
 use super::Hash256;
 
 
+#[derive(Debug, PartialEq)]
 pub struct CrystallizedState {
+    pub validator_set_change_slot: u64,
     pub validators: Vec<ValidatorRecord>,
-    pub epoch_number: u64,
-    pub indicies_for_heights: Vec<ShardAndCommittee>,
-    pub last_justified_slot: u64,
-    pub justified_streak: u16,
+    pub crosslinks: Vec<CrosslinkRecord>,
+    pub last_state_recalculation_slot: u64,
     pub last_finalized_slot: u64,
-    pub current_dynasty: u64,
-    pub crosslinking_shard_start: u16,
-    pub crosslink_records: Vec<CrosslinkRecord>,
-    pub total_deposits: U256,
-    pub dynasty_seed: Hash256,
-    pub dynasty_seed_last_reset: u64,
-}
-
-impl CrystallizedState {
-    /// Returns a new instance where all fields are either zero or an
-    /// empty vector.
-    pub fn zero() -> Self {
-        Self {
-            validators: vec![],
-            epoch_number: 0,
-            indicies_for_heights: vec![],
-            last_justified_slot: 0,
-            justified_streak: 0,
-            last_finalized_slot: 0,
-            current_dynasty: 0,
-            crosslinking_shard_start: 0,
-            crosslink_records: vec![],
-            total_deposits: U256::zero(),
-            dynasty_seed: Hash256::zero(),
-            dynasty_seed_last_reset: 0,
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_cry_state_zero() {
-        let c = CrystallizedState::zero();
-        assert_eq!(c.validators.len(), 0);
-        assert_eq!(c.epoch_number, 0);
-        assert_eq!(c.indicies_for_heights.len(), 0);
-        assert_eq!(c.last_justified_slot, 0);
-        assert_eq!(c.justified_streak, 0);
-        assert_eq!(c.last_finalized_slot, 0);
-        assert_eq!(c.current_dynasty, 0);
-        assert_eq!(c.crosslinking_shard_start, 0);
-        assert_eq!(c.crosslink_records.len(), 0);
-        assert!(c.total_deposits.is_zero());
-        assert!(c.dynasty_seed.is_zero());
-        assert_eq!(c.dynasty_seed_last_reset, 0);
-    }
-
+    pub last_justified_slot: u64,
+    pub justified_streak: u64,
+    pub shard_and_committee_for_slots: Vec<Vec<ShardAndCommittee>>,
+    pub deposits_penalized_in_period: Vec<u32>,
+    pub validator_set_delta_hash_chain: Hash256,
+    pub pre_fork_version: u32,
+    pub post_fork_version: u32,
+    pub fork_slot_number: u32,
 }

beacon_chain/types/src/shard_and_committee.rs

@@ -1,6 +1,6 @@
-#[derive(Clone,Debug)]
+#[derive(Clone, Debug, PartialEq)]
 pub struct ShardAndCommittee {
-    pub shard_id: u16,
+    pub shard: u16,
     pub committee: Vec<usize>
 }
 
@@ -9,7 +9,7 @@ impl ShardAndCommittee {
     /// committee is an empty vector.
     pub fn zero() -> Self {
         Self {
-            shard_id: 0,
+            shard: 0,
             committee: vec![],
         }
     }
@@ -22,7 +22,7 @@ mod tests {
     #[test]
     fn test_shard_and_committee_zero() {
         let s = ShardAndCommittee::zero();
-        assert_eq!(s.shard_id, 0);
+        assert_eq!(s.shard, 0);
         assert_eq!(s.committee.len(), 0);
     }
 }

beacon_chain/types/src/special_record.rs

@@ -49,7 +49,7 @@ impl SpecialRecord {
     /// Match `self.kind` to a `SpecialRecordKind`.
     ///
     /// Returns `None` if `self.kind` is an unknown value.
-    fn resolve_kind(&self) -> Option<SpecialRecordKind> {
+    pub fn resolve_kind(&self) -> Option<SpecialRecordKind> {
         match self.kind {
             x if x == SpecialRecordKind::Logout as u8
                 => Some(SpecialRecordKind::Logout),

beacon_chain/types/src/validator_registration.rs

@@ -1,4 +1,5 @@
 use bls::{
+    create_proof_of_possession,
     Keypair,
     PublicKey,
     Signature,
@@ -18,3 +19,17 @@ pub struct ValidatorRegistration {
     pub randao_commitment: Hash256,
     pub proof_of_possession: Signature,
 }
+
+impl ValidatorRegistration {
+    pub fn random() -> Self {
+        let keypair = Keypair::random();
+
+        Self {
+            pubkey: keypair.pk.clone(),
+            withdrawal_shard: 0,
+            withdrawal_address: Address::random(),
+            randao_commitment: Hash256::random(),
+            proof_of_possession: create_proof_of_possession(&keypair),
+        }
+    }
+}

beacon_chain/utils/active-validators/Cargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "active-validators"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+
+[dependencies]
+types = { path = "../../types" }

beacon_chain/utils/active-validators/src/lib.rs

@@ -0,0 +1,68 @@
+extern crate types;
+
+use types::{
+    ValidatorRecord,
+    ValidatorStatus,
+};
+
+pub fn validator_is_active(v: &ValidatorRecord) -> bool {
+    v.status == ValidatorStatus::Active as u8
+}
+
+/// Returns the indicies of each active validator in a given vec of validators.
+pub fn active_validator_indices(validators: &[ValidatorRecord])
+    -> Vec<usize>
+{
+    validators.iter()
+        .enumerate()
+        .filter_map(|(i, validator)| {
+            if validator_is_active(&validator) {
+                Some(i)
+            } else {
+                None
+            }
+        })
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_active_validator() {
+        let mut validators = vec![];
+
+        let (mut v, _) = ValidatorRecord::zero_with_thread_rand_keypair();
+        v.status = ValidatorStatus::Active as u8;
+        assert!(validator_is_active(&v));
+        validators.push(v);
+
+        let (mut v, _) = ValidatorRecord::zero_with_thread_rand_keypair();
+        v.status = ValidatorStatus::PendingActivation as u8;
+        assert!(!validator_is_active(&v));
+        validators.push(v);
+
+        let (mut v, _) = ValidatorRecord::zero_with_thread_rand_keypair();
+        v.status = ValidatorStatus::PendingExit as u8;
+        assert!(!validator_is_active(&v));
+        validators.push(v);
+
+        let (mut v, _) = ValidatorRecord::zero_with_thread_rand_keypair();
+        v.status = ValidatorStatus::PendingWithdraw as u8;
+        assert!(!validator_is_active(&v));
+        validators.push(v);
+
+        let (mut v, _) = ValidatorRecord::zero_with_thread_rand_keypair();
+        v.status = ValidatorStatus::Withdrawn as u8;
+        assert!(!validator_is_active(&v));
+        validators.push(v);
+
+        let (mut v, _) = ValidatorRecord::zero_with_thread_rand_keypair();
+        v.status = ValidatorStatus::Penalized as u8;
+        assert!(!validator_is_active(&v));
+        validators.push(v);
+
+        assert_eq!(active_validator_indices(&validators), vec![0]);
+    }
+}

beacon_chain/utils/slot-clock/Cargo.toml

@@ -0,0 +1,6 @@
+[package]
+name = "slot-clock"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+
+[dependencies]

beacon_chain/utils/slot-clock/src/lib.rs

@@ -0,0 +1,71 @@
+use std::time::{
+    Duration,
+    SystemTime,
+    SystemTimeError,
+};
+
+pub fn slot_now(genesis_seconds: u64, slot_duration_seconds: u64)
+    -> Result<Option<u64>, SystemTimeError>
+{
+    let sys_time = SystemTime::now();
+    let duration_since_epoch = sys_time.duration_since(SystemTime::UNIX_EPOCH)?;
+    let duration_since_genesis = duration_since_epoch
+        .checked_sub(Duration::from_secs(genesis_seconds));
+    match duration_since_genesis {
+        None => Ok(None),
+        Some(d) => Ok(slot_from_duration(slot_duration_seconds, d))
+    }
+}
+
+fn slot_from_duration(slot_duration_seconds: u64, duration: Duration)
+    -> Option<u64>
+{
+    duration.as_secs().checked_div(slot_duration_seconds)
+}
+
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /*
+     * Note: these tests are using actual system times and could fail if they are executed on a
+     * very slow machine.
+     */
+    #[test]
+    fn test_slot_now() {
+        let s_time = 100;
+
+        let now = SystemTime::now();
+        let since_epoch = now.duration_since(SystemTime::UNIX_EPOCH).unwrap();
+
+        let genesis = since_epoch.as_secs() - s_time * 89;
+        assert_eq!(slot_now(genesis, s_time).unwrap(), Some(89));
+
+        let genesis = since_epoch.as_secs();
+        assert_eq!(slot_now(genesis, s_time).unwrap(), Some(0));
+
+        let genesis = since_epoch.as_secs() - s_time * 42 - 5;
+        assert_eq!(slot_now(genesis, s_time).unwrap(), Some(42));
+    }
+
+    #[test]
+    fn test_slot_from_duration() {
+        let s_time = 100;
+
+        assert_eq!(slot_from_duration(s_time, Duration::from_secs(0)), Some(0));
+        assert_eq!(slot_from_duration(s_time, Duration::from_secs(10)), Some(0));
+        assert_eq!(slot_from_duration(s_time, Duration::from_secs(100)), Some(1));
+        assert_eq!(slot_from_duration(s_time, Duration::from_secs(101)), Some(1));
+        assert_eq!(slot_from_duration(s_time, Duration::from_secs(1000)), Some(10));
+    }
+
+    #[test]
+    fn test_slot_from_duration_slot_time_zero() {
+        let s_time = 0;
+
+        assert_eq!(slot_from_duration(s_time, Duration::from_secs(0)), None);
+        assert_eq!(slot_from_duration(s_time, Duration::from_secs(10)), None);
+        assert_eq!(slot_from_duration(s_time, Duration::from_secs(1000)), None);
+    }
+}

beacon_chain/utils/ssz_helpers/src/ssz_beacon_block.rs

@@ -127,6 +127,7 @@ impl<'a> SszBeaconBlock<'a> {
     }
 
     pub fn len(&self) -> usize { self.ssz.len() }
+    pub fn is_empty(&self) -> bool { self.ssz.is_empty() }
 
     /// Return the canonical hash for this block.
     pub fn block_hash(&self) -> Vec<u8> {

beacon_chain/utils/vec_shuffle/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "shuffling"
+name = "vec_shuffle"
 version = "0.1.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 

beacon_chain/validation/src/attestation_validation.rs

@@ -63,7 +63,7 @@ pub struct AttestationValidationContext<T>
     /// The last justified slot as per the client's view of the canonical chain.
     pub last_justified_slot: u64,
     /// A vec of the hashes of the blocks preceeding the present slot.
-    pub parent_hashes: Arc<Vec<Hash256>>,
+    pub recent_block_hashes: Arc<Vec<Hash256>>,
     /// The store containing block information.
     pub block_store: Arc<BeaconBlockStore<T>>,
     /// The store containing validator information.
@@ -155,7 +155,7 @@ impl<T> AttestationValidationContext<T>
             self.cycle_length,
             self.block_slot,
             a.slot,
-            &self.parent_hashes,
+            &self.recent_block_hashes,
             &a.oblique_parent_hashes)?;
 
         /*

beacon_chain/validation/src/block_validation.rs

@@ -80,7 +80,7 @@ pub struct BeaconBlockValidationContext<T>
     /// The last finalized slot as per the client's view of the canonical chain.
     pub last_finalized_slot: u64,
     /// A vec of the hashes of the blocks preceeding the present slot.
-    pub parent_hashes: Arc<Vec<Hash256>>,
+    pub recent_block_hashes: Arc<Vec<Hash256>>,
     /// A map of slots to a block proposer validation index.
     pub proposer_map: Arc<ProposerMap>,
     /// A map of (slot, shard_id) to the attestation set of validation indices.
@@ -109,7 +109,7 @@ impl<T> BeaconBlockValidationContext<T>
     /// Note: this function does not implement randao_reveal checking as it is not in the
     /// specification.
     #[allow(dead_code)]
-    pub fn validate_ssz_block(&self, b: &SszBeaconBlock)
+    pub fn validate_ssz_block(&self, block_hash: &Hash256,  b: &SszBeaconBlock)
         -> Result<(BeaconBlockStatus, Option<BeaconBlock>), SszBeaconBlockValidationError>
         where T: ClientDB + Sized
     {
@@ -118,7 +118,6 @@ impl<T> BeaconBlockValidationContext<T>
          * If this block is already known, return immediately and indicate the the block is
          * known. Don't attempt to deserialize the block.
          */
-        let block_hash = &b.block_hash();
         if self.block_store.block_exists(&block_hash)? {
             return Ok((BeaconBlockStatus::KnownBlock, None));
         }
@@ -225,7 +224,7 @@ impl<T> BeaconBlockValidationContext<T>
             parent_block_slot,
             cycle_length: self.cycle_length,
             last_justified_slot: self.last_justified_slot,
-            parent_hashes: self.parent_hashes.clone(),
+            recent_block_hashes: self.recent_block_hashes.clone(),
             block_store: self.block_store.clone(),
             validator_store: self.validator_store.clone(),
             attester_map: self.attester_map.clone(),

beacon_chain/validation/tests/attestation_validation/helpers.rs

@@ -208,7 +208,7 @@ pub fn setup_attestation_validation_test(shard_id: u16, attester_count: usize)
         parent_block_slot,
         cycle_length,
         last_justified_slot,
-        parent_hashes: parent_hashes.clone(),
+        recent_block_hashes: parent_hashes.clone(),
         block_store: stores.block.clone(),
         validator_store: stores.validator.clone(),
         attester_map: Arc::new(attester_map),

beacon_chain/validation/tests/block_validation/helpers.rs

@@ -225,14 +225,15 @@ pub fn run_block_validation_scenario<F>(
         last_justified_slot: params.validation_context_justified_slot,
         last_justified_block_hash: params.validation_context_justified_block_hash,
         last_finalized_slot: params.validation_context_finalized_slot,
-        parent_hashes: Arc::new(parent_hashes),
+        recent_block_hashes: Arc::new(parent_hashes),
         proposer_map: Arc::new(proposer_map),
         attester_map: Arc::new(attester_map),
         block_store: stores.block.clone(),
         validator_store: stores.validator.clone(),
         pow_store: stores.pow_chain.clone()
     };
-    let validation_status = context.validate_ssz_block(&ssz_block);
+    let block_hash = Hash256::from(&ssz_block.block_hash()[..]);
+    let validation_status = context.validate_ssz_block(&block_hash, &ssz_block);
     /*
      * If validation returned a block, make sure it's the same block we supplied to it.
      *

beacon_chain/validator_change/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "validator_change"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+
+[dependencies]
+active-validators = { path = "../utils/active-validators" }
+bytes = "0.4.10"
+hashing = { path = "../utils/hashing" }
+types = { path = "../types" }

beacon_chain/validator_change/src/lib.rs

@@ -0,0 +1,148 @@
+extern crate active_validators;
+extern crate bytes;
+extern crate hashing;
+extern crate types;
+
+use active_validators::validator_is_active;
+use bytes::{
+    BytesMut,
+    BufMut,
+};
+use hashing::canonical_hash;
+use std::cmp::max;
+use types::{
+    Hash256,
+    ValidatorRecord,
+    ValidatorStatus,
+};
+
+pub enum UpdateValidatorSetError {
+    ArithmeticOverflow,
+}
+
+const VALIDATOR_FLAG_ENTRY: u8 = 0;
+const VALIDATOR_FLAG_EXIT:  u8 = 1;
+
+pub fn update_validator_set(
+    validators: &mut Vec<ValidatorRecord>,
+    hash_chain: Hash256,
+    present_slot: u64,
+    deposit_size_gwei: u64,
+    max_validator_churn_quotient: u64)
+    -> Result<(), UpdateValidatorSetError>
+{
+    /*
+     * Total balance of all active validators.
+     *
+     * Return an error if an overflow occurs.
+     */
+    let total_balance = {
+        let mut bal: u64 = 0;
+        for v in validators.iter() {
+            if validator_is_active(&v) {
+                bal = bal.checked_add(v.balance)
+                    .ok_or(UpdateValidatorSetError::ArithmeticOverflow)?;
+            }
+        }
+        bal
+    };
+
+    /*
+     * Note: this is not the maximum allowable change, it can actually be higher.
+     */
+    let max_allowable_change = {
+        let double_deposit_size = deposit_size_gwei.checked_mul(2)
+            .ok_or(UpdateValidatorSetError::ArithmeticOverflow)?;
+        max(double_deposit_size, total_balance / max_validator_churn_quotient)
+    };
+
+    let mut hasher = ValidatorChangeHashChain {
+        bytes: hash_chain.to_vec(),
+    };
+    let mut total_changed: u64 = 0;
+    for (i, v) in validators.iter_mut().enumerate() {
+        match v.status {
+            /*
+             * Validator is pending activiation.
+             */
+            x if x == ValidatorStatus::PendingActivation as u8 => {
+                let new_total_changed = total_changed.checked_add(deposit_size_gwei)
+                    .ok_or(UpdateValidatorSetError::ArithmeticOverflow)?;
+                /*
+                 * If entering this validator would not exceed the max balance delta,
+                 * activate the validator.
+                 */
+                if new_total_changed <= max_allowable_change {
+                    v.status = ValidatorStatus::Active as u8;
+                    hasher.extend(i, &v.pubkey.as_bytes(), VALIDATOR_FLAG_ENTRY);
+                    total_changed = new_total_changed;
+                } else {
+                    // Entering the validator would exceed the balance delta.
+                    break;
+                }
+            }
+            /*
+             * Validator is pending exit.
+             */
+            x if x == ValidatorStatus::PendingExit as u8 => {
+                let new_total_changed = total_changed.checked_add(v.balance)
+                    .ok_or(UpdateValidatorSetError::ArithmeticOverflow)?;
+                /*
+                 * If exiting this validator would not exceed the max balance delta,
+                 * exit the validator
+                 */
+                if new_total_changed <= max_allowable_change {
+                    v.status = ValidatorStatus::PendingWithdraw as u8;
+                    v.exit_slot = present_slot;
+                    hasher.extend(i, &v.pubkey.as_bytes(), VALIDATOR_FLAG_EXIT);
+                    total_changed = new_total_changed;
+                } else {
+                    // Exiting the validator would exceed the balance delta.
+                    break;
+                }
+            }
+            _ => ()
+        };
+        if total_changed >= max_allowable_change {
+            break;
+        }
+    }
+    Ok(())
+}
+
+pub struct ValidatorChangeHashChain {
+    bytes: Vec<u8>,
+}
+
+impl ValidatorChangeHashChain {
+    pub fn extend(&mut self, index: usize, pubkey: &Vec<u8>, flag: u8)
+    {
+        let mut message = self.bytes.clone();
+        message.append(&mut serialize_validator_change_record(index, pubkey, flag));
+        self.bytes = canonical_hash(&message);
+    }
+}
+
+fn serialize_validator_change_record(index: usize, pubkey: &Vec<u8>, flag: u8)
+    -> Vec<u8>
+{
+    let mut buf = BytesMut::with_capacity(68);
+    buf.put_u8(flag);
+    let index_bytes = {
+        let mut buf = BytesMut::with_capacity(8);
+        buf.put_u64_be(index as u64);
+        buf.take()[8 - 3..8].to_vec()
+    };
+    buf.put(index_bytes);
+    buf.put(pubkey);
+    buf.take().to_vec()
+}
+
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn it_works() {
+        assert_eq!(2 + 2, 4);
+    }
+}

beacon_chain/validator_induction/src/inductor.rs

@@ -40,15 +40,15 @@ impl ValidatorInductor {
     ///
     /// Returns an error if the registration is invalid, otherwise returns the index of the
     /// validator in `CrystallizedState.validators`.
-    pub fn induct(&mut self, rego: &ValidatorRegistration)
+    pub fn induct(&mut self, rego: &ValidatorRegistration, status: ValidatorStatus)
         -> Result<usize, ValidatorInductionError>
     {
-        let v = self.process_registration(rego)?;
+        let v = self.process_registration(rego, status)?;
         Ok(self.add_validator(v))
     }
 
     /// Verify a `ValidatorRegistration` and return a `ValidatorRecord` if valid.
-    fn process_registration(&self, r: &ValidatorRegistration)
+    fn process_registration(&self, r: &ValidatorRegistration, status: ValidatorStatus)
         -> Result<ValidatorRecord, ValidatorInductionError>
     {
         /*
@@ -72,7 +72,7 @@ impl ValidatorInductor {
             randao_commitment: r.randao_commitment,
             randao_last_change: self.current_slot,
             balance: DEPOSIT_GWEI,
-            status: ValidatorStatus::PendingActivation as u8,
+            status: status as u8,
             exit_slot: 0,
         })
     }
@@ -166,7 +166,7 @@ mod tests {
         let r = get_registration();
 
         let mut inductor = ValidatorInductor::new(0, 1024, validators);
-        let result = inductor.induct(&r);
+        let result = inductor.induct(&r, ValidatorStatus::PendingActivation);
         let validators = inductor.to_vec();
 
         assert_eq!(result.unwrap(), 0);
@@ -174,6 +174,22 @@ mod tests {
         assert_eq!(validators.len(), 1);
     }
 
+    #[test]
+    fn test_validator_inductor_status() {
+        let validators = vec![];
+
+        let r = get_registration();
+
+        let mut inductor = ValidatorInductor::new(0, 1024, validators);
+        let _ = inductor.induct(&r, ValidatorStatus::PendingActivation);
+        let _ = inductor.induct(&r, ValidatorStatus::Active);
+        let validators = inductor.to_vec();
+
+        assert!(validators[0].status == ValidatorStatus::PendingActivation as u8);
+        assert!(validators[1].status == ValidatorStatus::Active as u8);
+        assert_eq!(validators.len(), 2);
+    }
+
     #[test]
     fn test_validator_inductor_valid_all_active_validators() {
         let mut validators = vec![];
@@ -186,7 +202,7 @@ mod tests {
         let r = get_registration();
 
         let mut inductor = ValidatorInductor::new(0, 1024, validators);
-        let result = inductor.induct(&r);
+        let result = inductor.induct(&r, ValidatorStatus::PendingActivation);
         let validators = inductor.to_vec();
 
         assert_eq!(result.unwrap(), 5);
@@ -209,7 +225,7 @@ mod tests {
         let r = get_registration();
 
         let mut inductor = ValidatorInductor::new(0, 1024, validators);
-        let result = inductor.induct(&r);
+        let result = inductor.induct(&r, ValidatorStatus::PendingActivation);
         let validators = inductor.to_vec();
 
         assert_eq!(result.unwrap(), 1);
@@ -231,7 +247,7 @@ mod tests {
          */
         let r = get_registration();
         let mut inductor = ValidatorInductor::new(0, 1024, validators);
-        let result = inductor.induct(&r);
+        let result = inductor.induct(&r, ValidatorStatus::PendingActivation);
         let validators = inductor.to_vec();
         assert_eq!(result.unwrap(), 0);
         assert!(registration_equals_record(&r, &validators[0]));
@@ -241,7 +257,7 @@ mod tests {
          */
         let r_two = get_registration();
         let mut inductor = ValidatorInductor::new(0, 1024, validators);
-        let result = inductor.induct(&r_two);
+        let result = inductor.induct(&r_two, ValidatorStatus::PendingActivation);
         let validators = inductor.to_vec();
         assert_eq!(result.unwrap(), 1);
         assert!(registration_equals_record(&r_two, &validators[1]));
@@ -256,7 +272,7 @@ mod tests {
         r.withdrawal_shard = 1025;
 
         let mut inductor = ValidatorInductor::new(0, 1024, validators);
-        let result = inductor.induct(&r);
+        let result = inductor.induct(&r, ValidatorStatus::PendingActivation);
         let validators = inductor.to_vec();
 
         assert_eq!(result, Err(ValidatorInductionError::InvalidShard));
@@ -272,7 +288,7 @@ mod tests {
         r.proof_of_possession = get_proof_of_possession(&kp);
 
         let mut inductor = ValidatorInductor::new(0, 1024, validators);
-        let result = inductor.induct(&r);
+        let result = inductor.induct(&r, ValidatorStatus::PendingActivation);
         let validators = inductor.to_vec();
 
         assert_eq!(result, Err(ValidatorInductionError::InvaidProofOfPossession));

beacon_chain/validator_shuffling/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "validator_shuffling"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+
+[dependencies]
+active-validators = { path = "../utils/active-validators" }
+honey-badger-split = { path = "../utils/honey-badger-split" }
+types = { path = "../types" }
+vec_shuffle = { path = "../utils/vec_shuffle" }

beacon_chain/validator_shuffling/src/lib.rs

@@ -0,0 +1,11 @@
+extern crate active_validators;
+extern crate honey_badger_split;
+extern crate vec_shuffle;
+extern crate types;
+
+mod shuffle;
+
+pub use shuffle::{
+    shard_and_committees_for_cycle,
+    ValidatorAssignmentError,
+};

beacon_chain/validator_shuffling/src/shuffle.rs

@@ -1,50 +1,40 @@
-use super::honey_badger_split::SplitExt;
-use super::types::{
+use std::cmp::min;
+
+use active_validators::active_validator_indices;
+use honey_badger_split::SplitExt;
+use vec_shuffle::{
+    shuffle,
+    ShuffleErr,
+};
+use types::{
     ShardAndCommittee,
     ValidatorRecord,
-    ValidatorStatus,
     ChainConfig,
 };
-use super::TransitionError;
-use super::shuffle;
-use std::cmp::min;
+
 
 type DelegatedCycle = Vec<Vec<ShardAndCommittee>>;
 
-/// Returns the indicies of each active validator in a given vec of validators.
-fn active_validator_indicies(validators: &[ValidatorRecord])
-    -> Vec<usize>
-{
-    validators.iter()
-        .enumerate()
-        .filter_map(|(i, validator)| {
-            match validator.status {
-                x if x == ValidatorStatus::Active as u8 => Some(i),
-                _ => None
+#[derive(Debug, PartialEq)]
+pub enum ValidatorAssignmentError {
+    TooManyValidators,
+    TooFewShards,
 }
-        })
-        .collect()
-}
-
 
 /// Delegates active validators into slots for a given cycle, given a random seed.
 /// Returns a vector or ShardAndComitte vectors representing the shards and committiees for
 /// each slot.
 /// References get_new_shuffling (ethereum 2.1 specification)
-pub fn delegate_validators(
+pub fn shard_and_committees_for_cycle(
     seed: &[u8],
     validators: &[ValidatorRecord],
     crosslinking_shard_start: u16,
     config: &ChainConfig)
-    -> Result<DelegatedCycle, TransitionError>
+    -> Result<DelegatedCycle, ValidatorAssignmentError>
 {
     let shuffled_validator_indices = {
-        let mut validator_indices = active_validator_indicies(validators);
-        match shuffle(seed, validator_indices) {
-            Ok(shuffled) => shuffled,
-            _ => return Err(TransitionError::InvalidInput(
-                    String::from("Shuffle list length exceed.")))
-        }
+        let mut validator_indices = active_validator_indices(validators);
+        shuffle(seed, validator_indices)?
     };
     let shard_indices: Vec<usize> = (0_usize..config.shard_count as usize).into_iter().collect();
     let crosslinking_shard_start = crosslinking_shard_start as usize;
@@ -65,17 +55,14 @@ fn generate_cycle(
     crosslinking_shard_start: usize,
     cycle_length: usize,
     min_committee_size: usize)
-    -> Result<DelegatedCycle, TransitionError>
+    -> Result<DelegatedCycle, ValidatorAssignmentError>
 {
 
     let validator_count = validator_indices.len();
     let shard_count = shard_indices.len();
 
     if shard_count / cycle_length == 0 {
-	    return Err(TransitionError::InvalidInput(String::from("Number of
-					    shards needs to be greater than
-					    cycle length")));
-
+        return Err(ValidatorAssignmentError::TooFewShards)
     }
 
     let (committees_per_slot, slots_per_committee) = {
@@ -99,12 +86,12 @@ fn generate_cycle(
     let cycle = validator_indices.honey_badger_split(cycle_length)
         .enumerate()
         .map(|(i, slot_indices)| {
-            let shard_id_start = crosslinking_shard_start + i * committees_per_slot / slots_per_committee;
+            let shard_start = crosslinking_shard_start + i * committees_per_slot / slots_per_committee;
             slot_indices.honey_badger_split(committees_per_slot)
                 .enumerate()
                 .map(|(j, shard_indices)| {
                     ShardAndCommittee{
-                        shard_id: ((shard_id_start + j) % shard_count) as u16,
+                        shard: ((shard_start + j) % shard_count) as u16,
                         committee: shard_indices.to_vec(),
                     }
                 })
@@ -114,6 +101,14 @@ fn generate_cycle(
     Ok(cycle)
 }
 
+impl From<ShuffleErr> for ValidatorAssignmentError {
+    fn from(e: ShuffleErr) -> ValidatorAssignmentError {
+        match e {
+            ShuffleErr::ExceedsListLength => ValidatorAssignmentError::TooManyValidators,
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -124,7 +119,7 @@ mod tests {
         crosslinking_shard_start: usize,
         cycle_length: usize,
         min_committee_size: usize)
-        -> (Vec<usize>, Vec<usize>, Result<DelegatedCycle, TransitionError>)
+        -> (Vec<usize>, Vec<usize>, Result<DelegatedCycle, ValidatorAssignmentError>)
     {
         let validator_indices: Vec<usize> = (0_usize..*validator_count).into_iter().collect();
         let shard_indices: Vec<usize> = (0_usize..*shard_count).into_iter().collect();
@@ -146,8 +141,8 @@ mod tests {
                 slot.iter()
                     .enumerate()
                     .for_each(|(i, sac)| {
-                        println!("#{:?}\tshard_id={}\tcommittee.len()={}",
-                            &i, &sac.shard_id, &sac.committee.len())
+                        println!("#{:?}\tshard={}\tcommittee.len()={}",
+                            &i, &sac.shard, &sac.committee.len())
                     })
             });
     }
@@ -172,7 +167,7 @@ mod tests {
         let mut flattened = vec![];
         for slot in cycle.iter() {
             for sac in slot.iter() {
-                flattened.push(sac.shard_id as usize);
+                flattened.push(sac.shard as usize);
             }
         }
         flattened.dedup();
@@ -186,7 +181,7 @@ mod tests {
         for slot in cycle.iter() {
             let mut shards: Vec<usize> = vec![];
             for sac in slot.iter() {
-                shards.push(sac.shard_id as usize);
+                shards.push(sac.shard as usize);
             }
             shards_in_slots.push(shards);
         }