Progress further with block validation tests

lighthouse/state/attestation_record/validation/attestation_validation.rs

@@ -1,4 +1,5 @@
 use std::collections::HashSet;
+use std::sync::Arc;
 use super::attestation_record::AttestationRecord;
 use super::AttesterMap;
 use super::attestation_parent_hashes::{
@@ -13,13 +14,11 @@ use super::db::stores::{
     BlockStore,
     ValidatorStore,
 };
-use super::ssz::SszStream;
-use super::utils::hash::canonical_hash;
 use super::utils::types::{
     Hash256,
 };
-use std::sync::Arc;
-use super::signatures::{
+use super::message_generation::generate_signed_message;
+use super::signature_verification::{
     verify_aggregate_signature_for_indices,
     SignatureVerificationError,
 };
@@ -37,7 +36,9 @@ pub enum AttestationValidationError {
     IntWrapping,
     PublicKeyCorrupt,
     NoPublicKeyForValidator,
-    IncorrectBitField,
+    BadBitfieldLength,
+    InvalidBitfield,
+    InvalidBitfieldEndBits,
     NoSignatures,
     NonZeroTrailingBits,
     AggregateSignatureFail,
@@ -115,7 +116,7 @@ pub fn validate_attestation<T>(a: &AttestationRecord,
     if a.attester_bitfield.num_bytes() !=
         bytes_for_bits(attestation_indices.len())
     {
-        return Err(AttestationValidationError::IncorrectBitField);
+        return Err(AttestationValidationError::BadBitfieldLength);
     }
 
     /*
@@ -126,10 +127,10 @@ pub fn validate_attestation<T>(a: &AttestationRecord,
      * refer to the same AttesationRecord.
      */
     let last_byte =
-        a.attester_bitfield.get_byte(a.attester_bitfield.num_bytes())
-        .ok_or(AttestationValidationError::IncorrectBitField)?;
+        a.attester_bitfield.get_byte(a.attester_bitfield.num_bytes() - 1)
+        .ok_or(AttestationValidationError::InvalidBitfield)?;
     if any_of_last_n_bits_are_set(last_byte, a.attester_bitfield.len() % 8) {
-        return Err(AttestationValidationError::IncorrectBitField)
+        return Err(AttestationValidationError::InvalidBitfieldEndBits)
     }
 
     /*
@@ -165,37 +166,6 @@ pub fn validate_attestation<T>(a: &AttestationRecord,
     Ok(voted_hashmap)
 }
 
-/// Generates the message used to validate the signature provided with an AttestationRecord.
-///
-/// Ensures that the signer of the message has a view of the chain that is compatible with ours.
-fn generate_signed_message(slot: u64,
-                           parent_hashes: &[Hash256],
-                           shard_id: u16,
-                           shard_block_hash: &Hash256,
-                           justified_slot: u64)
-    -> Vec<u8>
-{
-    /*
-     * Note: it's a little risky here to use SSZ, because the encoding is not necessarily SSZ
-     * (for example, SSZ might change whilst this doesn't).
-     *
-     * I have suggested switching this to ssz here:
-     * https://github.com/ethereum/eth2.0-specs/issues/5
-     *
-     * If this doesn't happen, it would be safer to not use SSZ at all.
-     */
-    let mut ssz_stream = SszStream::new();
-    ssz_stream.append(&slot);
-    for h in parent_hashes {
-        ssz_stream.append_encoded_raw(&h.to_vec())
-    }
-    ssz_stream.append(&shard_id);
-    ssz_stream.append(shard_block_hash);
-    ssz_stream.append(&justified_slot);
-    let bytes = ssz_stream.drain();
-    canonical_hash(&bytes)
-}
-
 impl From<ParentHashesError> for AttestationValidationError {
     fn from(e: ParentHashesError) -> Self {
         match e {

lighthouse/state/attestation_record/validation/message_generation.rs

@@ -0,0 +1,34 @@
+use super::ssz::SszStream;
+use super::utils::hash::canonical_hash;
+use super::utils::types::Hash256;
+
+/// Generates the message used to validate the signature provided with an AttestationRecord.
+///
+/// Ensures that the signer of the message has a view of the chain that is compatible with ours.
+pub fn generate_signed_message(slot: u64,
+                           parent_hashes: &[Hash256],
+                           shard_id: u16,
+                           shard_block_hash: &Hash256,
+                           justified_slot: u64)
+    -> Vec<u8>
+{
+    /*
+     * Note: it's a little risky here to use SSZ, because the encoding is not necessarily SSZ
+     * (for example, SSZ might change whilst this doesn't).
+     *
+     * I have suggested switching this to ssz here:
+     * https://github.com/ethereum/eth2.0-specs/issues/5
+     *
+     * If this doesn't happen, it would be safer to not use SSZ at all.
+     */
+    let mut ssz_stream = SszStream::new();
+    ssz_stream.append(&slot);
+    for h in parent_hashes {
+        ssz_stream.append_encoded_raw(&h.to_vec())
+    }
+    ssz_stream.append(&shard_id);
+    ssz_stream.append(shard_block_hash);
+    ssz_stream.append(&justified_slot);
+    let bytes = ssz_stream.drain();
+    canonical_hash(&bytes)
+}

lighthouse/state/attestation_record/validation/mod.rs

@@ -7,7 +7,8 @@ use super::common::attestation_parent_hashes;
 use super::utils;
 
 mod attestation_validation;
-mod signatures;
+mod signature_verification;
+mod message_generation;
 
 pub use self::attestation_validation::{
     validate_attestation,

lighthouse/state/block/mod.rs

@@ -1,7 +1,6 @@
 extern crate blake2_rfc;
 
 use super::common;
-use super::Logger;
 use super::db;
 use super::ssz;
 use super::utils;

lighthouse/state/block/validation/mod.rs

@@ -1,8 +1,10 @@
 mod validate_ssz_block;
+#[cfg(test)]
+mod tests;
 
 use super::attestation_record;
+use super::Block;
 use super::SszBlock;
-use super::Logger;
 use super::db;
 use super::ssz;
 use super::utils;
@@ -13,5 +15,6 @@ use super::common::maps::{
 };
 pub use self::validate_ssz_block::{
     validate_ssz_block,
-    SszBlockValidationError
+    SszBlockValidationError,
+    BlockStatus,
 };

lighthouse/state/block/validation/tests.rs

@@ -0,0 +1,185 @@
+extern crate ssz;
+
+use self::ssz::{
+    SszStream,
+};
+use std::sync::Arc;
+use super::{
+    validate_ssz_block,
+    BlockStatus,
+    AttesterMap,
+    ProposerMap,
+};
+use super::db::stores::{
+    BlockStore,
+    PoWChainStore,
+    ValidatorStore,
+};
+use super::db::{
+    MemoryDB,
+};
+use super::utils::types::{
+    Hash256,
+    Bitfield,
+};
+use super::{
+    Block,
+    SszBlock,
+};
+use super::super::attestation_record::AttestationRecord;
+use super::super::super::bls::{
+    Keypair,
+    Signature,
+    AggregateSignature,
+};
+
+struct TestStore {
+    db: Arc<MemoryDB>,
+    block: Arc<BlockStore<MemoryDB>>,
+    pow_chain: Arc<PoWChainStore<MemoryDB>>,
+    validator: Arc<ValidatorStore<MemoryDB>>,
+}
+
+impl TestStore {
+    pub fn new() -> Self {
+        let db = Arc::new(MemoryDB::open());
+        let block = Arc::new(BlockStore::new(db.clone()));
+        let pow_chain = Arc::new(PoWChainStore::new(db.clone()));
+        let validator = Arc::new(ValidatorStore::new(db.clone()));
+        Self {
+            db,
+            block,
+            pow_chain,
+            validator,
+        }
+    }
+}
+
+#[test]
+fn test_block_validation() {
+    let stores = TestStore::new();
+
+    let cycle_length = 2;
+    let shard_count = 2;
+    let present_slot = u64::from(cycle_length) * 10000;
+    let justified_slot = present_slot - u64::from(cycle_length);
+    let justified_block_hash = Hash256::from("justified_hash".as_bytes());
+    let shard_block_hash = Hash256::from("shard_hash".as_bytes());
+    let parent_hashes = (0..(cycle_length * 2))
+        .map(|i| Hash256::from(i as u64))
+        .collect();
+    let pow_chain_ref = Hash256::from("pow_chain".as_bytes());
+    let active_state_root = Hash256::from("active_state".as_bytes());
+    let crystallized_state_root = Hash256::from("cry_state".as_bytes());
+
+    stores.pow_chain.put_block_hash(pow_chain_ref.as_ref()).unwrap();
+    stores.block.put_block(justified_block_hash.as_ref(), &vec![42]).unwrap();
+
+    let validators_per_shard = 2;
+
+    let block_slot = present_slot;
+    let validator_index: usize = 0;
+    let proposer_map = {
+        let mut proposer_map = ProposerMap::new();
+        proposer_map.insert(present_slot, validator_index);
+        proposer_map
+    };
+    let (attester_map, attestations, _keypairs) = {
+        let mut i = 0;
+        let mut attester_map = AttesterMap::new();
+        let mut attestations = vec![];
+        let mut keypairs = vec![];
+        for shard in 0..shard_count {
+            println!("hello");
+            let mut attesters = vec![];
+            let mut attester_bitfield = Bitfield::new();
+            let mut aggregate_sig = AggregateSignature::new();
+            let attestation_slot = block_slot - 1;
+
+            let attestation_message = {
+                let mut stream = SszStream::new();
+                stream.append(&attestation_slot);
+                stream.append_vec(&parent_hashes);
+                stream.append(&shard);
+                stream.append(&shard_block_hash);
+                stream.append(&justified_slot);
+                stream.drain()
+            };
+
+            for _ in 0..validators_per_shard {
+               /*
+                * Add the attester to the attestation indices for this shard.
+                */
+               attesters.push(i);
+               /*
+                * Set the voters bit on the bitfield to true.
+                */
+               attester_bitfield.set_bit(i, true);
+               /*
+                * Generate a random keypair for this validatior and clone it into the
+                * list of keypairs.
+                */
+               let keypair = Keypair::random();
+               keypairs.push(keypair.clone());
+               /*
+                * Store the validators public key in the database.
+                */
+               stores.validator.put_public_key_by_index(i, &keypair.pk).unwrap();
+               /*
+                * Generate a new signature and aggregate it on the rolling signature.
+                */
+               let sig = Signature::new(&attestation_message, &keypair.sk);
+               aggregate_sig.add(&sig);
+               /*
+                * Increment the validator counter to monotonically assign validators.
+                */
+               i += 1;
+            }
+
+            attester_map.insert((attestation_slot, shard), attesters);
+            let attestation = AttestationRecord {
+                slot: attestation_slot,
+                shard_id: shard,
+                oblique_parent_hashes: vec![],
+                shard_block_hash: Hash256::from("shardhash".as_bytes()),
+                attester_bitfield,
+                justified_slot,
+                justified_block_hash,
+                aggregate_sig,
+            };
+            attestations.push(attestation);
+        }
+        (attester_map, attestations, keypairs)
+    };
+
+    let block = Block {
+        parent_hash: Hash256::from("parent".as_bytes()),
+        slot_number: block_slot,
+        randao_reveal: Hash256::from("randao".as_bytes()),
+        attestations,
+        pow_chain_ref,
+        active_state_root,
+        crystallized_state_root,
+    };
+
+    let mut stream = SszStream::new();
+    stream.append(&block);
+    let serialized_block = stream.drain();
+    let ssz_block = SszBlock::from_slice(&serialized_block[..]).unwrap();
+
+    println!("this happened");
+
+    let status = validate_ssz_block(
+        &ssz_block,
+        present_slot,
+        cycle_length,
+        justified_slot,
+        Arc::new(parent_hashes),
+        Arc::new(proposer_map),
+        Arc::new(attester_map),
+        stores.block.clone(),
+        stores.validator.clone(),
+        stores.pow_chain.clone()).unwrap();
+
+    assert_eq!(status, BlockStatus::NewBlock);
+}

lighthouse/state/block/validation/validate_ssz_block.rs

@@ -18,7 +18,6 @@ use super::db::{
     ClientDB,
     DBError,
 };
-use super::Logger;
 use super::db::stores::{
     BlockStore,
     PoWChainStore,
@@ -30,11 +29,13 @@ use super::ssz::{
 };
 use super::utils::types::Hash256;
 
+#[derive(Debug, PartialEq)]
 pub enum BlockStatus {
     NewBlock,
     KnownBlock,
 }
 
+#[derive(Debug, PartialEq)]
 pub enum SszBlockValidationError {
     FutureSlot,
     UnknownPoWChainRef,
@@ -62,7 +63,6 @@ pub enum SszBlockValidationError {
 #[allow(dead_code)]
 pub fn validate_ssz_block<T>(b: &SszBlock,
                              expected_slot: u64,
-                             pow_store: &PoWChainStore<T>,
                              cycle_length: u8,
                              last_justified_slot: u64,
                              parent_hashes: Arc<Vec<Hash256>>,
@@ -70,7 +70,7 @@ pub fn validate_ssz_block<T>(b: &SszBlock,
                              attester_map: Arc<AttesterMap>,
                              block_store: Arc<BlockStore<T>>,
                              validator_store: Arc<ValidatorStore<T>>,
-                             _log: &Logger)
+                             pow_store: Arc<PoWChainStore<T>>)
     -> Result<BlockStatus, SszBlockValidationError>
     where T: ClientDB + Sized
 {

lighthouse/state/chain_config.rs

@@ -5,13 +5,28 @@ pub struct ChainConfig {
     pub genesis_time: u64,
 }
 
+/*
+ * Presently this is just some arbitrary time in Sept 2018.
+ */
+const GENESIS_TIME: u64 = 1537488655;
+
 impl ChainConfig {
     pub fn standard() -> Self {
         Self {
-            cycle_length: 8,
+            cycle_length: 64,
             shard_count: 1024,
             min_committee_size: 128,
-            genesis_time: 1537488655,   // arbitrary
+            genesis_time: GENESIS_TIME,   // arbitrary
+        }
+    }
+
+    #[cfg(test)]
+    pub fn super_fast_tests() -> Self {
+        Self {
+            cycle_length: 2,
+            shard_count: 2,
+            min_committee_size: 2,
+            genesis_time: GENESIS_TIME,   // arbitrary
         }
     }
 }

lighthouse/state/common/maps.rs

@@ -1,7 +1,7 @@
 use std::collections::HashMap;
 
 /// Maps a (slot, shard_id) to attestation_indices.
-pub type AttesterMap = HashMap<(u64, u64), Vec<usize>>;
+pub type AttesterMap = HashMap<(u64, u16), Vec<usize>>;
 
 /// Maps a slot to a block proposer.
 pub type ProposerMap = HashMap<u64, usize>;

lighthouse/utils/macros.rs

@@ -6,5 +6,3 @@ macro_rules! assert_error {
         }
     }
 }
-
-