diff --git a/Cargo.toml b/Cargo.toml index c076461c3..50c7360bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,3 +36,6 @@ ring = { git = "https://github.com/paritytech/ring" } [[bin]] path = "lighthouse/main.rs" name = "lighthouse" + +[lib] +path = "lighthouse/lib.rs" diff --git a/benches/block_validation.rs b/benches/block_validation.rs new file mode 100644 index 000000000..03e9df498 --- /dev/null +++ b/benches/block_validation.rs @@ -0,0 +1,148 @@ +#![feature(test)] + +extern crate test; + +use self::test::Bencher; + +use std::sync::Arc; + +use super::{ + BlockValidationContext, + AttesterMap, + ProposerMap, +}; + +use super::tests::{ + TestStore, + TestParams, + setup_block_validation_scenario, + serialize_block, +}; + +use super::super::{ + Block, + SszBlock, +}; + +fn bench_block_validation_scenario( + b: &mut Bencher, + params: &TestParams, + mutator_func: F) + where F: FnOnce(Block, AttesterMap, ProposerMap, TestStore) + -> (Block, AttesterMap, ProposerMap, TestStore) +{ + let (block, + parent_hashes, + attester_map, + proposer_map, + stores) = setup_block_validation_scenario(¶ms); + + let (block, + attester_map, + proposer_map, + stores) = mutator_func(block, attester_map, proposer_map, stores); + + let ssz_bytes = serialize_block(&block); + let ssz_block = SszBlock::from_slice(&ssz_bytes[..]) + .unwrap(); + + let parent_hashes = Arc::new(parent_hashes); + let proposer_map = Arc::new(proposer_map); + let attester_map = Arc::new(attester_map); + b.iter(|| { + let context = BlockValidationContext { + present_slot: params.validation_context_slot, + cycle_length: params.cycle_length, + last_justified_slot: params.validation_context_justified_slot, + last_finalized_slot: params.validation_context_finalized_slot, + parent_hashes: parent_hashes.clone(), + proposer_map: proposer_map.clone(), + attester_map: attester_map.clone(), + block_store: stores.block.clone(), + validator_store: stores.validator.clone(), + pow_store: stores.pow_chain.clone() + }; + let result = context.validate_ssz_block(&ssz_block); + assert!(result.is_ok()); + }); +} + +#[bench] +#[ignore] +fn bench_block_validation_10m_eth(b: &mut Bencher) { + let total_validators: usize = 10_000_000 / 32; + let cycle_length: u8 = 64; + let shard_count: u16 = 1024; + let shards_per_slot: u16 = 1024 / u16::from(cycle_length); + let validators_per_shard: usize = total_validators / usize::from(shard_count); + let block_slot = u64::from(cycle_length) * 10000; + let attestations_justified_slot = block_slot - u64::from(cycle_length); + let parent_proposer_index = 0; + + let validation_context_slot = block_slot; + let validation_context_justified_slot = attestations_justified_slot; + let validation_context_finalized_slot = 0; + + let params = TestParams { + total_validators, + cycle_length, + shard_count, + shards_per_slot, + validators_per_shard, + parent_proposer_index, + block_slot, + attestations_justified_slot, + validation_context_slot, + validation_context_justified_slot, + validation_context_finalized_slot, + }; + + let no_mutate = |block, attester_map, proposer_map, stores| { + (block, attester_map, proposer_map, stores) + }; + + bench_block_validation_scenario( + b, + ¶ms, + no_mutate); +} + +#[bench] +#[ignore] +fn bench_block_validation_100m_eth(b: &mut Bencher) { + let total_validators: usize = 100_000_000 / 32; + let cycle_length: u8 = 64; + let shard_count: u16 = 1024; + let shards_per_slot: u16 = 1024 / u16::from(cycle_length); + let validators_per_shard: usize = total_validators / usize::from(shard_count); + let block_slot = u64::from(cycle_length) * 10000; + let attestations_justified_slot = block_slot - u64::from(cycle_length); + let parent_proposer_index = 0; + + let validation_context_slot = block_slot; + let validation_context_justified_slot = attestations_justified_slot; + let validation_context_finalized_slot = 0; + + let params = TestParams { + total_validators, + cycle_length, + shard_count, + shards_per_slot, + validators_per_shard, + parent_proposer_index, + block_slot, + attestations_justified_slot, + validation_context_slot, + validation_context_justified_slot, + validation_context_finalized_slot, + }; + + let no_mutate = |block, attester_map, proposer_map, stores| { + (block, attester_map, proposer_map, stores) + }; + + bench_block_validation_scenario( + b, + ¶ms, + no_mutate); +} diff --git a/benches/main.rs b/benches/main.rs new file mode 100644 index 000000000..0faf89f6d --- /dev/null +++ b/benches/main.rs @@ -0,0 +1,5 @@ +#![feature(test)] +extern crate lighthouse; +extern crate tests; + +mod block_validation; diff --git a/lighthouse/bls/mod.rs b/lighthouse/bls/mod.rs index fff21940c..ff9a0919a 100644 --- a/lighthouse/bls/mod.rs +++ b/lighthouse/bls/mod.rs @@ -5,5 +5,6 @@ pub use self::bls_aggregates::AggregatePublicKey; pub use self::bls_aggregates::Signature; pub use self::bls_aggregates::Keypair; pub use self::bls_aggregates::PublicKey; +pub use self::bls_aggregates::SecretKey; pub const BLS_AGG_SIG_BYTE_SIZE: usize = 97; diff --git a/lighthouse/lib.rs b/lighthouse/lib.rs new file mode 100644 index 000000000..9abf4184e --- /dev/null +++ b/lighthouse/lib.rs @@ -0,0 +1,22 @@ +#[macro_use] +extern crate slog; +extern crate slog_term; +extern crate slog_async; +extern crate ssz; +extern crate clap; +extern crate network_libp2p; +extern crate futures; + +#[macro_use] +#[allow(dead_code)] +pub mod utils; +#[allow(dead_code)] +pub mod bls; +#[allow(dead_code)] +pub mod db; +pub mod client; +#[allow(dead_code)] +pub mod state; +#[allow(dead_code)] +mod sync; +mod config; diff --git a/lighthouse/state/attestation_record/validation/attestation_validation.rs b/lighthouse/state/attestation_record/validation/attestation_validation.rs index f8ed3bdc0..077abf2ee 100644 --- a/lighthouse/state/attestation_record/validation/attestation_validation.rs +++ b/lighthouse/state/attestation_record/validation/attestation_validation.rs @@ -61,17 +61,7 @@ impl AttestationValidationContext where T: ClientDB { pub fn validate_attestation(&self, a: &AttestationRecord) - /* - block_slot: u64, - cycle_length: u8, - known_last_justified_slot: u64, - known_parent_hashes: &Arc>, - block_store: &Arc>, - validator_store: &Arc>, - attester_map: &Arc) - */ -> Result>, AttestationValidationError> - where T: ClientDB + Sized { /* * The attesation slot must not be higher than the block that contained it. diff --git a/lighthouse/state/block/mod.rs b/lighthouse/state/block/mod.rs index 9a461568d..eb6365bdc 100644 --- a/lighthouse/state/block/mod.rs +++ b/lighthouse/state/block/mod.rs @@ -8,7 +8,7 @@ use super::utils; mod structs; mod ssz_block; -mod validation; +pub mod validation; pub use self::structs::Block; pub use self::ssz_block::{ diff --git a/lighthouse/state/block/validation/mod.rs b/lighthouse/state/block/validation/mod.rs index d9c67fbcb..a18d3934f 100644 --- a/lighthouse/state/block/validation/mod.rs +++ b/lighthouse/state/block/validation/mod.rs @@ -1,8 +1,4 @@ mod block_validation; -#[cfg(test)] -mod tests; -#[cfg(test)] -mod benches; use super::attestation_record; use super::{ @@ -14,7 +10,7 @@ use super::db; use super::ssz; use super::utils; -use super::common::maps::{ +pub use super::common::maps::{ AttesterMap, ProposerMap, }; diff --git a/tests/attestation_validation/helpers.rs b/tests/attestation_validation/helpers.rs new file mode 100644 index 000000000..c06c23948 --- /dev/null +++ b/tests/attestation_validation/helpers.rs @@ -0,0 +1,177 @@ +use std::sync::Arc; + +use super::db::{ + MemoryDB, +}; +use super::db::stores::{ + BlockStore, + ValidatorStore, +}; +use super::state::attestation_record::{ + AttestationRecord, + AttestationValidationContext, + AttestationValidationError, +}; +use super::state::block::validation::AttesterMap; +use super::bls::{ + AggregateSignature, + Keypair, + SecretKey, + Signature, +}; +use super::ssz::SszStream; +use super::utils::types::{ + Hash256, + Bitfield, +}; +use super::utils::hash::{ + canonical_hash, +}; + + +pub struct TestStore { + pub db: Arc, + pub block: Arc>, + pub validator: Arc>, +} + +impl TestStore { + pub fn new() -> Self { + let db = Arc::new(MemoryDB::open()); + let block = Arc::new(BlockStore::new(db.clone())); + let validator = Arc::new(ValidatorStore::new(db.clone())); + Self { + db, + block, + validator, + } + } +} + +fn generate_message_hash(slot: u64, + parent_hashes: &[Hash256], + shard_id: u16, + shard_block_hash: &Hash256, + justified_slot: u64) + -> Vec +{ + let mut stream = SszStream::new(); + stream.append(&slot); + stream.append_vec(&parent_hashes.to_vec()); + stream.append(&shard_id); + stream.append(shard_block_hash); + stream.append(&justified_slot); + let bytes = stream.drain(); + canonical_hash(&bytes) +} + +pub fn generate_attestation(shard_id: u16, + shard_block_hash: &Hash256, + block_slot: u64, + attestation_slot: u64, + justified_slot: u64, + justified_block_hash: &Hash256, + cycle_length: u8, + parent_hashes: &[Hash256], + signing_keys: &[Option]) + -> AttestationRecord +{ + let mut attester_bitfield = Bitfield::new(); + let mut aggregate_sig = AggregateSignature::new(); + + let parent_hashes_slice = { + let distance: usize = (block_slot - attestation_slot) as usize; + let last: usize = parent_hashes.len() - distance; + let first: usize = last - usize::from(cycle_length); + &parent_hashes[first..last] + }; + + /* + * Generate the message that will be signed across for this attr record. + */ + let attestation_message = generate_message_hash( + attestation_slot, + parent_hashes_slice, + shard_id, + shard_block_hash, + justified_slot); + + for (i, secret_key) in signing_keys.iter().enumerate() { + /* + * If the signing key is Some, set the bitfield bit to true + * and sign the aggregate sig. + */ + if let Some(sk) = secret_key { + attester_bitfield.set_bit(i, true); + let sig = Signature::new(&attestation_message, sk); + aggregate_sig.add(&sig); + } + } + + AttestationRecord { + slot: attestation_slot, + shard_id, + oblique_parent_hashes: vec![], + shard_block_hash: shard_block_hash.clone(), + attester_bitfield, + justified_slot, + justified_block_hash: justified_block_hash.clone(), + aggregate_sig, + } +} + +/* +fn get_valid_attestation_and_context(shard_id: u16, + shard_block_hash: Hash256, + attester_count: usize, + signing_attesters: &[usize]) + -> (AttestationRecord, AttestationValidationContext) +{ + let stores = TestStore::new(); + + let block_slot = 10000; + let cycle_length: u8 = 64; + let last_justified_slot = block_slot - u64::from(cycle_length); + let parent_hashes: Vec = (0..(cycle_length * 2)) + .map(|i| Hash256::from(i as u64)) + .collect(); + let parent_hashes = Arc::new(parent_hashes); + let attester_map = Arc::new(AttesterMap::new()); + let justified_block_hash = Hash256::from("justified_block".as_bytes()); + + stores.block.put_serialized_block(&justified_block_hash.as_ref(), &[42]).unwrap(); + + let aggregate_sig = AggregateSignature::new(); + let attester_bitfield = Bitfield::new(); + + let mut attestation_indices = vec![]; + for attester_index in 0..attester_count { + let kp = Keypair::random(); + let validator_index = attester_count - attester_index; + attestation_indices.push(validator_index); + stores.validator.put_public_key_by_index(validator_index, &kp.pk); + } + + let context: AttestationValidationContext = AttestationValidationContext { + block_slot, + cycle_length, + last_justified_slot, + parent_hashes, + block_store: stores.block.clone(), + validator_store: stores.validator.clone(), + attester_map, + }; + + let attestation = AttestationRecord { + slot: block_slot - 1, + shard_id, + oblique_parent_hashes: vec![], + shard_block_hash, + attester_bitfield, + justified_slot: last_justified_slot, + justified_block_hash, + aggregate_sig, + }; + (attestation, context) +} +*/ diff --git a/tests/attestation_validation/mod.rs b/tests/attestation_validation/mod.rs new file mode 100644 index 000000000..0555fbe3a --- /dev/null +++ b/tests/attestation_validation/mod.rs @@ -0,0 +1,8 @@ +pub mod helpers; +mod tests; + +use super::bls; +use super::db; +use super::state; +use super::ssz; +use super::utils; diff --git a/tests/attestation_validation/tests.rs b/tests/attestation_validation/tests.rs new file mode 100644 index 000000000..a57192d8d --- /dev/null +++ b/tests/attestation_validation/tests.rs @@ -0,0 +1,31 @@ +use std::sync::Arc; + +use super::helpers::{ + TestStore, +}; +use super::state::attestation_record::{ + AttestationRecord, + AttestationValidationContext, + AttestationValidationError, +}; +use super::state::block::validation::AttesterMap; +use super::bls::{ + AggregateSignature, + Keypair, +}; +use super::db::{ + MemoryDB, +}; +use super::db::stores::{ + BlockStore, + ValidatorStore, +}; +use super::utils::types::{ + Hash256, + Bitfield, +}; + +#[test] +fn test_attestation_validation_valid() { + // TODO +} diff --git a/lighthouse/state/block/validation/tests/helpers.rs b/tests/block_validation/helpers.rs similarity index 71% rename from lighthouse/state/block/validation/tests/helpers.rs rename to tests/block_validation/helpers.rs index 3b2dda9e3..62fffc071 100644 --- a/lighthouse/state/block/validation/tests/helpers.rs +++ b/tests/block_validation/helpers.rs @@ -1,26 +1,52 @@ use std::sync::Arc; -use super::{ + +use super::generate_attestation; +use super::bls::{ + Keypair, +}; +use super::db::{ + MemoryDB, +}; +use super::db::stores::{ + BlockStore, + PoWChainStore, + ValidatorStore, +}; +use super::state::attestation_record::{ + AttestationRecord, +}; +use super::state::block::{ + SszBlock, + Block, +}; +use super::state::block::validation::{ BlockValidationContext, SszBlockValidationError, BlockStatus, AttesterMap, ProposerMap, - MemoryDB, - canonical_hash, - Hash256, - Bitfield, - Keypair, - Signature, - AggregateSignature, - BlockStore, - PoWChainStore, - ValidatorStore, - SszStream, - SszBlock, - Block, - AttestationRecord, - TestParams, }; +use super::ssz::{ + SszStream, +}; +use super::utils::types::{ + Hash256, +}; + +#[derive(Debug)] +pub struct BlockTestParams { + pub total_validators: usize, + pub cycle_length: u8, + pub shard_count: u16, + pub shards_per_slot: u16, + pub validators_per_shard: usize, + pub block_slot: u64, + pub attestations_justified_slot: u64, + pub parent_proposer_index: usize, + pub validation_context_slot: u64, + pub validation_context_justified_slot: u64, + pub validation_context_finalized_slot: u64, +} pub struct TestStore { pub db: Arc, @@ -48,7 +74,7 @@ type ParentHashes = Vec; /// Setup for a block validation function, without actually executing the /// block validation function. -pub fn setup_block_validation_scenario(params: &TestParams) +pub fn setup_block_validation_scenario(params: &BlockTestParams) -> (Block, ParentHashes, AttesterMap, ProposerMap, TestStore) { let stores = TestStore::new(); @@ -95,72 +121,36 @@ pub fn setup_block_validation_scenario(params: &TestParams) let mut attester_map = AttesterMap::new(); let mut attestations = vec![]; let mut keypairs = vec![]; + /* + * For each shard in this slot, generate an attestation. + */ for shard in 0..shards_per_slot { + let mut signing_keys = vec![]; let mut attesters = vec![]; - let mut attester_bitfield = Bitfield::new(); - let mut aggregate_sig = AggregateSignature::new(); - - let parent_hashes_slice = { - let distance: usize = (block_slot - attestation_slot) as usize; - let last: usize = parent_hashes.len() - distance; - let first: usize = last - usize::from(cycle_length); - &parent_hashes[first..last] - }; - - let attestation_message = { - let mut stream = SszStream::new(); - stream.append(&attestation_slot); - stream.append_vec(&parent_hashes_slice.to_vec()); - stream.append(&shard); - stream.append(&shard_block_hash); - stream.append(&attestations_justified_slot); - let bytes = stream.drain(); - canonical_hash(&bytes) - }; - - - - for attestation_index in 0..validators_per_shard { - /* - * Add the attester to the attestation indices for this shard. - */ - attesters.push(i); - /* - * Set the voters bit on the bitfield to true. - */ - attester_bitfield.set_bit(attestation_index, true); - /* - * Generate a random keypair for this validatior and clone it into the - * list of keypairs. - */ + /* + * Generate a random keypair for each validator and clone it into the + * list of keypairs. Store it in the database. + */ + for _ in 0..validators_per_shard { let keypair = Keypair::random(); keypairs.push(keypair.clone()); - /* - * Store the validators public key in the database. - */ stores.validator.put_public_key_by_index(i, &keypair.pk).unwrap(); - /* - * Generate a new signature and aggregate it on the rolling signature. - */ - let sig = Signature::new(&attestation_message, &keypair.sk); - aggregate_sig.add(&sig); - /* - * Increment the validator counter to monotonically assign validators. - */ + signing_keys.push(Some(keypair.sk.clone())); + attesters.push(i); i += 1; } - attester_map.insert((attestation_slot, shard), attesters); - let attestation = AttestationRecord { - slot: attestation_slot, - shard_id: shard, - oblique_parent_hashes: vec![], - shard_block_hash, - attester_bitfield, - justified_slot: attestations_justified_slot, - justified_block_hash, - aggregate_sig, - }; + + let attestation = generate_attestation( + shard, + &shard_block_hash, + block_slot, + attestation_slot, + attestations_justified_slot, + &justified_block_hash, + cycle_length, + &parent_hashes, + &signing_keys[..]); attestations.push(attestation); } (attester_map, attestations, keypairs) @@ -194,7 +184,7 @@ pub fn serialize_block(b: &Block) -> Vec { /// /// Returns the Result returned from the block validation function. pub fn run_block_validation_scenario( - params: &TestParams, + params: &BlockTestParams, mutator_func: F) -> Result<(BlockStatus, Option), SszBlockValidationError> where F: FnOnce(Block, AttesterMap, ProposerMap, TestStore) diff --git a/tests/block_validation/mod.rs b/tests/block_validation/mod.rs new file mode 100644 index 000000000..3ff5741cb --- /dev/null +++ b/tests/block_validation/mod.rs @@ -0,0 +1,10 @@ +mod helpers; +mod tests; + +use super::bls; +use super::db; +use super::ssz; +use super::state; +use super::utils; + +use super::attestation_validation::helpers::generate_attestation; diff --git a/lighthouse/state/block/validation/tests/mod.rs b/tests/block_validation/tests.rs similarity index 87% rename from lighthouse/state/block/validation/tests/mod.rs rename to tests/block_validation/tests.rs index 02398dbe0..6adf026b0 100644 --- a/lighthouse/state/block/validation/tests/mod.rs +++ b/tests/block_validation/tests.rs @@ -1,62 +1,27 @@ -extern crate ssz; - -mod helpers; - -pub use self::helpers::{ +use super::bls::{ + AggregateSignature, +}; +use super::helpers::{ + BlockTestParams, TestStore, - setup_block_validation_scenario, run_block_validation_scenario, serialize_block, }; - -use self::ssz::{ - SszStream, +use super::state::block::{ + SszBlock, + Block, }; -use super::{ - BlockValidationContext, +use super::state::block::validation::{ SszBlockValidationError, BlockStatus, - AttesterMap, ProposerMap, }; -use super::db::stores::{ - BlockStore, - PoWChainStore, - ValidatorStore, -}; -use super::db::{ - MemoryDB, -}; use super::utils::hash::canonical_hash; use super::utils::types::{ Hash256, - Bitfield, -}; -use super::SszBlock; -use super::super::Block; -use super::super::attestation_record::AttestationRecord; -use super::super::super::bls::{ - Keypair, - Signature, - AggregateSignature, }; -#[derive(Debug)] -pub struct TestParams { - pub total_validators: usize, - pub cycle_length: u8, - pub shard_count: u16, - pub shards_per_slot: u16, - pub validators_per_shard: usize, - pub block_slot: u64, - pub attestations_justified_slot: u64, - pub parent_proposer_index: usize, - pub validation_context_slot: u64, - pub validation_context_justified_slot: u64, - pub validation_context_finalized_slot: u64, -} - -fn get_simple_params() -> TestParams { +fn get_simple_params() -> BlockTestParams { let validators_per_shard: usize = 5; let cycle_length: u8 = 2; let shard_count: u16 = 4; @@ -70,7 +35,7 @@ fn get_simple_params() -> TestParams { let validation_context_justified_slot = attestations_justified_slot; let validation_context_finalized_slot = 0; - TestParams { + BlockTestParams { total_validators, cycle_length, shard_count, diff --git a/tests/main.rs b/tests/main.rs new file mode 100644 index 000000000..caa601105 --- /dev/null +++ b/tests/main.rs @@ -0,0 +1,12 @@ +extern crate lighthouse; +extern crate ssz; + +#[cfg(test)] +mod attestation_validation; +#[cfg(test)] +mod block_validation; + +use lighthouse::bls; +use lighthouse::db; +use lighthouse::state; +use lighthouse::utils;