From c4f9f927f3dea540503735256af5834efc1170e5 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 23 Aug 2018 18:31:58 +1000 Subject: [PATCH 01/13] Rename "state" to "chain_state" --- lighthouse/state/{config.rs => chain_config.rs} | 4 ++-- lighthouse/state/mod.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename lighthouse/state/{config.rs => chain_config.rs} (73%) diff --git a/lighthouse/state/config.rs b/lighthouse/state/chain_config.rs similarity index 73% rename from lighthouse/state/config.rs rename to lighthouse/state/chain_config.rs index 63aa7de02..c5b244f20 100644 --- a/lighthouse/state/config.rs +++ b/lighthouse/state/chain_config.rs @@ -1,8 +1,8 @@ -pub struct Config { +pub struct ChainConfig { pub cycle_length: u8, } -impl Config { +impl ChainConfig { pub fn standard() -> Self { Self { cycle_length: 8, diff --git a/lighthouse/state/mod.rs b/lighthouse/state/mod.rs index 45e845fef..5d5c2d7da 100644 --- a/lighthouse/state/mod.rs +++ b/lighthouse/state/mod.rs @@ -9,7 +9,7 @@ use super::utils; pub mod active_state; pub mod attestation_record; pub mod crystallized_state; -pub mod config; +pub mod chain_config; pub mod block; pub mod crosslink_record; pub mod shard_and_committee; From 8ba7601ab3d4cf480ac9535e1402feaaa65f5f4c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 23 Aug 2018 18:32:18 +1000 Subject: [PATCH 02/13] Add get_signed_parent_hashes (untested) --- lighthouse/state/mod.rs | 1 + lighthouse/state/transition/helpers.rs | 68 ++++++++++++++++++++++++++ lighthouse/state/transition/mod.rs | 11 +++++ 3 files changed, 80 insertions(+) create mode 100644 lighthouse/state/transition/helpers.rs create mode 100644 lighthouse/state/transition/mod.rs diff --git a/lighthouse/state/mod.rs b/lighthouse/state/mod.rs index 5d5c2d7da..d10acf8fd 100644 --- a/lighthouse/state/mod.rs +++ b/lighthouse/state/mod.rs @@ -13,4 +13,5 @@ pub mod chain_config; pub mod block; pub mod crosslink_record; pub mod shard_and_committee; +pub mod transition; pub mod validator_record; diff --git a/lighthouse/state/transition/helpers.rs b/lighthouse/state/transition/helpers.rs new file mode 100644 index 000000000..c8e98316e --- /dev/null +++ b/lighthouse/state/transition/helpers.rs @@ -0,0 +1,68 @@ +use super::Hash256; +use super::TransitionError; + +pub fn get_signed_parent_hashes( + cycle_length: u64, + block_slot: u64, + attestation_slot: u64, + current_hashes: Vec, + oblique_hashes: Vec) + -> Result, TransitionError> +{ + let start = cycle_length.checked_add(attestation_slot) + .and_then(|x| x.checked_sub(block_slot)) + .ok_or(TransitionError::IntWrapping)?; + let start = start as usize; + + let end = cycle_length.checked_mul(2) + .and_then(|x| x.checked_add(attestation_slot)) + .and_then(|x| x.checked_sub(block_slot)) + .and_then(|x| x.checked_sub(oblique_hashes.len() as u64)) + .ok_or(TransitionError::IntWrapping)?; + let end = end as usize; + + println!("start: {}, end: {}", start, end); + + if end >= current_hashes.len() { + return Err(TransitionError::OutOfBounds); + } + if start > end { + return Err(TransitionError::InvalidInput("cats")); + } + + let mut hashes = Vec::new(); + + hashes.extend_from_slice( + ¤t_hashes[start..end]); + hashes.append(&mut oblique_hashes.clone()); + + Ok(hashes) +} + + +#[cfg(test)] +mod tests { + use super::*; + + fn get_n_hashes(value: &[u8], n: usize) -> Vec { + (0..n).map(|_| Hash256::from_slice(value)).collect() + } + + #[test] + fn test_get_signed_hashes() { + let cycle_length: u64 = 8; + let block_slot: u64 = 500; + let attestation_slot: u64 = 498; + let current_hashes = + get_n_hashes(b"0", 100); + let oblique_hashes = get_n_hashes(b"1", 2); + let result = get_signed_parent_hashes( + cycle_length, + block_slot, + attestation_slot, + current_hashes, + oblique_hashes); + // TODO: complete testing + assert!(result.is_ok()); + } +} diff --git a/lighthouse/state/transition/mod.rs b/lighthouse/state/transition/mod.rs new file mode 100644 index 000000000..676266d95 --- /dev/null +++ b/lighthouse/state/transition/mod.rs @@ -0,0 +1,11 @@ +use super::super::utils::types::Hash256; + +pub mod helpers; + +#[derive(Debug)] +pub enum TransitionError { + IntWrapping, + OutOfBounds, + InvalidInput(&str), +} + From 859c48d5e58e5cb218c6c8e7836bebb6aa29607d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 24 Aug 2018 14:33:05 +1000 Subject: [PATCH 03/13] Progress on parent hash selection. Add test. --- lighthouse/state/transition/helpers.rs | 91 ++++++++++++++++++-------- lighthouse/state/transition/mod.rs | 2 +- 2 files changed, 63 insertions(+), 30 deletions(-) diff --git a/lighthouse/state/transition/helpers.rs b/lighthouse/state/transition/helpers.rs index c8e98316e..94de8a3d9 100644 --- a/lighthouse/state/transition/helpers.rs +++ b/lighthouse/state/transition/helpers.rs @@ -2,38 +2,62 @@ use super::Hash256; use super::TransitionError; pub fn get_signed_parent_hashes( - cycle_length: u64, + cycle_length: u8, block_slot: u64, attestation_slot: u64, current_hashes: Vec, oblique_hashes: Vec) -> Result, TransitionError> { - let start = cycle_length.checked_add(attestation_slot) - .and_then(|x| x.checked_sub(block_slot)) - .ok_or(TransitionError::IntWrapping)?; - let start = start as usize; + // This cast places a limit on cycle_length. If you change it, check math + // for overflow. + let cycle_length: u64 = cycle_length as u64; - let end = cycle_length.checked_mul(2) - .and_then(|x| x.checked_add(attestation_slot)) - .and_then(|x| x.checked_sub(block_slot)) + if current_hashes.len() as u64 != (cycle_length * 2) { + return Err(TransitionError::InvalidInput(String::from( + "current_hashes.len() must equal cycle_length * 2"))); + } + if attestation_slot >= block_slot { + return Err(TransitionError::InvalidInput(String::from( + "attestation_slot must be less than block_slot"))); + } + if oblique_hashes.len() as u64 > cycle_length { + return Err(TransitionError::InvalidInput(String::from( + "oblique_hashes.len() must be <= cycle_length * 2"))); + } + + /* + * Cannot underflow as block_slot cannot be less + * than attestation_slot. + */ + let attestation_distance = block_slot - attestation_slot; + + if attestation_distance > cycle_length { + return Err(TransitionError::InvalidInput(String::from( + "attestation_slot must be withing one cycle of block_slot"))); + } + + /* + * Cannot underflow because attestation_distance cannot + * be larger than cycle_length. + */ + let start = cycle_length - attestation_distance; + + /* + * Overflow is potentially impossible, but proof is complicated + * enough to just use checked math. + * + * Math is: + * start + cycle_length - oblique_hashes.len() + */ + let end = start.checked_add(cycle_length) .and_then(|x| x.checked_sub(oblique_hashes.len() as u64)) .ok_or(TransitionError::IntWrapping)?; - let end = end as usize; - - println!("start: {}, end: {}", start, end); - if end >= current_hashes.len() { - return Err(TransitionError::OutOfBounds); - } - if start > end { - return Err(TransitionError::InvalidInput("cats")); - } let mut hashes = Vec::new(); - hashes.extend_from_slice( - ¤t_hashes[start..end]); + ¤t_hashes[(start as usize)..(end as usize)]); hashes.append(&mut oblique_hashes.clone()); Ok(hashes) @@ -44,25 +68,34 @@ pub fn get_signed_parent_hashes( mod tests { use super::*; - fn get_n_hashes(value: &[u8], n: usize) -> Vec { - (0..n).map(|_| Hash256::from_slice(value)).collect() + fn get_range_of_hashes(from: usize, to: usize) -> Vec { + (from..to).map(|i| get_hash(&vec![i as u8])).collect() + } + + fn get_hash(value: &[u8]) -> Hash256 { + Hash256::from_slice(value) } #[test] - fn test_get_signed_hashes() { - let cycle_length: u64 = 8; - let block_slot: u64 = 500; - let attestation_slot: u64 = 498; - let current_hashes = - get_n_hashes(b"0", 100); - let oblique_hashes = get_n_hashes(b"1", 2); + fn test_get_signed_hashes_no_oblique() { + /* + * Google Slides example. + */ + let cycle_length: u8 = 8; + let block_slot: u64 = 19; + let attestation_slot: u64 = 15; + let current_hashes = get_range_of_hashes(3, 19); + let oblique_hashes = vec![]; let result = get_signed_parent_hashes( cycle_length, block_slot, attestation_slot, current_hashes, oblique_hashes); - // TODO: complete testing assert!(result.is_ok()); + let result = result.unwrap(); + assert_eq!(result.len(), cycle_length as usize); + let expected_result = get_range_of_hashes(7, 15); + assert_eq!(result, expected_result); } } diff --git a/lighthouse/state/transition/mod.rs b/lighthouse/state/transition/mod.rs index 676266d95..e958f7831 100644 --- a/lighthouse/state/transition/mod.rs +++ b/lighthouse/state/transition/mod.rs @@ -6,6 +6,6 @@ pub mod helpers; pub enum TransitionError { IntWrapping, OutOfBounds, - InvalidInput(&str), + InvalidInput(String), } From 4ec0de6f5e1bd18c06583e310d89644c4096c3ed Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 24 Aug 2018 15:26:27 +1000 Subject: [PATCH 04/13] Use borrows for get_signed_parent_hases, add tests --- lighthouse/state/transition/helpers.rs | 137 ++++++++++++++++++++++--- 1 file changed, 124 insertions(+), 13 deletions(-) diff --git a/lighthouse/state/transition/helpers.rs b/lighthouse/state/transition/helpers.rs index 94de8a3d9..ddd7d4635 100644 --- a/lighthouse/state/transition/helpers.rs +++ b/lighthouse/state/transition/helpers.rs @@ -2,16 +2,16 @@ use super::Hash256; use super::TransitionError; pub fn get_signed_parent_hashes( - cycle_length: u8, - block_slot: u64, - attestation_slot: u64, - current_hashes: Vec, - oblique_hashes: Vec) + cycle_length: &u8, + block_slot: &u64, + attestation_slot: &u64, + current_hashes: &Vec, + oblique_hashes: &Vec) -> Result, TransitionError> { // This cast places a limit on cycle_length. If you change it, check math // for overflow. - let cycle_length: u64 = cycle_length as u64; + let cycle_length: u64 = *cycle_length as u64; if current_hashes.len() as u64 != (cycle_length * 2) { return Err(TransitionError::InvalidInput(String::from( @@ -47,7 +47,7 @@ pub fn get_signed_parent_hashes( * Overflow is potentially impossible, but proof is complicated * enough to just use checked math. * - * Math is: + * Arithmetic is: * start + cycle_length - oblique_hashes.len() */ let end = start.checked_add(cycle_length) @@ -77,9 +77,59 @@ mod tests { } #[test] - fn test_get_signed_hashes_no_oblique() { + fn test_get_signed_hashes_oblique_scenario_1() { + /* + * Two oblique hashes. + */ + let cycle_length: u8 = 8; + let block_slot: u64 = 19; + let attestation_slot: u64 = 15; + let current_hashes = get_range_of_hashes(3, 19); + let oblique_hashes = get_range_of_hashes(100, 102); + let result = get_signed_parent_hashes( + &cycle_length, + &block_slot, + &attestation_slot, + ¤t_hashes, + &oblique_hashes); + assert!(result.is_ok()); + let result = result.unwrap(); + assert_eq!(result.len(), cycle_length as usize); + + let mut expected_result = get_range_of_hashes(7, 13); + expected_result.append(&mut get_range_of_hashes(100, 102)); + assert_eq!(result, expected_result); + } + + #[test] + fn test_get_signed_hashes_oblique_scenario_2() { + /* + * All oblique hashes. + */ + let cycle_length: u8 = 8; + let block_slot: u64 = 19; + let attestation_slot: u64 = 15; + let current_hashes = get_range_of_hashes(3, 19); + let oblique_hashes = get_range_of_hashes(100, 108); + let result = get_signed_parent_hashes( + &cycle_length, + &block_slot, + &attestation_slot, + ¤t_hashes, + &oblique_hashes); + assert!(result.is_ok()); + let result = result.unwrap(); + assert_eq!(result.len(), cycle_length as usize); + + let expected_result = get_range_of_hashes(100, 108); + assert_eq!(result, expected_result); + } + + #[test] + fn test_get_signed_hashes_scenario_1() { /* * Google Slides example. + * https://tinyurl.com/ybzn2spw */ let cycle_length: u8 = 8; let block_slot: u64 = 19; @@ -87,15 +137,76 @@ mod tests { let current_hashes = get_range_of_hashes(3, 19); let oblique_hashes = vec![]; let result = get_signed_parent_hashes( - cycle_length, - block_slot, - attestation_slot, - current_hashes, - oblique_hashes); + &cycle_length, + &block_slot, + &attestation_slot, + ¤t_hashes, + &oblique_hashes); assert!(result.is_ok()); let result = result.unwrap(); assert_eq!(result.len(), cycle_length as usize); let expected_result = get_range_of_hashes(7, 15); assert_eq!(result, expected_result); } + + #[test] + fn test_get_signed_hashes_scenario_2() { + /* + * Block 1, attestation 0. + */ + let cycle_length: u8 = 8; + let block_slot: u64 = 1; + let attestation_slot: u64 = 0; + let current_hashes = get_range_of_hashes(0, 16); + let oblique_hashes = vec![]; + let result = get_signed_parent_hashes( + &cycle_length, + &block_slot, + &attestation_slot, + ¤t_hashes, + &oblique_hashes); + assert!(result.is_ok()); + let result = result.unwrap(); + assert_eq!(result.len(), cycle_length as usize); + let expected_result = get_range_of_hashes(7, 15); + assert_eq!(result, expected_result); + } + + #[test] + fn test_get_signed_hashes_scenario_3() { + /* + * attestation_slot too large + */ + let cycle_length: u8 = 8; + let block_slot: u64 = 100; + let attestation_slot: u64 = 100; + let current_hashes = get_range_of_hashes(0, 16); + let oblique_hashes = vec![]; + let result = get_signed_parent_hashes( + &cycle_length, + &block_slot, + &attestation_slot, + ¤t_hashes, + &oblique_hashes); + assert!(result.is_err()); + } + + #[test] + fn test_get_signed_hashes_scenario_4() { + /* + * Current hashes too small + */ + let cycle_length: u8 = 8; + let block_slot: u64 = 100; + let attestation_slot: u64 = 99; + let current_hashes = get_range_of_hashes(0, 15); + let oblique_hashes = vec![]; + let result = get_signed_parent_hashes( + &cycle_length, + &block_slot, + &attestation_slot, + ¤t_hashes, + &oblique_hashes); + assert!(result.is_err()); + } } From 1a3da944e5efec79980d5047070356927a8ab91e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 24 Aug 2018 15:36:26 +1000 Subject: [PATCH 05/13] Add comment to get_signed_parent_hashes --- lighthouse/state/transition/helpers.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lighthouse/state/transition/helpers.rs b/lighthouse/state/transition/helpers.rs index ddd7d4635..d1a228865 100644 --- a/lighthouse/state/transition/helpers.rs +++ b/lighthouse/state/transition/helpers.rs @@ -1,6 +1,14 @@ use super::Hash256; use super::TransitionError; +/// This function is used to select the hashes used in +/// the signing of an AttestationRecord. +/// +/// It either returns Result with a vector of length `cycle_length,` or +/// returns an Error. +/// +/// See this slide for more information: +/// https://tinyurl.com/ybzn2spw pub fn get_signed_parent_hashes( cycle_length: &u8, block_slot: &u64, From d0c020db9d89b76b1f1d73351ecab8e41585315c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 24 Aug 2018 15:41:00 +1000 Subject: [PATCH 06/13] Rename "get_signed_parent_hashes" --- lighthouse/state/transition/helpers.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lighthouse/state/transition/helpers.rs b/lighthouse/state/transition/helpers.rs index d1a228865..624d8c443 100644 --- a/lighthouse/state/transition/helpers.rs +++ b/lighthouse/state/transition/helpers.rs @@ -7,9 +7,12 @@ use super::TransitionError; /// It either returns Result with a vector of length `cycle_length,` or /// returns an Error. /// +/// This function corresponds to the `get_signed_parent_hashes` function +/// in the Python reference implentation. +/// /// See this slide for more information: /// https://tinyurl.com/ybzn2spw -pub fn get_signed_parent_hashes( +pub fn attestation_parent_hashes( cycle_length: &u8, block_slot: &u64, attestation_slot: &u64, @@ -94,7 +97,7 @@ mod tests { let attestation_slot: u64 = 15; let current_hashes = get_range_of_hashes(3, 19); let oblique_hashes = get_range_of_hashes(100, 102); - let result = get_signed_parent_hashes( + let result = attestation_parent_hashes( &cycle_length, &block_slot, &attestation_slot, @@ -119,7 +122,7 @@ mod tests { let attestation_slot: u64 = 15; let current_hashes = get_range_of_hashes(3, 19); let oblique_hashes = get_range_of_hashes(100, 108); - let result = get_signed_parent_hashes( + let result = attestation_parent_hashes( &cycle_length, &block_slot, &attestation_slot, @@ -144,7 +147,7 @@ mod tests { let attestation_slot: u64 = 15; let current_hashes = get_range_of_hashes(3, 19); let oblique_hashes = vec![]; - let result = get_signed_parent_hashes( + let result = attestation_parent_hashes( &cycle_length, &block_slot, &attestation_slot, @@ -167,7 +170,7 @@ mod tests { let attestation_slot: u64 = 0; let current_hashes = get_range_of_hashes(0, 16); let oblique_hashes = vec![]; - let result = get_signed_parent_hashes( + let result = attestation_parent_hashes( &cycle_length, &block_slot, &attestation_slot, @@ -190,7 +193,7 @@ mod tests { let attestation_slot: u64 = 100; let current_hashes = get_range_of_hashes(0, 16); let oblique_hashes = vec![]; - let result = get_signed_parent_hashes( + let result = attestation_parent_hashes( &cycle_length, &block_slot, &attestation_slot, @@ -209,7 +212,7 @@ mod tests { let attestation_slot: u64 = 99; let current_hashes = get_range_of_hashes(0, 15); let oblique_hashes = vec![]; - let result = get_signed_parent_hashes( + let result = attestation_parent_hashes( &cycle_length, &block_slot, &attestation_slot, From 1a00f5b429d8218716c9cea2ca373f915c320cc1 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 24 Aug 2018 15:48:43 +1000 Subject: [PATCH 07/13] Rename helpers -> attestation_parent_hashes --- .../transition/{helpers.rs => attestation_parent_hashes.rs} | 0 lighthouse/state/transition/mod.rs | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename lighthouse/state/transition/{helpers.rs => attestation_parent_hashes.rs} (100%) diff --git a/lighthouse/state/transition/helpers.rs b/lighthouse/state/transition/attestation_parent_hashes.rs similarity index 100% rename from lighthouse/state/transition/helpers.rs rename to lighthouse/state/transition/attestation_parent_hashes.rs diff --git a/lighthouse/state/transition/mod.rs b/lighthouse/state/transition/mod.rs index e958f7831..ef7b4e778 100644 --- a/lighthouse/state/transition/mod.rs +++ b/lighthouse/state/transition/mod.rs @@ -1,6 +1,6 @@ use super::super::utils::types::Hash256; -pub mod helpers; +pub mod attestation_parent_hashes; #[derive(Debug)] pub enum TransitionError { From 6f08700ea01066863844c3960e2fd798e7a6a356 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 24 Aug 2018 16:01:24 +1000 Subject: [PATCH 08/13] Move shuffling mod into state/transition --- lighthouse/main.rs | 7 +++---- lighthouse/state/transition/mod.rs | 2 ++ lighthouse/{ => state/transition}/shuffling/mod.rs | 6 +++--- lighthouse/{ => state/transition}/shuffling/rng.rs | 8 ++++---- 4 files changed, 12 insertions(+), 11 deletions(-) rename lighthouse/{ => state/transition}/shuffling/mod.rs (96%) rename lighthouse/{ => state/transition}/shuffling/rng.rs (99%) diff --git a/lighthouse/main.rs b/lighthouse/main.rs index 867a3ed00..0a9f35288 100644 --- a/lighthouse/main.rs +++ b/lighthouse/main.rs @@ -8,13 +8,12 @@ extern crate futures; pub mod db; pub mod client; -pub mod shuffling; pub mod state; pub mod sync; pub mod utils; pub mod config; -use std::path::PathBuf; +use std::path::PathBuf; use slog::Drain; use clap::{ Arg, App }; @@ -59,9 +58,9 @@ fn main() { return; } } - + // Log configuration - info!(log, ""; + info!(log, ""; "data_dir" => &config.data_dir.to_str(), "port" => &config.p2p_listen_port); diff --git a/lighthouse/state/transition/mod.rs b/lighthouse/state/transition/mod.rs index ef7b4e778..40dd10b8e 100644 --- a/lighthouse/state/transition/mod.rs +++ b/lighthouse/state/transition/mod.rs @@ -9,3 +9,5 @@ pub enum TransitionError { InvalidInput(String), } + + diff --git a/lighthouse/shuffling/mod.rs b/lighthouse/state/transition/shuffling/mod.rs similarity index 96% rename from lighthouse/shuffling/mod.rs rename to lighthouse/state/transition/shuffling/mod.rs index d706fad2c..432f38ff4 100644 --- a/lighthouse/shuffling/mod.rs +++ b/lighthouse/state/transition/shuffling/mod.rs @@ -9,18 +9,18 @@ pub enum ShuffleErr { ExceedsListLength, } -/// Performs a deterministic, in-place shuffle of a vector of bytes. +/// Performs a deterministic, in-place shuffle of a vector of bytes. /// The final order of the shuffle is determined by successive hashes /// of the supplied `seed`. pub fn shuffle( seed: &[u8], - mut list: Vec) + mut list: Vec) -> Result, ShuffleErr> { let mut rng = ShuffleRng::new(seed); if list.len() > rng.rand_max as usize { return Err(ShuffleErr::ExceedsListLength); - } + } for i in 0..(list.len() - 1) { let n = list.len() - i; let j = rng.rand_range(n as u32) as usize + i; diff --git a/lighthouse/shuffling/rng.rs b/lighthouse/state/transition/shuffling/rng.rs similarity index 99% rename from lighthouse/shuffling/rng.rs rename to lighthouse/state/transition/shuffling/rng.rs index 7564b2ab1..224e9130f 100644 --- a/lighthouse/shuffling/rng.rs +++ b/lighthouse/state/transition/shuffling/rng.rs @@ -94,22 +94,22 @@ mod tests { &[0, 1, 1], 0); assert_eq!(x, 257); - + x = int_from_byte_slice( &[1, 1, 1], 0); assert_eq!(x, 65793); - + x = int_from_byte_slice( &[255, 1, 1], 0); assert_eq!(x, 16711937); - + x = int_from_byte_slice( &[255, 255, 255], 0); assert_eq!(x, 16777215); - + x = int_from_byte_slice( &[0x8f, 0xbb, 0xc7], 0); From 133769e6e5d562db397041cba0481ec34c9ecb36 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 24 Aug 2018 16:01:57 +1000 Subject: [PATCH 09/13] Expose function directly in transition --- lighthouse/state/transition/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lighthouse/state/transition/mod.rs b/lighthouse/state/transition/mod.rs index 40dd10b8e..37038e49a 100644 --- a/lighthouse/state/transition/mod.rs +++ b/lighthouse/state/transition/mod.rs @@ -1,6 +1,8 @@ use super::super::utils::types::Hash256; -pub mod attestation_parent_hashes; +mod attestation_parent_hashes; + +pub use self::attestation_parent_hashes::attestation_parent_hashes; #[derive(Debug)] pub enum TransitionError { From d57f86c9add50371042f8c2d6d8024d8af560f0a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 28 Aug 2018 17:51:23 +1000 Subject: [PATCH 10/13] Add readme for shuffling --- lighthouse/state/transition/shuffling/README.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 lighthouse/state/transition/shuffling/README.md diff --git a/lighthouse/state/transition/shuffling/README.md b/lighthouse/state/transition/shuffling/README.md new file mode 100644 index 000000000..b0f4f5245 --- /dev/null +++ b/lighthouse/state/transition/shuffling/README.md @@ -0,0 +1,2 @@ +This module includes the fundamental shuffling function. It does not do the +full validator delegation amongst slots. From c1a0df8c17faed29bc8cd8fd67c8288a0f4a20fc Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 28 Aug 2018 17:51:57 +1000 Subject: [PATCH 11/13] Implement get_new_shuffling w/o tests --- lighthouse/state/chain_config.rs | 4 + lighthouse/state/shard_and_committee.rs | 4 +- lighthouse/state/transition/mod.rs | 7 + .../state/transition/validator_allocation.rs | 145 ++++++++++++++++++ 4 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 lighthouse/state/transition/validator_allocation.rs diff --git a/lighthouse/state/chain_config.rs b/lighthouse/state/chain_config.rs index c5b244f20..3d285e7a7 100644 --- a/lighthouse/state/chain_config.rs +++ b/lighthouse/state/chain_config.rs @@ -1,11 +1,15 @@ pub struct ChainConfig { pub cycle_length: u8, + pub shard_count: u16, + pub min_committee_size: u64, } impl ChainConfig { pub fn standard() -> Self { Self { cycle_length: 8, + shard_count: 1024, + min_committee_size: 128, } } } diff --git a/lighthouse/state/shard_and_committee.rs b/lighthouse/state/shard_and_committee.rs index 788d78c4b..7f57fa6c8 100644 --- a/lighthouse/state/shard_and_committee.rs +++ b/lighthouse/state/shard_and_committee.rs @@ -1,6 +1,8 @@ + +#[derive(Clone,Debug)] pub struct ShardAndCommittee { pub shard_id: u16, - pub committee: Vec + pub committee: Vec } impl ShardAndCommittee { diff --git a/lighthouse/state/transition/mod.rs b/lighthouse/state/transition/mod.rs index 37038e49a..a79e92d06 100644 --- a/lighthouse/state/transition/mod.rs +++ b/lighthouse/state/transition/mod.rs @@ -1,8 +1,15 @@ use super::super::utils::types::Hash256; +use super::chain_config::ChainConfig; +use super::shard_and_committee::ShardAndCommittee; +use super::validator_record::ValidatorRecord; mod attestation_parent_hashes; +mod shuffling; +mod validator_allocation; pub use self::attestation_parent_hashes::attestation_parent_hashes; +pub use self::validator_allocation::get_new_shuffling; +pub use self::shuffling::shuffle; #[derive(Debug)] pub enum TransitionError { diff --git a/lighthouse/state/transition/validator_allocation.rs b/lighthouse/state/transition/validator_allocation.rs new file mode 100644 index 000000000..eb1d49413 --- /dev/null +++ b/lighthouse/state/transition/validator_allocation.rs @@ -0,0 +1,145 @@ +use super::shuffle; +use super::ChainConfig; +use super::TransitionError; +use super::ValidatorRecord; +use super::ShardAndCommittee; + +pub fn get_new_shuffling( + seed: &[u8], + validators: &Vec, + dynasty: &u64, + crosslinking_shard_start: &u16, + config: &ChainConfig) + -> Result +{ + let shuffled_validator_indices = { + let mut validator_indices = active_validator_indicies(dynasty, validators); + match shuffle(seed, validator_indices) { + Ok(shuffled) => shuffled, + _ => return Err(TransitionError::InvalidInput( + String::from("Shuffle list length exceed."))) + } + }; + let shard_indices = (0_usize..config.shard_count as usize).into_iter().collect(); + let crosslinking_shard_start = *crosslinking_shard_start as usize; + let cycle_length = config.cycle_length as usize; + let min_committee_size = config.min_committee_size as usize; + generate_cycle( + &shuffled_validator_indices, + &shard_indices, + &crosslinking_shard_start, + &cycle_length, + &min_committee_size) +} + +/// Produce a vector of validators indicies where those +/// validators start and end dynasties are within the supplied +/// `dynasty`. +fn active_validator_indicies( + dynasty: &u64, + validators: &Vec) + -> Vec +{ + validators.iter() + .enumerate() + .filter_map(|(i, validator)| { + if (validator.start_dynasty >= *dynasty) & + (validator.end_dynasty < *dynasty) + { + Some(i) + } else { + None + } + }) + .collect() +} + +type DelegatedSlot = Vec; +type DelegatedCycle = Vec; + +fn generate_cycle( + validator_indices: &Vec, + shard_indices: &Vec, + crosslinking_shard_start: &usize, + cycle_length: &usize, + min_committee_size: &usize) + -> Result +{ + let validator_count = validator_indices.len(); + let shard_count = shard_indices.len(); + + let (committees_per_slot, slots_per_committee) = { + if validator_count >= cycle_length * min_committee_size { + let committees_per_slot = validator_count / cycle_length / (min_committee_size * 2); + let slots_per_committee = 1; + (committees_per_slot, slots_per_committee) + } else { + let committees_per_slot = 1; + let mut slots_per_committee = 1; + while (validator_count * slots_per_committee < cycle_length * min_committee_size) & + (slots_per_committee < *cycle_length) { + slots_per_committee = slots_per_committee * 2; + } + (committees_per_slot, slots_per_committee) + } + }; + + let mut cycle: DelegatedCycle = vec![]; + let split_iter = validator_indices.split(|i| i % cycle_length == 0); + for (i, slot_indices) in split_iter.enumerate() { + let shard_id_start = crosslinking_shard_start * i * committees_per_slot / slots_per_committee; + + let shard_iter = slot_indices.split(|i| i % committees_per_slot == 0); + let slot: DelegatedSlot = shard_iter + .enumerate() + .map(|(j, shard_indices)| { + ShardAndCommittee{ + shard_id: ((shard_id_start + j) % shard_count) as u16, + committee: shard_indices.to_vec(), + } + }) + .collect(); + cycle.push(slot); + }; + Ok(cycle) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn generate_cycle_helper( + validator_count: &usize, + shard_count: &usize, + crosslinking_shard_start: &usize, + cycle_length: &usize, + min_committee_size: &usize) + -> Result + { + let validator_indices = (0_usize..*validator_count).into_iter().collect(); + let shard_indices = (0_usize..*shard_count).into_iter().collect(); + generate_cycle( + &validator_indices, + &shard_indices, + &crosslinking_shard_start, + &cycle_length, + &min_committee_size) + } + + #[test] + fn test_generate_cycle() { + let validator_count: usize = 100; + let shard_count: usize = 10; + let crosslinking_shard_start: usize = 0; + let cycle_length: usize = 20; + let min_committee_size: usize = 10; + let result = generate_cycle_helper( + &validator_count, + &shard_count, + &crosslinking_shard_start, + &cycle_length, + &min_committee_size); + println!("{:?}", result); + } + +} From 2277273caa49ac03cda2fe2e9fe6591ebcc8e713 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 29 Aug 2018 16:57:25 +1000 Subject: [PATCH 12/13] Fix get_new_shuffling, add functioning test. --- .../state/transition/validator_allocation.rs | 125 ++++++++++++++---- 1 file changed, 102 insertions(+), 23 deletions(-) diff --git a/lighthouse/state/transition/validator_allocation.rs b/lighthouse/state/transition/validator_allocation.rs index eb1d49413..f425eded6 100644 --- a/lighthouse/state/transition/validator_allocation.rs +++ b/lighthouse/state/transition/validator_allocation.rs @@ -70,7 +70,7 @@ fn generate_cycle( let (committees_per_slot, slots_per_committee) = { if validator_count >= cycle_length * min_committee_size { - let committees_per_slot = validator_count / cycle_length / (min_committee_size * 2); + let committees_per_slot = validator_count / cycle_length / (min_committee_size * 2) + 1; let slots_per_committee = 1; (committees_per_slot, slots_per_committee) } else { @@ -84,23 +84,24 @@ fn generate_cycle( } }; - let mut cycle: DelegatedCycle = vec![]; - let split_iter = validator_indices.split(|i| i % cycle_length == 0); - for (i, slot_indices) in split_iter.enumerate() { - let shard_id_start = crosslinking_shard_start * i * committees_per_slot / slots_per_committee; + let cycle = validator_indices + .chunks(validator_indices.len() / *cycle_length) + .enumerate() + .map(|(i, slot_indices)| { + let shard_id_start = crosslinking_shard_start + i * committees_per_slot / slots_per_committee; + return slot_indices + .chunks(slot_indices.len() / committees_per_slot) + .enumerate() + .map(|(j, shard_indices)| { + return ShardAndCommittee{ + shard_id: ((shard_id_start + j) % shard_count) as u16, + committee: shard_indices.to_vec(), + } + }) + .collect() + }) + .collect(); - let shard_iter = slot_indices.split(|i| i % committees_per_slot == 0); - let slot: DelegatedSlot = shard_iter - .enumerate() - .map(|(j, shard_indices)| { - ShardAndCommittee{ - shard_id: ((shard_id_start + j) % shard_count) as u16, - committee: shard_indices.to_vec(), - } - }) - .collect(); - cycle.push(slot); - }; Ok(cycle) } @@ -114,18 +115,76 @@ mod tests { crosslinking_shard_start: &usize, cycle_length: &usize, min_committee_size: &usize) - -> Result + -> (Vec, Vec, Result) { let validator_indices = (0_usize..*validator_count).into_iter().collect(); let shard_indices = (0_usize..*shard_count).into_iter().collect(); - generate_cycle( + let result = generate_cycle( &validator_indices, &shard_indices, &crosslinking_shard_start, &cycle_length, - &min_committee_size) + &min_committee_size); + (validator_indices, shard_indices, result) } + #[allow(dead_code)] + fn print_cycle(cycle: &DelegatedCycle) { + cycle.iter() + .enumerate() + .for_each(|(i, slot)| { + println!("slot {:?}", &i); + slot.iter() + .enumerate() + .for_each(|(i, sac)| { + println!("#{:?}\tshard_id={}\tcommittee.len()={}", + &i, &sac.shard_id, &sac.committee.len()) + }) + }); + } + + fn flatten_validators(cycle: &DelegatedCycle) + -> Vec + { + let mut flattened = vec![]; + for slot in cycle.iter() { + for sac in slot.iter() { + for validator in sac.committee.iter() { + flattened.push(*validator); + } + } + } + flattened + } + + fn flatten_and_dedup_shards(cycle: &DelegatedCycle) + -> Vec + { + let mut flattened = vec![]; + for slot in cycle.iter() { + for sac in slot.iter() { + flattened.push(sac.shard_id as usize); + } + } + flattened.dedup(); + flattened + } + + fn flatten_shards_in_slots(cycle: &DelegatedCycle) + -> Vec> + { + let mut shards_in_slots: Vec> = vec![]; + for slot in cycle.iter() { + let mut shards: Vec = vec![]; + for sac in slot.iter() { + shards.push(sac.shard_id as usize); + } + shards_in_slots.push(shards); + } + shards_in_slots + } + + #[test] fn test_generate_cycle() { let validator_count: usize = 100; @@ -133,13 +192,33 @@ mod tests { let crosslinking_shard_start: usize = 0; let cycle_length: usize = 20; let min_committee_size: usize = 10; - let result = generate_cycle_helper( + let (validators, shards, result) = generate_cycle_helper( &validator_count, &shard_count, &crosslinking_shard_start, &cycle_length, &min_committee_size); - println!("{:?}", result); - } + let cycle = result.unwrap(); + let assigned_validators = flatten_validators(&cycle); + let assigned_shards = flatten_and_dedup_shards(&cycle); + let shards_in_slots = flatten_shards_in_slots(&cycle); + assert_eq!(assigned_validators, validators, "Validator assignment incorrect"); + assert_eq!(assigned_shards, shards, "Shard assignment incorrect"); + + let expected_shards_in_slots: Vec> = vec![ + vec![0], vec![0], // Each line is 2 slots.. + vec![1], vec![1], + vec![2], vec![2], + vec![3], vec![3], + vec![4], vec![4], + vec![5], vec![5], + vec![6], vec![6], + vec![7], vec![7], + vec![8], vec![8], + vec![9], vec![9], + ]; + // assert!(compare_shards_in_slots(&cycle, &expected_shards_in_slots)); + assert_eq!(expected_shards_in_slots, shards_in_slots, "Shard assignment incorrect.") + } } From b4ca8cbde83c17e8ad6eed3b3eda29d6c4d79355 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 31 Aug 2018 02:20:10 +1000 Subject: [PATCH 13/13] Remove get_new_shuffling I am not sure it is designed correctly --- lighthouse/state/transition/mod.rs | 5 - .../state/transition/validator_allocation.rs | 224 ------------------ 2 files changed, 229 deletions(-) delete mode 100644 lighthouse/state/transition/validator_allocation.rs diff --git a/lighthouse/state/transition/mod.rs b/lighthouse/state/transition/mod.rs index a79e92d06..dd8967e65 100644 --- a/lighthouse/state/transition/mod.rs +++ b/lighthouse/state/transition/mod.rs @@ -1,14 +1,9 @@ use super::super::utils::types::Hash256; -use super::chain_config::ChainConfig; -use super::shard_and_committee::ShardAndCommittee; -use super::validator_record::ValidatorRecord; mod attestation_parent_hashes; mod shuffling; -mod validator_allocation; pub use self::attestation_parent_hashes::attestation_parent_hashes; -pub use self::validator_allocation::get_new_shuffling; pub use self::shuffling::shuffle; #[derive(Debug)] diff --git a/lighthouse/state/transition/validator_allocation.rs b/lighthouse/state/transition/validator_allocation.rs deleted file mode 100644 index f425eded6..000000000 --- a/lighthouse/state/transition/validator_allocation.rs +++ /dev/null @@ -1,224 +0,0 @@ -use super::shuffle; -use super::ChainConfig; -use super::TransitionError; -use super::ValidatorRecord; -use super::ShardAndCommittee; - -pub fn get_new_shuffling( - seed: &[u8], - validators: &Vec, - dynasty: &u64, - crosslinking_shard_start: &u16, - config: &ChainConfig) - -> Result -{ - let shuffled_validator_indices = { - let mut validator_indices = active_validator_indicies(dynasty, validators); - match shuffle(seed, validator_indices) { - Ok(shuffled) => shuffled, - _ => return Err(TransitionError::InvalidInput( - String::from("Shuffle list length exceed."))) - } - }; - let shard_indices = (0_usize..config.shard_count as usize).into_iter().collect(); - let crosslinking_shard_start = *crosslinking_shard_start as usize; - let cycle_length = config.cycle_length as usize; - let min_committee_size = config.min_committee_size as usize; - generate_cycle( - &shuffled_validator_indices, - &shard_indices, - &crosslinking_shard_start, - &cycle_length, - &min_committee_size) -} - -/// Produce a vector of validators indicies where those -/// validators start and end dynasties are within the supplied -/// `dynasty`. -fn active_validator_indicies( - dynasty: &u64, - validators: &Vec) - -> Vec -{ - validators.iter() - .enumerate() - .filter_map(|(i, validator)| { - if (validator.start_dynasty >= *dynasty) & - (validator.end_dynasty < *dynasty) - { - Some(i) - } else { - None - } - }) - .collect() -} - -type DelegatedSlot = Vec; -type DelegatedCycle = Vec; - -fn generate_cycle( - validator_indices: &Vec, - shard_indices: &Vec, - crosslinking_shard_start: &usize, - cycle_length: &usize, - min_committee_size: &usize) - -> Result -{ - let validator_count = validator_indices.len(); - let shard_count = shard_indices.len(); - - let (committees_per_slot, slots_per_committee) = { - if validator_count >= cycle_length * min_committee_size { - let committees_per_slot = validator_count / cycle_length / (min_committee_size * 2) + 1; - let slots_per_committee = 1; - (committees_per_slot, slots_per_committee) - } else { - let committees_per_slot = 1; - let mut slots_per_committee = 1; - while (validator_count * slots_per_committee < cycle_length * min_committee_size) & - (slots_per_committee < *cycle_length) { - slots_per_committee = slots_per_committee * 2; - } - (committees_per_slot, slots_per_committee) - } - }; - - let cycle = validator_indices - .chunks(validator_indices.len() / *cycle_length) - .enumerate() - .map(|(i, slot_indices)| { - let shard_id_start = crosslinking_shard_start + i * committees_per_slot / slots_per_committee; - return slot_indices - .chunks(slot_indices.len() / committees_per_slot) - .enumerate() - .map(|(j, shard_indices)| { - return ShardAndCommittee{ - shard_id: ((shard_id_start + j) % shard_count) as u16, - committee: shard_indices.to_vec(), - } - }) - .collect() - }) - .collect(); - - Ok(cycle) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn generate_cycle_helper( - validator_count: &usize, - shard_count: &usize, - crosslinking_shard_start: &usize, - cycle_length: &usize, - min_committee_size: &usize) - -> (Vec, Vec, Result) - { - let validator_indices = (0_usize..*validator_count).into_iter().collect(); - let shard_indices = (0_usize..*shard_count).into_iter().collect(); - let result = generate_cycle( - &validator_indices, - &shard_indices, - &crosslinking_shard_start, - &cycle_length, - &min_committee_size); - (validator_indices, shard_indices, result) - } - - #[allow(dead_code)] - fn print_cycle(cycle: &DelegatedCycle) { - cycle.iter() - .enumerate() - .for_each(|(i, slot)| { - println!("slot {:?}", &i); - slot.iter() - .enumerate() - .for_each(|(i, sac)| { - println!("#{:?}\tshard_id={}\tcommittee.len()={}", - &i, &sac.shard_id, &sac.committee.len()) - }) - }); - } - - fn flatten_validators(cycle: &DelegatedCycle) - -> Vec - { - let mut flattened = vec![]; - for slot in cycle.iter() { - for sac in slot.iter() { - for validator in sac.committee.iter() { - flattened.push(*validator); - } - } - } - flattened - } - - fn flatten_and_dedup_shards(cycle: &DelegatedCycle) - -> Vec - { - let mut flattened = vec![]; - for slot in cycle.iter() { - for sac in slot.iter() { - flattened.push(sac.shard_id as usize); - } - } - flattened.dedup(); - flattened - } - - fn flatten_shards_in_slots(cycle: &DelegatedCycle) - -> Vec> - { - let mut shards_in_slots: Vec> = vec![]; - for slot in cycle.iter() { - let mut shards: Vec = vec![]; - for sac in slot.iter() { - shards.push(sac.shard_id as usize); - } - shards_in_slots.push(shards); - } - shards_in_slots - } - - - #[test] - fn test_generate_cycle() { - let validator_count: usize = 100; - let shard_count: usize = 10; - let crosslinking_shard_start: usize = 0; - let cycle_length: usize = 20; - let min_committee_size: usize = 10; - let (validators, shards, result) = generate_cycle_helper( - &validator_count, - &shard_count, - &crosslinking_shard_start, - &cycle_length, - &min_committee_size); - let cycle = result.unwrap(); - - let assigned_validators = flatten_validators(&cycle); - let assigned_shards = flatten_and_dedup_shards(&cycle); - let shards_in_slots = flatten_shards_in_slots(&cycle); - assert_eq!(assigned_validators, validators, "Validator assignment incorrect"); - assert_eq!(assigned_shards, shards, "Shard assignment incorrect"); - - let expected_shards_in_slots: Vec> = vec![ - vec![0], vec![0], // Each line is 2 slots.. - vec![1], vec![1], - vec![2], vec![2], - vec![3], vec![3], - vec![4], vec![4], - vec![5], vec![5], - vec![6], vec![6], - vec![7], vec![7], - vec![8], vec![8], - vec![9], vec![9], - ]; - // assert!(compare_shards_in_slots(&cycle, &expected_shards_in_slots)); - assert_eq!(expected_shards_in_slots, shards_in_slots, "Shard assignment incorrect.") - } -}