From c41b743d2d38ab5b57294b8ccea7b5fb1c89257a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 14 Feb 2019 14:57:29 +1100 Subject: [PATCH 01/30] Rename `vec_shuffle` to `fisher_yates_shuffle` We're adding another shuffler so it's nice to be specific --- Cargo.toml | 2 +- eth2/types/Cargo.toml | 2 +- eth2/types/src/beacon_state.rs | 2 +- eth2/utils/{vec_shuffle => fisher_yates_shuffle}/Cargo.toml | 2 +- eth2/utils/{vec_shuffle => fisher_yates_shuffle}/src/lib.rs | 0 eth2/utils/{vec_shuffle => fisher_yates_shuffle}/src/rng.rs | 0 .../src/specs/shuffle_test_vectors.yaml | 0 7 files changed, 4 insertions(+), 4 deletions(-) rename eth2/utils/{vec_shuffle => fisher_yates_shuffle}/Cargo.toml (85%) rename eth2/utils/{vec_shuffle => fisher_yates_shuffle}/src/lib.rs (100%) rename eth2/utils/{vec_shuffle => fisher_yates_shuffle}/src/rng.rs (100%) rename eth2/utils/{vec_shuffle => fisher_yates_shuffle}/src/specs/shuffle_test_vectors.yaml (100%) diff --git a/Cargo.toml b/Cargo.toml index a2c464366..7d545c885 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ members = [ "eth2/utils/honey-badger-split", "eth2/utils/slot_clock", "eth2/utils/ssz", - "eth2/utils/vec_shuffle", + "eth2/utils/fisher_yates_shuffle", "beacon_node", "beacon_node/db", "beacon_node/beacon_chain", diff --git a/eth2/types/Cargo.toml b/eth2/types/Cargo.toml index 24aabf148..9ae5d0090 100644 --- a/eth2/types/Cargo.toml +++ b/eth2/types/Cargo.toml @@ -18,4 +18,4 @@ serde_derive = "1.0" serde_json = "1.0" slog = "^2.2.3" ssz = { path = "../utils/ssz" } -vec_shuffle = { path = "../utils/vec_shuffle" } +fisher_yates_shuffle = { path = "../utils/fisher_yates_shuffle" } diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index ed53bfea9..a0f8c5e2c 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -5,12 +5,12 @@ use crate::{ PendingAttestation, PublicKey, Signature, Slot, Validator, }; use bls::verify_proof_of_possession; +use fisher_yates_shuffle::shuffle; use honey_badger_split::SplitExt; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; use std::ops::Range; -use vec_shuffle::shuffle; pub enum Error { InsufficientValidators, diff --git a/eth2/utils/vec_shuffle/Cargo.toml b/eth2/utils/fisher_yates_shuffle/Cargo.toml similarity index 85% rename from eth2/utils/vec_shuffle/Cargo.toml rename to eth2/utils/fisher_yates_shuffle/Cargo.toml index aaeb50074..7d33c2e91 100644 --- a/eth2/utils/vec_shuffle/Cargo.toml +++ b/eth2/utils/fisher_yates_shuffle/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "vec_shuffle" +name = "fisher_yates_shuffle" version = "0.1.0" authors = ["Paul Hauner "] edition = "2018" diff --git a/eth2/utils/vec_shuffle/src/lib.rs b/eth2/utils/fisher_yates_shuffle/src/lib.rs similarity index 100% rename from eth2/utils/vec_shuffle/src/lib.rs rename to eth2/utils/fisher_yates_shuffle/src/lib.rs diff --git a/eth2/utils/vec_shuffle/src/rng.rs b/eth2/utils/fisher_yates_shuffle/src/rng.rs similarity index 100% rename from eth2/utils/vec_shuffle/src/rng.rs rename to eth2/utils/fisher_yates_shuffle/src/rng.rs diff --git a/eth2/utils/vec_shuffle/src/specs/shuffle_test_vectors.yaml b/eth2/utils/fisher_yates_shuffle/src/specs/shuffle_test_vectors.yaml similarity index 100% rename from eth2/utils/vec_shuffle/src/specs/shuffle_test_vectors.yaml rename to eth2/utils/fisher_yates_shuffle/src/specs/shuffle_test_vectors.yaml From 7a03ee79aaab7907429f9eb962a1236995759606 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 14 Feb 2019 16:53:53 +1100 Subject: [PATCH 02/30] Update SlowLMDGhost to use SlotHeight. --- eth2/fork_choice/src/optimised_lmd_ghost.rs | 16 +++++++--------- eth2/fork_choice/src/slow_lmd_ghost.rs | 4 +--- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/eth2/fork_choice/src/optimised_lmd_ghost.rs b/eth2/fork_choice/src/optimised_lmd_ghost.rs index 7104834cb..40dc029be 100644 --- a/eth2/fork_choice/src/optimised_lmd_ghost.rs +++ b/eth2/fork_choice/src/optimised_lmd_ghost.rs @@ -30,10 +30,8 @@ use fast_math::log2_raw; use std::collections::HashMap; use std::sync::Arc; use types::{ - readers::BeaconBlockReader, - slot_epoch_height::{Height, Slot}, - validator_registry::get_active_validator_indices, - BeaconBlock, Hash256, + readers::BeaconBlockReader, slot_epoch::Slot, slot_height::SlotHeight, + validator_registry::get_active_validator_indices, BeaconBlock, Hash256, }; //TODO: Pruning - Children @@ -77,7 +75,7 @@ pub struct OptimisedLMDGhost { block_store: Arc>, /// State storage access. state_store: Arc>, - max_known_height: Height, + max_known_height: SlotHeight, } impl OptimisedLMDGhost @@ -93,7 +91,7 @@ where ancestors: vec![HashMap::new(); 16], latest_attestation_targets: HashMap::new(), children: HashMap::new(), - max_known_height: Height::new(0), + max_known_height: SlotHeight::new(0), block_store, state_store, } @@ -118,7 +116,7 @@ where .ok_or_else(|| ForkChoiceError::MissingBeaconState(*state_root))?; let active_validator_indices = get_active_validator_indices( - ¤t_state.validator_registry, + ¤t_state.validator_registry[..], block_slot.epoch(EPOCH_LENGTH), ); @@ -137,7 +135,7 @@ where } /// Gets the ancestor at a given height `at_height` of a block specified by `block_hash`. - fn get_ancestor(&mut self, block_hash: Hash256, at_height: Height) -> Option { + fn get_ancestor(&mut self, block_hash: Hash256, at_height: SlotHeight) -> Option { // return None if we can't get the block from the db. let block_height = { let block_slot = self @@ -186,7 +184,7 @@ where fn get_clear_winner( &mut self, latest_votes: &HashMap, - block_height: Height, + block_height: SlotHeight, ) -> Option { // map of vote counts for every hash at this height let mut current_votes: HashMap = HashMap::new(); diff --git a/eth2/fork_choice/src/slow_lmd_ghost.rs b/eth2/fork_choice/src/slow_lmd_ghost.rs index e0e347cef..609d28ab2 100644 --- a/eth2/fork_choice/src/slow_lmd_ghost.rs +++ b/eth2/fork_choice/src/slow_lmd_ghost.rs @@ -28,9 +28,7 @@ use db::{ use std::collections::HashMap; use std::sync::Arc; use types::{ - readers::{BeaconBlockReader, BeaconStateReader}, - slot_epoch_height::Slot, - validator_registry::get_active_validator_indices, + readers::BeaconBlockReader, slot_epoch::Slot, validator_registry::get_active_validator_indices, BeaconBlock, Hash256, }; From 7b39dad232f97b2d69f0189602cf4d30c948373d Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 14 Feb 2019 16:56:12 +1100 Subject: [PATCH 03/30] Removes protolambda as an option of fork-choice. --- eth2/fork_choice/src/lib.rs | 7 ++++--- eth2/fork_choice/src/protolambda_lmd_ghost.rs | 0 2 files changed, 4 insertions(+), 3 deletions(-) delete mode 100644 eth2/fork_choice/src/protolambda_lmd_ghost.rs diff --git a/eth2/fork_choice/src/lib.rs b/eth2/fork_choice/src/lib.rs index f79f7e8c1..820a33ec1 100644 --- a/eth2/fork_choice/src/lib.rs +++ b/eth2/fork_choice/src/lib.rs @@ -44,7 +44,7 @@ extern crate types; pub mod longest_chain; pub mod optimised_lmd_ghost; -pub mod protolambda_lmd_ghost; +//pub mod protolambda_lmd_ghost; pub mod slow_lmd_ghost; use db::stores::BeaconBlockAtSlotError; @@ -113,6 +113,7 @@ pub enum ForkChoiceAlgorithms { SlowLMDGhost, /// An optimised version of LMD-GHOST by Vitalik. OptimisedLMDGhost, - /// An optimised version of LMD-GHOST by Protolambda. - ProtoLMDGhost, + // Protolambda currently not implemented + // /// An optimised version of LMD-GHOST by Protolambda. + //ProtoLMDGhost, } diff --git a/eth2/fork_choice/src/protolambda_lmd_ghost.rs b/eth2/fork_choice/src/protolambda_lmd_ghost.rs deleted file mode 100644 index e69de29bb..000000000 From fe13d98469d9abff5479fd66ac06cb8862dd37bd Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 14 Feb 2019 18:09:09 +1100 Subject: [PATCH 04/30] Updates longest chain to match new fork-choice structure. --- eth2/fork_choice/src/lib.rs | 1 + eth2/fork_choice/src/longest_chain.rs | 161 ++++++++++++++------------ 2 files changed, 86 insertions(+), 76 deletions(-) diff --git a/eth2/fork_choice/src/lib.rs b/eth2/fork_choice/src/lib.rs index 820a33ec1..a6e4ea422 100644 --- a/eth2/fork_choice/src/lib.rs +++ b/eth2/fork_choice/src/lib.rs @@ -83,6 +83,7 @@ pub enum ForkChoiceError { CannotFindBestChild, ChildrenNotFound, StorageError(String), + HeadNotFound, } impl From for ForkChoiceError { diff --git a/eth2/fork_choice/src/longest_chain.rs b/eth2/fork_choice/src/longest_chain.rs index 277d6b950..c25cd43a1 100644 --- a/eth2/fork_choice/src/longest_chain.rs +++ b/eth2/fork_choice/src/longest_chain.rs @@ -1,93 +1,102 @@ -use db::stores::BeaconBlockStore; -use db::{ClientDB, DBError}; -use ssz::{Decodable, DecodeError}; +use crate::{ForkChoice, ForkChoiceError}; +use db::{stores::BeaconBlockStore, ClientDB}; use std::sync::Arc; use types::{BeaconBlock, Hash256, Slot}; -pub enum ForkChoiceError { - BadSszInDatabase, - MissingBlock, - DBError(String), -} - -pub fn longest_chain( - head_block_hashes: &[Hash256], - block_store: &Arc>, -) -> Result, ForkChoiceError> +pub struct LongestChain where T: ClientDB + Sized, { - let mut head_blocks: Vec<(usize, BeaconBlock)> = vec![]; + /// List of head block hashes + head_block_hashes: Vec, + /// Block storage access. + block_store: Arc>, +} - /* - * Load all the head_block hashes from the DB as SszBeaconBlocks. - */ - for (index, block_hash) in head_block_hashes.iter().enumerate() { - let ssz = block_store - .get(&block_hash)? - .ok_or(ForkChoiceError::MissingBlock)?; - let (block, _) = BeaconBlock::ssz_decode(&ssz, 0)?; - head_blocks.push((index, block)); - } - - /* - * Loop through all the head blocks and find the highest slot. - */ - let highest_slot: Option = None; - for (_, block) in &head_blocks { - let slot = block.slot; - - match highest_slot { - None => Some(slot), - Some(winning_slot) => { - if slot > winning_slot { - Some(slot) - } else { - Some(winning_slot) - } - } - }; - } - - /* - * Loop through all the highest blocks and sort them by highest hash. - * - * Ultimately, the index of the head_block hash with the highest slot and highest block - * hash will be the winner. - */ - match highest_slot { - None => Ok(None), - Some(highest_slot) => { - let mut highest_blocks = vec![]; - for (index, block) in head_blocks { - if block.slot == highest_slot { - highest_blocks.push((index, block)) - } - } - - highest_blocks.sort_by(|a, b| head_block_hashes[a.0].cmp(&head_block_hashes[b.0])); - let (index, _) = highest_blocks[0]; - Ok(Some(index)) +impl LongestChain +where + T: ClientDB + Sized, +{ + pub fn new(block_store: Arc>) -> Self { + LongestChain { + head_block_hashes: Vec::new(), + block_store, } } } -impl From for ForkChoiceError { - fn from(_: DecodeError) -> Self { - ForkChoiceError::BadSszInDatabase +impl ForkChoice for LongestChain { + fn add_block( + &mut self, + block: &BeaconBlock, + block_hash: &Hash256, + ) -> Result<(), ForkChoiceError> { + // add the block hash to head_block_hashes removing the parent if it exists + self.head_block_hashes + .retain(|hash| *hash != block.parent_root); + self.head_block_hashes.push(*block_hash); + Ok(()) } -} -impl From for ForkChoiceError { - fn from(e: DBError) -> Self { - ForkChoiceError::DBError(e.message) + fn add_attestation(&mut self, _: u64, _: &Hash256) -> Result<(), ForkChoiceError> { + // do nothing + Ok(()) } -} -#[cfg(test)] -mod tests { - #[test] - fn test_naive_fork_choice() { - assert_eq!(2 + 2, 4); + fn find_head(&mut self, _: &Hash256) -> Result { + let mut head_blocks: Vec<(usize, BeaconBlock)> = vec![]; + /* + * Load all the head_block hashes from the DB as SszBeaconBlocks. + */ + for (index, block_hash) in self.head_block_hashes.iter().enumerate() { + let block = self + .block_store + .get_deserialized(&block_hash)? + .ok_or(ForkChoiceError::MissingBeaconBlock(*block_hash))?; + head_blocks.push((index, block)); + } + + /* + * Loop through all the head blocks and find the highest slot. + */ + let highest_slot = head_blocks + .iter() + .fold(Slot::from(0u64), |highest, (_, block)| { + std::cmp::max(block.slot, highest) + }); + + // if we find no blocks, return Error + if highest_slot == 0 { + return Err(ForkChoiceError::HeadNotFound); + } + + /* + * Loop through all the highest blocks and sort them by highest hash. + * + * Ultimately, the index of the head_block hash with the highest slot and highest block + * hash will be the winner. + */ + + let head_index: Option = + head_blocks + .iter() + .fold(None, |smallest_index, (index, block)| { + if block.slot == highest_slot { + if smallest_index.is_none() { + return Some(*index); + } + return Some(std::cmp::min( + *index, + smallest_index.expect("Cannot be None"), + )); + } + smallest_index + }); + + if head_index.is_none() { + return Err(ForkChoiceError::HeadNotFound); + } + + Ok(self.head_block_hashes[head_index.unwrap()]) } } From 18e85a3cf8032acf5c7a65fcd86cbaef65f8977e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 14 Feb 2019 18:22:55 +1100 Subject: [PATCH 05/30] Add swap_or_not_shuffle and tests. The implementation is not matching the EF implementation at this point. --- Cargo.toml | 1 + eth2/utils/swap_or_not_shuffle/Cargo.toml | 13 ++ eth2/utils/swap_or_not_shuffle/src/lib.rs | 115 ++++++++++++++++++ .../specs/permutated_index_test_vectors.yaml | 74 +++++++++++ 4 files changed, 203 insertions(+) create mode 100644 eth2/utils/swap_or_not_shuffle/Cargo.toml create mode 100644 eth2/utils/swap_or_not_shuffle/src/lib.rs create mode 100644 eth2/utils/swap_or_not_shuffle/src/specs/permutated_index_test_vectors.yaml diff --git a/Cargo.toml b/Cargo.toml index 7d545c885..f0488b33d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "eth2/utils/honey-badger-split", "eth2/utils/slot_clock", "eth2/utils/ssz", + "eth2/utils/swap_or_not_shuffle", "eth2/utils/fisher_yates_shuffle", "beacon_node", "beacon_node/db", diff --git a/eth2/utils/swap_or_not_shuffle/Cargo.toml b/eth2/utils/swap_or_not_shuffle/Cargo.toml new file mode 100644 index 000000000..1c898f7b1 --- /dev/null +++ b/eth2/utils/swap_or_not_shuffle/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "swap_or_not_shuffle" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" + +[dependencies] +bytes = "0.4" +hashing = { path = "../hashing" } + +[dev-dependencies] +yaml-rust = "0.4.2" +hex = "0.3" diff --git a/eth2/utils/swap_or_not_shuffle/src/lib.rs b/eth2/utils/swap_or_not_shuffle/src/lib.rs new file mode 100644 index 000000000..957372a89 --- /dev/null +++ b/eth2/utils/swap_or_not_shuffle/src/lib.rs @@ -0,0 +1,115 @@ +use bytes::{Buf, BufMut, BytesMut}; +use hashing::hash; +use std::cmp::max; +use std::io::Cursor; + +pub fn get_permutated_index( + index: usize, + list_size: usize, + seed: &[u8], + shuffle_round_count: usize, +) -> usize { + let mut index = index; + for round in 0..shuffle_round_count { + let pivot = bytes_to_int64(&hash_with_round(seed, round)[..]) as usize % list_size; + let flip = (pivot - index) % list_size; + let position = max(index, flip); + let source = hash_with_round_and_position(seed, round, position); + let byte = source[(position % 256) / 8]; + let bit = (byte >> (position % 8)) % 2; + index = if bit == 1 { flip } else { index } + } + index +} + +fn hash_with_round_and_position(seed: &[u8], round: usize, position: usize) -> Vec { + let mut seed = seed.to_vec(); + seed.append(&mut int_to_bytes1(round as u64)); + seed.append(&mut int_to_bytes4(position as u64 / 256)); + hash(&seed[..]) +} + +fn hash_with_round(seed: &[u8], round: usize) -> Vec { + let mut seed = seed.to_vec(); + seed.append(&mut int_to_bytes1(round as u64)); + hash(&seed[..]) +} + +fn int_to_bytes1(int: u64) -> Vec { + let mut bytes = BytesMut::with_capacity(8); + bytes.put_u64_le(int); + vec![bytes[0]] +} + +fn int_to_bytes4(int: u64) -> Vec { + let mut bytes = BytesMut::with_capacity(8); + bytes.put_u64_le(int); + bytes[0..4].to_vec() +} + +fn bytes_to_int64(bytes: &[u8]) -> u64 { + let mut cursor = Cursor::new(bytes); + cursor.get_u64_le() +} + +#[cfg(test)] +mod tests { + use super::*; + use hex; + use std::{fs::File, io::prelude::*, path::PathBuf}; + use yaml_rust::yaml; + + #[test] + fn test_shuffling() { + let mut file = { + let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + file_path_buf.push("src/specs/permutated_index_test_vectors.yaml"); + + File::open(file_path_buf).unwrap() + }; + + let mut yaml_str = String::new(); + + file.read_to_string(&mut yaml_str).unwrap(); + + let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap(); + let doc = &docs[0]; + let test_cases = doc["test_cases"].as_vec().unwrap(); + + for (i, test_case) in test_cases.iter().enumerate() { + let index = test_case["index"].as_i64().unwrap() as usize; + let list_size = test_case["list_size"].as_i64().unwrap() as usize; + let permutated_index = test_case["permutated_index"].as_i64().unwrap() as usize; + let shuffle_round_count = test_case["shuffle_round_count"].as_i64().unwrap() as usize; + let seed_string = test_case["seed"].clone().into_string().unwrap(); + let seed = hex::decode(seed_string.replace("0x", "")).unwrap(); + + println!("case: {}", i); + + assert_eq!( + permutated_index, + get_permutated_index(index, list_size, &seed[..], shuffle_round_count), + "Failure on case #{} index: {}, list_size: {}, round_count: {}, seed: {}", + i, + index, + list_size, + shuffle_round_count, + seed_string, + ); + + /* + let input = test_case["input"].clone().into_vec().unwrap(); + let output = test_case["output"].clone().into_vec().unwrap(); + let seed_bytes = test_case["seed"].as_str().unwrap().as_bytes(); + + let seed = if seed_bytes.len() > 0 { + hash(seed_bytes) + } else { + vec![] + }; + + assert_eq!(shuffle(&seed, input).unwrap(), output); + */ + } + } +} diff --git a/eth2/utils/swap_or_not_shuffle/src/specs/permutated_index_test_vectors.yaml b/eth2/utils/swap_or_not_shuffle/src/specs/permutated_index_test_vectors.yaml new file mode 100644 index 000000000..f753976f7 --- /dev/null +++ b/eth2/utils/swap_or_not_shuffle/src/specs/permutated_index_test_vectors.yaml @@ -0,0 +1,74 @@ +fork: tchaikovsky +summary: Test vectors for list shuffling using `get_permutated_index` +test_suite: permutated_index +title: Permutated Index Tests +version: 1.0 +test_cases: +- {index: 0, list_size: 1024, permutated_index: 216, seed: '0xc0c7f226fbd574a8c63dc26864c27833ea931e7c70b34409ba765f3d2031633d', + shuffle_round_count: 90} +- {index: 1024, list_size: 1024, permutated_index: 433, seed: '0xb20420b2b7b1c64600cbe962544052d0bbe13da403950d198d4f4ea28762953f', + shuffle_round_count: 90} +- {index: 806, list_size: 704, permutated_index: 584, seed: '0x3a4cfce20efb7d7eca50291470043d6e8a2c62956e687571607d3f0e5bd0af3f', + shuffle_round_count: 90} +- {index: 1681, list_size: 2154, permutated_index: 1628, seed: '0xbb99b3ecc0ea15a403456ce708c05ceeeddc0a4205caf072ba06ff9bde03f37e', + shuffle_round_count: 90} +- {index: 3253, list_size: 621, permutated_index: 31, seed: '0x1a51109676d549c1bea3b81edd82df68cc03a97ff58a8970c63ca86dd3b8b8a6', + shuffle_round_count: 90} +- {index: 749, list_size: 3081, permutated_index: 1221, seed: '0x6cddea1279bf4a2725c781ce6aba348d383556e23fcb9e73c23ad33cfb50f4c0', + shuffle_round_count: 90} +- {index: 2415, list_size: 2184, permutated_index: 394, seed: '0xa84b128c2885960e5f1b39822ee5dab30ad1580cdabb175a4b1512cac5566866', + shuffle_round_count: 90} +- {index: 339, list_size: 3728, permutated_index: 427, seed: '0xe22c8444f460c9dcfc34a3c13f211e63c56e9e1187f31a56a4230d8d5bf5e584', + shuffle_round_count: 90} +- {index: 4073, list_size: 2365, permutated_index: 2089, seed: '0x946af91cce976e1346e3970815107154b58b1eff411bfca3342ea0d8282a8630', + shuffle_round_count: 90} +- {index: 3438, list_size: 720, permutated_index: 559, seed: '0x5f3c5270d20810c104b75e25bf89c0066deebc3461937fc0e72ae04ee74f2456', + shuffle_round_count: 90} +- {index: 1394, list_size: 388, permutated_index: 311, seed: '0xdf6fdc34adb35f3fc2880d220e520120a032bbaa0f4bd7a5fcf1c2269de21075', + shuffle_round_count: 90} +- {index: 1604, list_size: 1816, permutated_index: 863, seed: '0xc9b0c76e11f4c3c3c38b447aca5352d93132ad5678da420ca2e69d92588e0fba', + shuffle_round_count: 90} +- {index: 671, list_size: 798, permutated_index: 318, seed: '0x45c31aeb3eb29ccdf3327d0f3dd4592cdfb2fad3703229c6c2e720dc792f9f77', + shuffle_round_count: 90} +- {index: 3161, list_size: 4020, permutated_index: 1590, seed: '0xb2a397fd8ea36dbfcec0d733d0af7ec3a03d789a66231f3bc7cafa5e966e5657', + shuffle_round_count: 90} +- {index: 3414, list_size: 3557, permutated_index: 2643, seed: '0x524e6dee54d1b8b5882ad8e55c18a30462ac02c4bb86c27d26cbe52951395b1a', + shuffle_round_count: 90} +- {index: 2693, list_size: 3414, permutated_index: 2230, seed: '0x3b776c7fc8bdad6030de943c4e3f938202ac553f44381dc67a74aac23523cb45', + shuffle_round_count: 90} +- {index: 1319, list_size: 3413, permutated_index: 2830, seed: '0xd2f7b2d24ebc6bf2d63ef189efccabc4a16bb17cd80e4be4083e61b31931bad6', + shuffle_round_count: 90} +- {index: 637, list_size: 1421, permutated_index: 370, seed: '0xe8bc30a4ce7b26b6897c2221a3358f25fdc1d82baa8089c1f242aa48c6611180', + shuffle_round_count: 90} +- {index: 3101, list_size: 525, permutated_index: 157, seed: '0xe930adeecaf3084b2b85f9b1dfebe34f6300047fda0ab6a746b6b0206febb825', + shuffle_round_count: 90} +- {index: 3596, list_size: 3688, permutated_index: 3621, seed: '0xf88bf52143b20d6c78caf7caf8e7b3453287b210d000b5f57e9834388d4bc2b8', + shuffle_round_count: 90} +- {index: 2866, list_size: 307, permutated_index: 191, seed: '0x83fa10a34b029546c2ebabb8075f0670a78b38e0419aaead5d1cc8f40f58044b', + shuffle_round_count: 90} +- {index: 3307, list_size: 3458, permutated_index: 3099, seed: '0x193c08b580dd95a1355574dbf78665190a6133191e91ab35b1106e8984dfc0df', + shuffle_round_count: 90} +- {index: 399, list_size: 3, permutated_index: 1, seed: '0x4f880b431c2a14f66354bf0192292ffae0bf39b39f12e0540b97591af0a2980d', + shuffle_round_count: 90} +- {index: 1912, list_size: 3028, permutated_index: 1139, seed: '0x33201395d301ee43054417c6056404c586c907dfc5fceb66ebef541d143b00a3', + shuffle_round_count: 90} +- {index: 3902, list_size: 3079, permutated_index: 790, seed: '0xfbf4c5f45eabf289fdcfe0a3aba33a185fb1a4ae2f2b6f78daf61f5d356971e0', + shuffle_round_count: 90} +- {index: 38, list_size: 115, permutated_index: 109, seed: '0x7aca86322db56927d727101e31c93f616f746317d29aa10d88f371592963de92', + shuffle_round_count: 90} +- {index: 3295, list_size: 960, permutated_index: 14, seed: '0xfe274230a112bc68614882645339fda2f134501a042079d620ec65cf8d3fa675', + shuffle_round_count: 90} +- {index: 1535, list_size: 3974, permutated_index: 2568, seed: '0xa848af32d85c6d37c26e61a57e96780fcebc350ad1845e83fe5e4679ac820440', + shuffle_round_count: 90} +- {index: 2623, list_size: 431, permutated_index: 236, seed: '0x49c2391e025ec272c812510cb07c055f6201e84479499326330628888e31a0de', + shuffle_round_count: 90} +- {index: 3164, list_size: 3981, permutated_index: 233, seed: '0x09f413b7a36ec680ee8b19bbb9a39c4e207326155864bc8be5d6dd92f30d8556', + shuffle_round_count: 90} +- {index: 1826, list_size: 2643, permutated_index: 1395, seed: '0x797db811486e7a213e0145d6c946e5121aa6a8f761d1647d093fb976f2497361', + shuffle_round_count: 90} +- {index: 290, list_size: 3570, permutated_index: 361, seed: '0xa6dc019009eda2e48bbeb4b7c56d4aa5da7d5f8721b3a79b35beacbe48c662d6', + shuffle_round_count: 90} +- {index: 3192, list_size: 67, permutated_index: 61, seed: '0x040024c55ab879e5f61521013c5f45eb3b705c02c53c9c6ddf259716ff02e49a', + shuffle_round_count: 90} +- {index: 2984, list_size: 822, permutated_index: 57, seed: '0xe4ad255d7e90dbefdbc991adf603e5a4a5a4c57c5705ec697a74e6c7161191b1', + shuffle_round_count: 90} From c98a9fdf3e2b8093baf70a0327b7326318abe732 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 14 Feb 2019 23:55:05 +1100 Subject: [PATCH 06/30] Update test vectors, align to test vectors. --- eth2/utils/swap_or_not_shuffle/src/lib.rs | 42 +++++---- .../specs/permutated_index_test_vectors.yaml | 74 ---------------- .../specs/test_vector_permutated_index.yml | 86 +++++++++++++++++++ 3 files changed, 106 insertions(+), 96 deletions(-) delete mode 100644 eth2/utils/swap_or_not_shuffle/src/specs/permutated_index_test_vectors.yaml create mode 100644 eth2/utils/swap_or_not_shuffle/src/specs/test_vector_permutated_index.yml diff --git a/eth2/utils/swap_or_not_shuffle/src/lib.rs b/eth2/utils/swap_or_not_shuffle/src/lib.rs index 957372a89..88f846e0c 100644 --- a/eth2/utils/swap_or_not_shuffle/src/lib.rs +++ b/eth2/utils/swap_or_not_shuffle/src/lib.rs @@ -8,18 +8,22 @@ pub fn get_permutated_index( list_size: usize, seed: &[u8], shuffle_round_count: usize, -) -> usize { +) -> Option { + if list_size == 0 || index >= list_size { + return None; + } + let mut index = index; for round in 0..shuffle_round_count { let pivot = bytes_to_int64(&hash_with_round(seed, round)[..]) as usize % list_size; - let flip = (pivot - index) % list_size; + let flip = (pivot + list_size - index) % list_size; let position = max(index, flip); let source = hash_with_round_and_position(seed, round, position); let byte = source[(position % 256) / 8]; let bit = (byte >> (position % 8)) % 2; index = if bit == 1 { flip } else { index } } - index + Some(index) } fn hash_with_round_and_position(seed: &[u8], round: usize, position: usize) -> Vec { @@ -60,10 +64,20 @@ mod tests { use yaml_rust::yaml; #[test] - fn test_shuffling() { + fn returns_none_for_zero_length_list() { + assert_eq!(None, get_permutated_index(100, 0, &[42, 42], 90)); + } + + #[test] + fn test_vectors() { + /* + * Test vectors are generated here: + * + * https://github.com/ethereum/eth2.0-test-generators + */ let mut file = { let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - file_path_buf.push("src/specs/permutated_index_test_vectors.yaml"); + file_path_buf.push("src/specs/test_vector_permutated_index.yml"); File::open(file_path_buf).unwrap() }; @@ -84,10 +98,8 @@ mod tests { let seed_string = test_case["seed"].clone().into_string().unwrap(); let seed = hex::decode(seed_string.replace("0x", "")).unwrap(); - println!("case: {}", i); - assert_eq!( - permutated_index, + Some(permutated_index), get_permutated_index(index, list_size, &seed[..], shuffle_round_count), "Failure on case #{} index: {}, list_size: {}, round_count: {}, seed: {}", i, @@ -96,20 +108,6 @@ mod tests { shuffle_round_count, seed_string, ); - - /* - let input = test_case["input"].clone().into_vec().unwrap(); - let output = test_case["output"].clone().into_vec().unwrap(); - let seed_bytes = test_case["seed"].as_str().unwrap().as_bytes(); - - let seed = if seed_bytes.len() > 0 { - hash(seed_bytes) - } else { - vec![] - }; - - assert_eq!(shuffle(&seed, input).unwrap(), output); - */ } } } diff --git a/eth2/utils/swap_or_not_shuffle/src/specs/permutated_index_test_vectors.yaml b/eth2/utils/swap_or_not_shuffle/src/specs/permutated_index_test_vectors.yaml deleted file mode 100644 index f753976f7..000000000 --- a/eth2/utils/swap_or_not_shuffle/src/specs/permutated_index_test_vectors.yaml +++ /dev/null @@ -1,74 +0,0 @@ -fork: tchaikovsky -summary: Test vectors for list shuffling using `get_permutated_index` -test_suite: permutated_index -title: Permutated Index Tests -version: 1.0 -test_cases: -- {index: 0, list_size: 1024, permutated_index: 216, seed: '0xc0c7f226fbd574a8c63dc26864c27833ea931e7c70b34409ba765f3d2031633d', - shuffle_round_count: 90} -- {index: 1024, list_size: 1024, permutated_index: 433, seed: '0xb20420b2b7b1c64600cbe962544052d0bbe13da403950d198d4f4ea28762953f', - shuffle_round_count: 90} -- {index: 806, list_size: 704, permutated_index: 584, seed: '0x3a4cfce20efb7d7eca50291470043d6e8a2c62956e687571607d3f0e5bd0af3f', - shuffle_round_count: 90} -- {index: 1681, list_size: 2154, permutated_index: 1628, seed: '0xbb99b3ecc0ea15a403456ce708c05ceeeddc0a4205caf072ba06ff9bde03f37e', - shuffle_round_count: 90} -- {index: 3253, list_size: 621, permutated_index: 31, seed: '0x1a51109676d549c1bea3b81edd82df68cc03a97ff58a8970c63ca86dd3b8b8a6', - shuffle_round_count: 90} -- {index: 749, list_size: 3081, permutated_index: 1221, seed: '0x6cddea1279bf4a2725c781ce6aba348d383556e23fcb9e73c23ad33cfb50f4c0', - shuffle_round_count: 90} -- {index: 2415, list_size: 2184, permutated_index: 394, seed: '0xa84b128c2885960e5f1b39822ee5dab30ad1580cdabb175a4b1512cac5566866', - shuffle_round_count: 90} -- {index: 339, list_size: 3728, permutated_index: 427, seed: '0xe22c8444f460c9dcfc34a3c13f211e63c56e9e1187f31a56a4230d8d5bf5e584', - shuffle_round_count: 90} -- {index: 4073, list_size: 2365, permutated_index: 2089, seed: '0x946af91cce976e1346e3970815107154b58b1eff411bfca3342ea0d8282a8630', - shuffle_round_count: 90} -- {index: 3438, list_size: 720, permutated_index: 559, seed: '0x5f3c5270d20810c104b75e25bf89c0066deebc3461937fc0e72ae04ee74f2456', - shuffle_round_count: 90} -- {index: 1394, list_size: 388, permutated_index: 311, seed: '0xdf6fdc34adb35f3fc2880d220e520120a032bbaa0f4bd7a5fcf1c2269de21075', - shuffle_round_count: 90} -- {index: 1604, list_size: 1816, permutated_index: 863, seed: '0xc9b0c76e11f4c3c3c38b447aca5352d93132ad5678da420ca2e69d92588e0fba', - shuffle_round_count: 90} -- {index: 671, list_size: 798, permutated_index: 318, seed: '0x45c31aeb3eb29ccdf3327d0f3dd4592cdfb2fad3703229c6c2e720dc792f9f77', - shuffle_round_count: 90} -- {index: 3161, list_size: 4020, permutated_index: 1590, seed: '0xb2a397fd8ea36dbfcec0d733d0af7ec3a03d789a66231f3bc7cafa5e966e5657', - shuffle_round_count: 90} -- {index: 3414, list_size: 3557, permutated_index: 2643, seed: '0x524e6dee54d1b8b5882ad8e55c18a30462ac02c4bb86c27d26cbe52951395b1a', - shuffle_round_count: 90} -- {index: 2693, list_size: 3414, permutated_index: 2230, seed: '0x3b776c7fc8bdad6030de943c4e3f938202ac553f44381dc67a74aac23523cb45', - shuffle_round_count: 90} -- {index: 1319, list_size: 3413, permutated_index: 2830, seed: '0xd2f7b2d24ebc6bf2d63ef189efccabc4a16bb17cd80e4be4083e61b31931bad6', - shuffle_round_count: 90} -- {index: 637, list_size: 1421, permutated_index: 370, seed: '0xe8bc30a4ce7b26b6897c2221a3358f25fdc1d82baa8089c1f242aa48c6611180', - shuffle_round_count: 90} -- {index: 3101, list_size: 525, permutated_index: 157, seed: '0xe930adeecaf3084b2b85f9b1dfebe34f6300047fda0ab6a746b6b0206febb825', - shuffle_round_count: 90} -- {index: 3596, list_size: 3688, permutated_index: 3621, seed: '0xf88bf52143b20d6c78caf7caf8e7b3453287b210d000b5f57e9834388d4bc2b8', - shuffle_round_count: 90} -- {index: 2866, list_size: 307, permutated_index: 191, seed: '0x83fa10a34b029546c2ebabb8075f0670a78b38e0419aaead5d1cc8f40f58044b', - shuffle_round_count: 90} -- {index: 3307, list_size: 3458, permutated_index: 3099, seed: '0x193c08b580dd95a1355574dbf78665190a6133191e91ab35b1106e8984dfc0df', - shuffle_round_count: 90} -- {index: 399, list_size: 3, permutated_index: 1, seed: '0x4f880b431c2a14f66354bf0192292ffae0bf39b39f12e0540b97591af0a2980d', - shuffle_round_count: 90} -- {index: 1912, list_size: 3028, permutated_index: 1139, seed: '0x33201395d301ee43054417c6056404c586c907dfc5fceb66ebef541d143b00a3', - shuffle_round_count: 90} -- {index: 3902, list_size: 3079, permutated_index: 790, seed: '0xfbf4c5f45eabf289fdcfe0a3aba33a185fb1a4ae2f2b6f78daf61f5d356971e0', - shuffle_round_count: 90} -- {index: 38, list_size: 115, permutated_index: 109, seed: '0x7aca86322db56927d727101e31c93f616f746317d29aa10d88f371592963de92', - shuffle_round_count: 90} -- {index: 3295, list_size: 960, permutated_index: 14, seed: '0xfe274230a112bc68614882645339fda2f134501a042079d620ec65cf8d3fa675', - shuffle_round_count: 90} -- {index: 1535, list_size: 3974, permutated_index: 2568, seed: '0xa848af32d85c6d37c26e61a57e96780fcebc350ad1845e83fe5e4679ac820440', - shuffle_round_count: 90} -- {index: 2623, list_size: 431, permutated_index: 236, seed: '0x49c2391e025ec272c812510cb07c055f6201e84479499326330628888e31a0de', - shuffle_round_count: 90} -- {index: 3164, list_size: 3981, permutated_index: 233, seed: '0x09f413b7a36ec680ee8b19bbb9a39c4e207326155864bc8be5d6dd92f30d8556', - shuffle_round_count: 90} -- {index: 1826, list_size: 2643, permutated_index: 1395, seed: '0x797db811486e7a213e0145d6c946e5121aa6a8f761d1647d093fb976f2497361', - shuffle_round_count: 90} -- {index: 290, list_size: 3570, permutated_index: 361, seed: '0xa6dc019009eda2e48bbeb4b7c56d4aa5da7d5f8721b3a79b35beacbe48c662d6', - shuffle_round_count: 90} -- {index: 3192, list_size: 67, permutated_index: 61, seed: '0x040024c55ab879e5f61521013c5f45eb3b705c02c53c9c6ddf259716ff02e49a', - shuffle_round_count: 90} -- {index: 2984, list_size: 822, permutated_index: 57, seed: '0xe4ad255d7e90dbefdbc991adf603e5a4a5a4c57c5705ec697a74e6c7161191b1', - shuffle_round_count: 90} diff --git a/eth2/utils/swap_or_not_shuffle/src/specs/test_vector_permutated_index.yml b/eth2/utils/swap_or_not_shuffle/src/specs/test_vector_permutated_index.yml new file mode 100644 index 000000000..61b117fa9 --- /dev/null +++ b/eth2/utils/swap_or_not_shuffle/src/specs/test_vector_permutated_index.yml @@ -0,0 +1,86 @@ +fork: tchaikovsky +summary: Test vectors for list shuffling using `get_permutated_index` +test_suite: permutated_index +title: Permutated Index Tests +version: 1.0 +test_cases: +- {index: 0, list_size: 1, permutated_index: 0, seed: '0xc0c7f226fbd574a8c63dc26864c27833ea931e7c70b34409ba765f3d2031633d', + shuffle_round_count: 90} +- {index: 0, list_size: 2, permutated_index: 0, seed: '0xb20420b2b7b1c64600cbe962544052d0bbe13da403950d198d4f4ea28762953f', + shuffle_round_count: 90} +- {index: 1, list_size: 2, permutated_index: 0, seed: '0x11f1322c3a4cfce20efb7d7eca50291470043d6e8a2c62956e687571607d3f0e', + shuffle_round_count: 90} +- {index: 0, list_size: 3, permutated_index: 2, seed: '0x5bd0af3f74fe6986bb99b3ecc0ea15a403456ce708c05ceeeddc0a4205caf072', + shuffle_round_count: 90} +- {index: 1, list_size: 3, permutated_index: 1, seed: '0xba06ff9bde03f37eddeacb261a51109676d549c1bea3b81edd82df68cc03a97f', + shuffle_round_count: 90} +- {index: 2, list_size: 3, permutated_index: 2, seed: '0xf58a8970c63ca86dd3b8b8a615302ec06cddea1279bf4a2725c781ce6aba348d', + shuffle_round_count: 90} +- {index: 0, list_size: 1024, permutated_index: 1005, seed: '0x383556e23fcb9e73c23ad33cfb50f4c098f49688a84b128c2885960e5f1b3982', + shuffle_round_count: 90} +- {index: 1023, list_size: 1024, permutated_index: 934, seed: '0x2ee5dab30ad1580cdabb175a4b1512cac5566866d65a15e9e22c8444f460c9dc', + shuffle_round_count: 90} +- {index: 3925, list_size: 4040, permutated_index: 32, seed: '0x34a3c13f211e63c56e9e1187f31a56a4230d8d5bf5e584f0e4fe93946af91cce', + shuffle_round_count: 90} +- {index: 885, list_size: 2417, permutated_index: 1822, seed: '0x1346e3970815107154b58b1eff411bfca3342ea0d8282a86304d79d62d5f3c52', + shuffle_round_count: 90} +- {index: 840, list_size: 1805, permutated_index: 808, seed: '0x0810c104b75e25bf89c0066deebc3461937fc0e72ae04ee74f245616c15718df', + shuffle_round_count: 90} +- {index: 881, list_size: 1788, permutated_index: 582, seed: '0x34adb35f3fc2880d220e520120a032bbaa0f4bd7a5fcf1c2269de21075e7a464', + shuffle_round_count: 90} +- {index: 1362, list_size: 1817, permutated_index: 1018, seed: '0xc9b0c76e11f4c3c3c38b447aca5352d93132ad5678da420ca2e69d92588e0fba', + shuffle_round_count: 90} +- {index: 28, list_size: 111, permutated_index: 0, seed: '0x293145c31aeb3eb29ccdf3327d0f3dd4592cdfb2fad3703229c6c2e720dc792f', + shuffle_round_count: 90} +- {index: 959, list_size: 2558, permutated_index: 2094, seed: '0xc9f4c5fbb2a397fd8ea36dbfcec0d733d0af7ec3a03d789a66231f3bc7cafa5e', + shuffle_round_count: 90} +- {index: 887, list_size: 2406, permutated_index: 831, seed: '0x565729e0d5de524e6dee54d1b8b5882ad8e55c18a30462ac02c4bb86c27d26cb', + shuffle_round_count: 90} +- {index: 3526, list_size: 3674, permutated_index: 3531, seed: '0x2951395b1a1bbda8d53b776c7fc8bdad6030de943c4e3f938202ac553f44381d', + shuffle_round_count: 90} +- {index: 978, list_size: 3175, permutated_index: 2257, seed: '0x74aac23523cb45b7ee52d5d2f7b2d24ebc6bf2d63ef189efccabc4a16bb17cd8', + shuffle_round_count: 90} +- {index: 37, list_size: 231, permutated_index: 48, seed: '0xe4083e61b31931bad662392758e8bc30a4ce7b26b6897c2221a3358f25fdc1d8', + shuffle_round_count: 90} +- {index: 340, list_size: 693, permutated_index: 234, seed: '0x8089c1f242aa48c6611180f221c120e930adeecaf3084b2b85f9b1dfebe34f63', + shuffle_round_count: 90} +- {index: 0, list_size: 9, permutated_index: 1, seed: '0x7fda0ab6a746b6b0206febb8259891e0e6f88bf52143b20d6c78caf7caf8e7b3', + shuffle_round_count: 90} +- {index: 200, list_size: 1108, permutated_index: 952, seed: '0x87b210d000b5f57e9834388d4bc2b86ae8b31383fa10a34b029546c2ebabb807', + shuffle_round_count: 90} +- {index: 1408, list_size: 1531, permutated_index: 584, seed: '0x0670a78b38e0419aaead5d1cc8f40f58044b7076ced8193c08b580dd95a13555', + shuffle_round_count: 90} +- {index: 1704, list_size: 1863, permutated_index: 1022, seed: '0xdbf78665190a6133191e91ab35b1106e8984dfc0dfa36018004f880b431c2a14', + shuffle_round_count: 90} +- {index: 793, list_size: 3938, permutated_index: 2607, seed: '0x54bf0192292ffae0bf39b39f12e0540b97591af0a2980d32f277bd33201395d3', + shuffle_round_count: 90} +- {index: 14, list_size: 28, permutated_index: 10, seed: '0x43054417c6056404c586c907dfc5fceb66ebef541d143b00a3b676f3c0fbf4c5', + shuffle_round_count: 90} +- {index: 2909, list_size: 3920, permutated_index: 726, seed: '0x5eabf289fdcfe0a3aba33a185fb1a4ae2f2b6f78daf61f5d356971e0cb270207', + shuffle_round_count: 90} +- {index: 1943, list_size: 1959, permutated_index: 1292, seed: '0xca86322db56927d727101e31c93f616f746317d29aa10d88f371592963de92aa', + shuffle_round_count: 90} +- {index: 1647, list_size: 2094, permutated_index: 1805, seed: '0x3cfe274230a112bc68614882645339fda2f134501a042079d620ec65cf8d3fa6', + shuffle_round_count: 90} +- {index: 1012, list_size: 1877, permutated_index: 216, seed: '0x7b5ff8a848af32d85c6d37c26e61a57e96780fcebc350ad1845e83fe5e4679ac', + shuffle_round_count: 90} +- {index: 35, list_size: 2081, permutated_index: 1458, seed: '0x40691aa31a49c2391e025ec272c812510cb07c055f6201e84479499326330628', + shuffle_round_count: 90} +- {index: 1136, list_size: 2189, permutated_index: 1579, seed: '0x31a0deb2c8c5f809f413b7a36ec680ee8b19bbb9a39c4e207326155864bc8be5', + shuffle_round_count: 90} +- {index: 1775, list_size: 3434, permutated_index: 707, seed: '0x92f30d8556382b72a5797db811486e7a213e0145d6c946e5121aa6a8f761d164', + shuffle_round_count: 90} +- {index: 1109, list_size: 2010, permutated_index: 433, seed: '0x093fb976f2497361897012dfa6dc019009eda2e48bbeb4b7c56d4aa5da7d5f87', + shuffle_round_count: 90} +- {index: 359, list_size: 538, permutated_index: 115, seed: '0xa79b35beacbe48c662d60884c704040024c55ab879e5f61521013c5f45eb3b70', + shuffle_round_count: 90} +- {index: 1259, list_size: 1473, permutated_index: 1351, seed: '0x02c53c9c6ddf259716ff02e49a294eba33e4ad255d7e90dbefdbc991adf603e5', + shuffle_round_count: 90} +- {index: 2087, list_size: 2634, permutated_index: 1497, seed: '0xa5a4c57c5705ec697a74e6c7161191b18f58ca882a0fcc18f68dc3b57a1aa5b6', + shuffle_round_count: 90} +- {index: 2069, list_size: 2511, permutated_index: 1837, seed: '0xe7051ebc07f2e7b4d4b28f48d1e42d7b9dcec31c240ca6e1a0c06139ccfc4b8f', + shuffle_round_count: 90} +- {index: 1660, list_size: 3932, permutated_index: 3046, seed: '0x8687c029ffc443879527a64c31b7acbb38ab6e343779d0b2c6e250046fdb9de8', + shuffle_round_count: 90} +- {index: 379, list_size: 646, permutated_index: 32, seed: '0x17e854f4e80401345e13f72af45b221c9f7a840f6a8c1328ddf9c9ca9a088379', + shuffle_round_count: 90} From 91ba26a351c623014d2c0c34c493c15b14ffdc5b Mon Sep 17 00:00:00 2001 From: Age Manning Date: Fri, 15 Feb 2019 00:21:26 +1100 Subject: [PATCH 07/30] Add test dependencies for fork-choice. --- eth2/fork_choice/Cargo.toml | 4 ++++ eth2/fork_choice/src/lib.rs | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/eth2/fork_choice/Cargo.toml b/eth2/fork_choice/Cargo.toml index 566334c76..82d5785b9 100644 --- a/eth2/fork_choice/Cargo.toml +++ b/eth2/fork_choice/Cargo.toml @@ -10,3 +10,7 @@ ssz = { path = "../utils/ssz" } types = { path = "../types" } fast-math = "0.1.1" byteorder = "1.3.1" + +[dev-dependencies] +yaml-rust = "0.4.2" +bls = { path = "../utils/bls" } diff --git a/eth2/fork_choice/src/lib.rs b/eth2/fork_choice/src/lib.rs index a6e4ea422..e876ed60f 100644 --- a/eth2/fork_choice/src/lib.rs +++ b/eth2/fork_choice/src/lib.rs @@ -51,6 +51,10 @@ use db::stores::BeaconBlockAtSlotError; use db::DBError; use types::{BeaconBlock, Hash256}; +// export the main structs +pub use longest_chain::LongestChain; +pub use optimised_lmd_ghost::OptimisedLMDGhost; + /// Defines the interface for Fork Choices. Each Fork choice will define their own data structures /// which can be built in block processing through the `add_block` and `add_attestation` functions. /// The main fork choice algorithm is specified in `find_head From 8c54e55d995df3f43c5c4ef7447a86d8bd18bf7f Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 15 Feb 2019 00:28:24 +1100 Subject: [PATCH 08/30] Updates to permutated_index - Add comments. - Add more `None` conditions. - Add more tests. --- eth2/utils/swap_or_not_shuffle/src/lib.rs | 25 ++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/eth2/utils/swap_or_not_shuffle/src/lib.rs b/eth2/utils/swap_or_not_shuffle/src/lib.rs index 88f846e0c..3566ac23a 100644 --- a/eth2/utils/swap_or_not_shuffle/src/lib.rs +++ b/eth2/utils/swap_or_not_shuffle/src/lib.rs @@ -3,13 +3,23 @@ use hashing::hash; use std::cmp::max; use std::io::Cursor; +/// Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy. +/// +/// Utilizes 'swap or not' shuffling found in +/// https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf +/// See the 'generalized domain' algorithm on page 3. +/// +/// Returns `None` under any of the following conditions: +/// - `list_size == 0` +/// - `index >= list_size` +/// - `list_size >= usize::max_value() / 2` pub fn get_permutated_index( index: usize, list_size: usize, seed: &[u8], shuffle_round_count: usize, ) -> Option { - if list_size == 0 || index >= list_size { + if list_size == 0 || index >= list_size || list_size >= usize::max_value() / 2 { return None; } @@ -68,6 +78,19 @@ mod tests { assert_eq!(None, get_permutated_index(100, 0, &[42, 42], 90)); } + #[test] + fn returns_none_for_out_of_bounds_index() { + assert_eq!(None, get_permutated_index(100, 100, &[42, 42], 90)); + } + + #[test] + fn returns_none_for_too_large_list() { + assert_eq!( + None, + get_permutated_index(100, usize::max_value() / 2, &[42, 42], 90) + ); + } + #[test] fn test_vectors() { /* From 46673238b601eb157b29db25f187d5b689526ed7 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 15 Feb 2019 11:13:56 +1100 Subject: [PATCH 09/30] Add `eth2_bytes` crate Is passing test vectors --- Cargo.toml | 1 + eth2/utils/eth2_bytes/Cargo.toml | 12 + eth2/utils/eth2_bytes/src/lib.rs | 125 ++++++++++ .../src/specs/test_vector_int_to_bytes.yml | 215 ++++++++++++++++++ 4 files changed, 353 insertions(+) create mode 100644 eth2/utils/eth2_bytes/Cargo.toml create mode 100644 eth2/utils/eth2_bytes/src/lib.rs create mode 100644 eth2/utils/eth2_bytes/src/specs/test_vector_int_to_bytes.yml diff --git a/Cargo.toml b/Cargo.toml index f0488b33d..2dda6578f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "eth2/types", "eth2/utils/bls", "eth2/utils/boolean-bitfield", + "eth2/utils/eth2_bytes", "eth2/utils/hashing", "eth2/utils/honey-badger-split", "eth2/utils/slot_clock", diff --git a/eth2/utils/eth2_bytes/Cargo.toml b/eth2/utils/eth2_bytes/Cargo.toml new file mode 100644 index 000000000..afee169f2 --- /dev/null +++ b/eth2/utils/eth2_bytes/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "eth2_bytes" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" + +[dependencies] +bytes = "0.4" + +[dev-dependencies] +yaml-rust = "0.4.2" +hex = "0.3" diff --git a/eth2/utils/eth2_bytes/src/lib.rs b/eth2/utils/eth2_bytes/src/lib.rs new file mode 100644 index 000000000..537d7b5c8 --- /dev/null +++ b/eth2/utils/eth2_bytes/src/lib.rs @@ -0,0 +1,125 @@ +use bytes::{BufMut, BytesMut}; + +/// Returns `int` as little-endian bytes with a length of 1. +pub fn int_to_bytes1(int: u8) -> Vec { + vec![int] +} + +/// Returns `int` as little-endian bytes with a length of 2. +pub fn int_to_bytes2(int: u16) -> Vec { + let mut bytes = BytesMut::with_capacity(2); + bytes.put_u16_le(int); + bytes.to_vec() +} + +/// Returns `int` as little-endian bytes with a length of 3. +/// +/// An `Option` is returned as Rust does not support a native +/// `u24` type. +/// +/// The Eth 2.0 specification uses `int.to_bytes(2, 'little')`, which throws an error if `int` +/// doesn't fit within 3 bytes. The specification relies upon implicit asserts for some validity +/// conditions, so we ensure the calling function is aware of the error condition as opposed to +/// hiding it with a modulo. +pub fn int_to_bytes3(int: u32) -> Option> { + if int < 2_u32.pow(3 * 8) { + let mut bytes = BytesMut::with_capacity(4); + bytes.put_u32_le(int); + Some(bytes[0..3].to_vec()) + } else { + None + } +} + +/// Returns `int` as little-endian bytes with a length of 4. +pub fn int_to_bytes4(int: u32) -> Vec { + let mut bytes = BytesMut::with_capacity(4); + bytes.put_u32_le(int); + bytes.to_vec() +} + +/// Returns `int` as little-endian bytes with a length of 8. +pub fn int_to_bytes8(int: u64) -> Vec { + let mut bytes = BytesMut::with_capacity(8); + bytes.put_u64_le(int); + bytes.to_vec() +} + +/// Returns `int` as little-endian bytes with a length of 32. +pub fn int_to_bytes32(int: u64) -> Vec { + let mut bytes = BytesMut::with_capacity(32); + bytes.put_u64_le(int); + bytes.resize(32, 0); + bytes.to_vec() +} + +/// Returns `int` as little-endian bytes with a length of 48. +pub fn int_to_bytes48(int: u64) -> Vec { + let mut bytes = BytesMut::with_capacity(48); + bytes.put_u64_le(int); + bytes.resize(48, 0); + bytes.to_vec() +} + +/// Returns `int` as little-endian bytes with a length of 96. +pub fn int_to_bytes96(int: u64) -> Vec { + let mut bytes = BytesMut::with_capacity(96); + bytes.put_u64_le(int); + bytes.resize(96, 0); + bytes.to_vec() +} + +#[cfg(test)] +mod tests { + use super::*; + use hex; + use std::{fs::File, io::prelude::*, path::PathBuf}; + use yaml_rust::yaml; + + #[test] + fn int_to_bytes3_returns_none() { + assert_eq!(int_to_bytes3(2_u32.pow(24)), None); + } + + #[test] + fn test_vectors() { + /* + * Test vectors are generated here: + * + * https://github.com/ethereum/eth2.0-test-generators + */ + let mut file = { + let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + file_path_buf.push("src/specs/test_vector_int_to_bytes.yml"); + + File::open(file_path_buf).unwrap() + }; + + let mut yaml_str = String::new(); + + file.read_to_string(&mut yaml_str).unwrap(); + + let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap(); + let doc = &docs[0]; + let test_cases = doc["test_cases"].as_vec().unwrap(); + + for test_case in test_cases { + let byte_length = test_case["byte_length"].as_i64().unwrap() as u64; + let int = test_case["int"].as_i64().unwrap() as u64; + let bytes_string = test_case["bytes"].clone().into_string().unwrap(); + let bytes = hex::decode(bytes_string.replace("0x", "")).unwrap(); + + match byte_length { + 1 => assert_eq!(int_to_bytes1(int as u8), bytes), + 2 => assert_eq!(int_to_bytes2(int as u16), bytes), + 3 => assert_eq!(int_to_bytes3(int as u32), Some(bytes)), + 4 => assert_eq!(int_to_bytes4(int as u32), bytes), + 8 => assert_eq!(int_to_bytes8(int), bytes), + 32 => assert_eq!(int_to_bytes32(int), bytes), + 48 => assert_eq!(int_to_bytes48(int), bytes), + 96 => assert_eq!(int_to_bytes96(int), bytes), + _ => panic!("Unknown byte length in test vector."), + } + } + } +} diff --git a/eth2/utils/eth2_bytes/src/specs/test_vector_int_to_bytes.yml b/eth2/utils/eth2_bytes/src/specs/test_vector_int_to_bytes.yml new file mode 100644 index 000000000..3b7eaa9c1 --- /dev/null +++ b/eth2/utils/eth2_bytes/src/specs/test_vector_int_to_bytes.yml @@ -0,0 +1,215 @@ +fork: tchaikovsky +summary: Test vectors for the `int_to_bytes[n]` functions.` +test_suite: int_to_bytes +title: int_to_bytes Tests +version: 1.0 +test_cases: +- {byte_length: 1, bytes: '0x00', int: 0} +- {byte_length: 1, bytes: '0x01', int: 1} +- {byte_length: 1, bytes: '0xff', int: 255} +- {byte_length: 1, bytes: '0xc0', int: 192} +- {byte_length: 1, bytes: '0xc7', int: 199} +- {byte_length: 1, bytes: '0xf2', int: 242} +- {byte_length: 1, bytes: '0x26', int: 38} +- {byte_length: 1, bytes: '0xfb', int: 251} +- {byte_length: 1, bytes: '0xd5', int: 213} +- {byte_length: 1, bytes: '0x74', int: 116} +- {byte_length: 1, bytes: '0xa8', int: 168} +- {byte_length: 1, bytes: '0xc6', int: 198} +- {byte_length: 1, bytes: '0x3d', int: 61} +- {byte_length: 1, bytes: '0xc2', int: 194} +- {byte_length: 1, bytes: '0x68', int: 104} +- {byte_length: 1, bytes: '0x64', int: 100} +- {byte_length: 1, bytes: '0xc2', int: 194} +- {byte_length: 1, bytes: '0x78', int: 120} +- {byte_length: 1, bytes: '0x33', int: 51} +- {byte_length: 2, bytes: '0x0000', int: 0} +- {byte_length: 2, bytes: '0x0100', int: 1} +- {byte_length: 2, bytes: '0xffff', int: 65535} +- {byte_length: 2, bytes: '0xedea', int: 60141} +- {byte_length: 2, bytes: '0x2d93', int: 37677} +- {byte_length: 2, bytes: '0x611e', int: 7777} +- {byte_length: 2, bytes: '0x637c', int: 31843} +- {byte_length: 2, bytes: '0xe370', int: 28899} +- {byte_length: 2, bytes: '0x96b3', int: 45974} +- {byte_length: 2, bytes: '0xde44', int: 17630} +- {byte_length: 2, bytes: '0xa009', int: 2464} +- {byte_length: 2, bytes: '0xf6ba', int: 47862} +- {byte_length: 2, bytes: '0xef76', int: 30447} +- {byte_length: 2, bytes: '0x7e5f', int: 24446} +- {byte_length: 2, bytes: '0x393d', int: 15673} +- {byte_length: 2, bytes: '0xc820', int: 8392} +- {byte_length: 2, bytes: '0x9031', int: 12688} +- {byte_length: 2, bytes: '0x3963', int: 25401} +- {byte_length: 2, bytes: '0x033d', int: 15619} +- {byte_length: 3, bytes: '0x000000', int: 0} +- {byte_length: 3, bytes: '0x010000', int: 1} +- {byte_length: 3, bytes: '0xffffff', int: 16777215} +- {byte_length: 3, bytes: '0x1fdfb2', int: 11722527} +- {byte_length: 3, bytes: '0x2a7504', int: 292138} +- {byte_length: 3, bytes: '0x09fb20', int: 2161417} +- {byte_length: 3, bytes: '0xa4a6b2', int: 11708068} +- {byte_length: 3, bytes: '0x17feb7', int: 12058135} +- {byte_length: 3, bytes: '0x3ad0b1', int: 11653178} +- {byte_length: 3, bytes: '0xbc92c6', int: 13013692} +- {byte_length: 3, bytes: '0xb6c046', int: 4636854} +- {byte_length: 3, bytes: '0x937f00', int: 32659} +- {byte_length: 3, bytes: '0x8266cb', int: 13330050} +- {byte_length: 3, bytes: '0x8136e9', int: 15283841} +- {byte_length: 3, bytes: '0xe9e062', int: 6480105} +- {byte_length: 3, bytes: '0x50d054', int: 5558352} +- {byte_length: 3, bytes: '0xb95340', int: 4215737} +- {byte_length: 3, bytes: '0x779f52', int: 5414775} +- {byte_length: 3, bytes: '0x15aed0', int: 13676053} +- {byte_length: 4, bytes: '0x00000000', int: 0} +- {byte_length: 4, bytes: '0x01000000', int: 1} +- {byte_length: 4, bytes: '0xffffffff', int: 4294967295} +- {byte_length: 4, bytes: '0x389cd0ca', int: 3402669112} +- {byte_length: 4, bytes: '0xfb29dc70', int: 1893476859} +- {byte_length: 4, bytes: '0xf5f5c999', int: 2580149749} +- {byte_length: 4, bytes: '0xf4f0b8d1', int: 3518558452} +- {byte_length: 4, bytes: '0x830de883', int: 2213023107} +- {byte_length: 4, bytes: '0xe3b4e843', int: 1139324131} +- {byte_length: 4, bytes: '0x4c9ce594', int: 2498075724} +- {byte_length: 4, bytes: '0xa9826dab', int: 2876080809} +- {byte_length: 4, bytes: '0xc40aecb7', int: 3085699780} +- {byte_length: 4, bytes: '0x55490416', int: 369379669} +- {byte_length: 4, bytes: '0x4f2eedc5', int: 3320655439} +- {byte_length: 4, bytes: '0xdd07257e', int: 2116356061} +- {byte_length: 4, bytes: '0x481a57e9', int: 3914799688} +- {byte_length: 4, bytes: '0x4556a493', int: 2477020741} +- {byte_length: 4, bytes: '0xccb781ed', int: 3984701388} +- {byte_length: 4, bytes: '0x6b994065', int: 1698732395} +- {byte_length: 8, bytes: '0x0000000000000000', int: 0} +- {byte_length: 8, bytes: '0x0100000000000000', int: 1} +- {byte_length: 8, bytes: '0xffffffff00000000', int: 4294967295} +- {byte_length: 8, bytes: '0x77d6e31400000000', int: 350475895} +- {byte_length: 8, bytes: '0xf3e681bf00000000', int: 3212961523} +- {byte_length: 8, bytes: '0x62fa7bd800000000', int: 3632003682} +- {byte_length: 8, bytes: '0x82c67b4500000000', int: 1165739650} +- {byte_length: 8, bytes: '0x52577fba00000000', int: 3128907602} +- {byte_length: 8, bytes: '0x5eac939b00000000', int: 2610146398} +- {byte_length: 8, bytes: '0x12ba143700000000', int: 924105234} +- {byte_length: 8, bytes: '0x1d3b893a00000000', int: 982072093} +- {byte_length: 8, bytes: '0x8262153000000000', int: 806707842} +- {byte_length: 8, bytes: '0xbb9cc58e00000000', int: 2395315387} +- {byte_length: 8, bytes: '0x76fef6d100000000', int: 3522625142} +- {byte_length: 8, bytes: '0x0fc3d35700000000', int: 1473495823} +- {byte_length: 8, bytes: '0xc7f851de00000000', int: 3729914055} +- {byte_length: 8, bytes: '0x3a1e5cb200000000', int: 2992381498} +- {byte_length: 8, bytes: '0x3b748e3400000000', int: 881751099} +- {byte_length: 8, bytes: '0xdc92479600000000', int: 2521273052} +- {byte_length: 32, bytes: '0x0000000000000000000000000000000000000000000000000000000000000000', + int: 0} +- {byte_length: 32, bytes: '0x0100000000000000000000000000000000000000000000000000000000000000', + int: 1} +- {byte_length: 32, bytes: '0xffffffff00000000000000000000000000000000000000000000000000000000', + int: 4294967295} +- {byte_length: 32, bytes: '0x2395ad4c00000000000000000000000000000000000000000000000000000000', + int: 1286444323} +- {byte_length: 32, bytes: '0x38a735b800000000000000000000000000000000000000000000000000000000', + int: 3090523960} +- {byte_length: 32, bytes: '0x5a9416e100000000000000000000000000000000000000000000000000000000', + int: 3776353370} +- {byte_length: 32, bytes: '0x220f757500000000000000000000000000000000000000000000000000000000', + int: 1970605858} +- {byte_length: 32, bytes: '0x65bf635200000000000000000000000000000000000000000000000000000000', + int: 1382268773} +- {byte_length: 32, bytes: '0x033f902200000000000000000000000000000000000000000000000000000000', + int: 579878659} +- {byte_length: 32, bytes: '0x2b2d58ab00000000000000000000000000000000000000000000000000000000', + int: 2874682667} +- {byte_length: 32, bytes: '0x15af31da00000000000000000000000000000000000000000000000000000000', + int: 3660689173} +- {byte_length: 32, bytes: '0xd260642e00000000000000000000000000000000000000000000000000000000', + int: 778330322} +- {byte_length: 32, bytes: '0xcdf8429700000000000000000000000000000000000000000000000000000000', + int: 2537748685} +- {byte_length: 32, bytes: '0xc9304b0500000000000000000000000000000000000000000000000000000000', + int: 88813769} +- {byte_length: 32, bytes: '0xf7b7ba0200000000000000000000000000000000000000000000000000000000', + int: 45791223} +- {byte_length: 32, bytes: '0x1ee262d900000000000000000000000000000000000000000000000000000000', + int: 3647136286} +- {byte_length: 32, bytes: '0xb34b03d300000000000000000000000000000000000000000000000000000000', + int: 3540208563} +- {byte_length: 32, bytes: '0x3d52db4d00000000000000000000000000000000000000000000000000000000', + int: 1306219069} +- {byte_length: 32, bytes: '0xd86db47900000000000000000000000000000000000000000000000000000000', + int: 2041867736} +- {byte_length: 48, bytes: '0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 0} +- {byte_length: 48, bytes: '0x010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 1} +- {byte_length: 48, bytes: '0xffffffff0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 4294967295} +- {byte_length: 48, bytes: '0x61aeae650000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 1705946721} +- {byte_length: 48, bytes: '0xd1c08fac0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 2895102161} +- {byte_length: 48, bytes: '0x6f36b6c90000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 3384161903} +- {byte_length: 48, bytes: '0x102f2f3b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 992947984} +- {byte_length: 48, bytes: '0x0f53f9240000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 620319503} +- {byte_length: 48, bytes: '0x5c1d46b30000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 3007716700} +- {byte_length: 48, bytes: '0x955791510000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 1368479637} +- {byte_length: 48, bytes: '0xf934170f0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 253179129} +- {byte_length: 48, bytes: '0xc1a8b76f0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 1874307265} +- {byte_length: 48, bytes: '0xdf3f62c20000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 3261218783} +- {byte_length: 48, bytes: '0xbd741bc50000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 3306910909} +- {byte_length: 48, bytes: '0xfe5dc5540000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 1422220798} +- {byte_length: 48, bytes: '0x364f10df0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 3742388022} +- {byte_length: 48, bytes: '0x4a3909450000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 1158232394} +- {byte_length: 48, bytes: '0xe04760380000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 945833952} +- {byte_length: 48, bytes: '0x755c78540000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 1417174133} +- {byte_length: 96, bytes: '0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 0} +- {byte_length: 96, bytes: '0x010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 1} +- {byte_length: 96, bytes: '0xffffffff0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 4294967295} +- {byte_length: 96, bytes: '0xa3274ee20000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 3796772771} +- {byte_length: 96, bytes: '0x1658135c0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 1544771606} +- {byte_length: 96, bytes: '0x2af24fb30000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 3008361002} +- {byte_length: 96, bytes: '0x9e6bc40a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 180644766} +- {byte_length: 96, bytes: '0x0745b3c50000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 3316860167} +- {byte_length: 96, bytes: '0xe1b59f830000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 2208282081} +- {byte_length: 96, bytes: '0x985a9e6e0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 1855871640} +- {byte_length: 96, bytes: '0x3d4e3a090000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 154816061} +- {byte_length: 96, bytes: '0x6f5dfb630000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 1677417839} +- {byte_length: 96, bytes: '0x383cdecd0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 3453893688} +- {byte_length: 96, bytes: '0x38f55ceb0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 3948737848} +- {byte_length: 96, bytes: '0xcd746f5d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 1567585485} +- {byte_length: 96, bytes: '0x3d971e910000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 2434701117} +- {byte_length: 96, bytes: '0x3adff0c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 3237011258} +- {byte_length: 96, bytes: '0x5ed40a710000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 1896535134} +- {byte_length: 96, bytes: '0x755d2ed40000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + int: 3559808373} From 73484f04a14ee034f6882200f60dbdf1fd1a11ef Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 15 Feb 2019 11:19:25 +1100 Subject: [PATCH 10/30] Rename `eth2_bytes` to `int_to_bytes` --- Cargo.toml | 2 +- eth2/utils/{eth2_bytes => int_to_bytes}/Cargo.toml | 2 +- eth2/utils/{eth2_bytes => int_to_bytes}/src/lib.rs | 0 .../src/specs/test_vector_int_to_bytes.yml | 0 4 files changed, 2 insertions(+), 2 deletions(-) rename eth2/utils/{eth2_bytes => int_to_bytes}/Cargo.toml (88%) rename eth2/utils/{eth2_bytes => int_to_bytes}/src/lib.rs (100%) rename eth2/utils/{eth2_bytes => int_to_bytes}/src/specs/test_vector_int_to_bytes.yml (100%) diff --git a/Cargo.toml b/Cargo.toml index 2dda6578f..5ab0ba847 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,9 +7,9 @@ members = [ "eth2/types", "eth2/utils/bls", "eth2/utils/boolean-bitfield", - "eth2/utils/eth2_bytes", "eth2/utils/hashing", "eth2/utils/honey-badger-split", + "eth2/utils/int_to_bytes", "eth2/utils/slot_clock", "eth2/utils/ssz", "eth2/utils/swap_or_not_shuffle", diff --git a/eth2/utils/eth2_bytes/Cargo.toml b/eth2/utils/int_to_bytes/Cargo.toml similarity index 88% rename from eth2/utils/eth2_bytes/Cargo.toml rename to eth2/utils/int_to_bytes/Cargo.toml index afee169f2..f7424e032 100644 --- a/eth2/utils/eth2_bytes/Cargo.toml +++ b/eth2/utils/int_to_bytes/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "eth2_bytes" +name = "int_to_bytes" version = "0.1.0" authors = ["Paul Hauner "] edition = "2018" diff --git a/eth2/utils/eth2_bytes/src/lib.rs b/eth2/utils/int_to_bytes/src/lib.rs similarity index 100% rename from eth2/utils/eth2_bytes/src/lib.rs rename to eth2/utils/int_to_bytes/src/lib.rs diff --git a/eth2/utils/eth2_bytes/src/specs/test_vector_int_to_bytes.yml b/eth2/utils/int_to_bytes/src/specs/test_vector_int_to_bytes.yml similarity index 100% rename from eth2/utils/eth2_bytes/src/specs/test_vector_int_to_bytes.yml rename to eth2/utils/int_to_bytes/src/specs/test_vector_int_to_bytes.yml From fc04286ae6c996fd5567dce2045a307fc32752bd Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 15 Feb 2019 12:13:57 +1100 Subject: [PATCH 11/30] Use `int_to_bytes` in swap or not. I also fixed an error I found through strict typing. --- eth2/utils/swap_or_not_shuffle/Cargo.toml | 1 + eth2/utils/swap_or_not_shuffle/src/lib.rs | 50 ++++++++++++----------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/eth2/utils/swap_or_not_shuffle/Cargo.toml b/eth2/utils/swap_or_not_shuffle/Cargo.toml index 1c898f7b1..3dc03da82 100644 --- a/eth2/utils/swap_or_not_shuffle/Cargo.toml +++ b/eth2/utils/swap_or_not_shuffle/Cargo.toml @@ -7,6 +7,7 @@ edition = "2018" [dependencies] bytes = "0.4" hashing = { path = "../hashing" } +int_to_bytes = { path = "../int_to_bytes" } [dev-dependencies] yaml-rust = "0.4.2" diff --git a/eth2/utils/swap_or_not_shuffle/src/lib.rs b/eth2/utils/swap_or_not_shuffle/src/lib.rs index 3566ac23a..22d657bb4 100644 --- a/eth2/utils/swap_or_not_shuffle/src/lib.rs +++ b/eth2/utils/swap_or_not_shuffle/src/lib.rs @@ -1,5 +1,6 @@ -use bytes::{Buf, BufMut, BytesMut}; +use bytes::Buf; use hashing::hash; +use int_to_bytes::{int_to_bytes1, int_to_bytes4}; use std::cmp::max; use std::io::Cursor; @@ -12,14 +13,19 @@ use std::io::Cursor; /// Returns `None` under any of the following conditions: /// - `list_size == 0` /// - `index >= list_size` +/// - `list_size >= 2**24` /// - `list_size >= usize::max_value() / 2` pub fn get_permutated_index( index: usize, list_size: usize, seed: &[u8], - shuffle_round_count: usize, + shuffle_round_count: u8, ) -> Option { - if list_size == 0 || index >= list_size || list_size >= usize::max_value() / 2 { + if list_size == 0 + || index >= list_size + || list_size >= usize::max_value() / 2 + || list_size >= 2_usize.pow(24) + { return None; } @@ -28,7 +34,7 @@ pub fn get_permutated_index( let pivot = bytes_to_int64(&hash_with_round(seed, round)[..]) as usize % list_size; let flip = (pivot + list_size - index) % list_size; let position = max(index, flip); - let source = hash_with_round_and_position(seed, round, position); + let source = hash_with_round_and_position(seed, round, position)?; let byte = source[(position % 256) / 8]; let bit = (byte >> (position % 8)) % 2; index = if bit == 1 { flip } else { index } @@ -36,31 +42,23 @@ pub fn get_permutated_index( Some(index) } -fn hash_with_round_and_position(seed: &[u8], round: usize, position: usize) -> Vec { +fn hash_with_round_and_position(seed: &[u8], round: u8, position: usize) -> Option> { let mut seed = seed.to_vec(); - seed.append(&mut int_to_bytes1(round as u64)); - seed.append(&mut int_to_bytes4(position as u64 / 256)); - hash(&seed[..]) + seed.append(&mut int_to_bytes1(round)); + /* + * Note: the specification has an implicit assertion in `int_to_bytes4` that `position / 256 < + * 2**24`. For efficiency, we do not check for that here as it is checked in `get_permutated_index`. + */ + seed.append(&mut int_to_bytes4((position / 256) as u32)); + Some(hash(&seed[..])) } -fn hash_with_round(seed: &[u8], round: usize) -> Vec { +fn hash_with_round(seed: &[u8], round: u8) -> Vec { let mut seed = seed.to_vec(); - seed.append(&mut int_to_bytes1(round as u64)); + seed.append(&mut int_to_bytes1(round)); hash(&seed[..]) } -fn int_to_bytes1(int: u64) -> Vec { - let mut bytes = BytesMut::with_capacity(8); - bytes.put_u64_le(int); - vec![bytes[0]] -} - -fn int_to_bytes4(int: u64) -> Vec { - let mut bytes = BytesMut::with_capacity(8); - bytes.put_u64_le(int); - bytes[0..4].to_vec() -} - fn bytes_to_int64(bytes: &[u8]) -> u64 { let mut cursor = Cursor::new(bytes); cursor.get_u64_le() @@ -117,10 +115,16 @@ mod tests { let index = test_case["index"].as_i64().unwrap() as usize; let list_size = test_case["list_size"].as_i64().unwrap() as usize; let permutated_index = test_case["permutated_index"].as_i64().unwrap() as usize; - let shuffle_round_count = test_case["shuffle_round_count"].as_i64().unwrap() as usize; + let shuffle_round_count = test_case["shuffle_round_count"].as_i64().unwrap(); let seed_string = test_case["seed"].clone().into_string().unwrap(); let seed = hex::decode(seed_string.replace("0x", "")).unwrap(); + let shuffle_round_count = if shuffle_round_count < (u8::max_value() as i64) { + shuffle_round_count as u8 + } else { + panic!("shuffle_round_count must be a u8") + }; + assert_eq!( Some(permutated_index), get_permutated_index(index, list_size, &seed[..], shuffle_round_count), From 210ec89b0b1c0d3b1849ff335936b5b42999a957 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 15 Feb 2019 12:21:19 +1100 Subject: [PATCH 12/30] Replace `ssz_encode` with `int_to_bytes32` Only in the relevant places I can think of.. I might have missed some. --- eth2/block_producer/Cargo.toml | 7 ++++--- eth2/block_producer/src/lib.rs | 4 ++-- eth2/state_processing/Cargo.toml | 1 + eth2/state_processing/src/block_processable.rs | 3 ++- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/eth2/block_producer/Cargo.toml b/eth2/block_producer/Cargo.toml index 86dde92f7..15d1343cc 100644 --- a/eth2/block_producer/Cargo.toml +++ b/eth2/block_producer/Cargo.toml @@ -5,6 +5,7 @@ authors = ["Paul Hauner "] edition = "2018" [dependencies] -slot_clock = { path = "../../eth2/utils/slot_clock" } -ssz = { path = "../../eth2/utils/ssz" } -types = { path = "../../eth2/types" } +int_to_bytes = { path = "../utils/int_to_bytes" } +slot_clock = { path = "../utils/slot_clock" } +ssz = { path = "../utils/ssz" } +types = { path = "../types" } diff --git a/eth2/block_producer/src/lib.rs b/eth2/block_producer/src/lib.rs index f6a0fd6df..7b15eb4e9 100644 --- a/eth2/block_producer/src/lib.rs +++ b/eth2/block_producer/src/lib.rs @@ -1,8 +1,8 @@ pub mod test_utils; mod traits; +use int_to_bytes::int_to_bytes32; use slot_clock::SlotClock; -use ssz::ssz_encode; use std::sync::Arc; use types::{BeaconBlock, ChainSpec, Slot}; @@ -132,7 +132,7 @@ impl BlockProducer Result { let randao_reveal = { // TODO: add domain, etc to this message. Also ensure result matches `into_to_bytes32`. - let message = ssz_encode(&slot.epoch(self.spec.epoch_length)); + let message = int_to_bytes32(slot.epoch(self.spec.epoch_length).as_u64()); match self.signer.sign_randao_reveal(&message) { None => return Ok(PollOutcome::SignerRejection(slot)), diff --git a/eth2/state_processing/Cargo.toml b/eth2/state_processing/Cargo.toml index 683475f47..b6b0ea57c 100644 --- a/eth2/state_processing/Cargo.toml +++ b/eth2/state_processing/Cargo.toml @@ -6,6 +6,7 @@ edition = "2018" [dependencies] hashing = { path = "../utils/hashing" } +int_to_bytes = { path = "../utils/int_to_bytes" } integer-sqrt = "0.1" log = "0.4" ssz = { path = "../utils/ssz" } diff --git a/eth2/state_processing/src/block_processable.rs b/eth2/state_processing/src/block_processable.rs index f043a723d..368460116 100644 --- a/eth2/state_processing/src/block_processable.rs +++ b/eth2/state_processing/src/block_processable.rs @@ -1,5 +1,6 @@ use crate::SlotProcessingError; use hashing::hash; +use int_to_bytes::int_to_bytes32; use log::debug; use ssz::{ssz_encode, TreeHash}; use types::{ @@ -110,7 +111,7 @@ fn per_block_processing_signature_optional( ensure!( bls_verify( &block_proposer.pubkey, - &ssz_encode(&state.current_epoch(spec)), + &int_to_bytes32(state.current_epoch(spec).as_u64()), &block.randao_reveal, get_domain(&state.fork, state.current_epoch(spec), DOMAIN_RANDAO) ), From af1d44e8b0716be1d1e68a4255448e3613567604 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 15 Feb 2019 12:56:50 +1100 Subject: [PATCH 13/30] Add shuffling to fuzzer, fixing a bug We were being to strict on list length, it can be 2**24 as all index's will need to be less than that. --- eth2/utils/swap_or_not_shuffle/Cargo.toml | 1 + eth2/utils/swap_or_not_shuffle/src/lib.rs | 46 +++++++++++++++++++++-- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/eth2/utils/swap_or_not_shuffle/Cargo.toml b/eth2/utils/swap_or_not_shuffle/Cargo.toml index 3dc03da82..272abf608 100644 --- a/eth2/utils/swap_or_not_shuffle/Cargo.toml +++ b/eth2/utils/swap_or_not_shuffle/Cargo.toml @@ -12,3 +12,4 @@ int_to_bytes = { path = "../int_to_bytes" } [dev-dependencies] yaml-rust = "0.4.2" hex = "0.3" +ethereum-types = "0.5" diff --git a/eth2/utils/swap_or_not_shuffle/src/lib.rs b/eth2/utils/swap_or_not_shuffle/src/lib.rs index 22d657bb4..753265f3e 100644 --- a/eth2/utils/swap_or_not_shuffle/src/lib.rs +++ b/eth2/utils/swap_or_not_shuffle/src/lib.rs @@ -13,8 +13,8 @@ use std::io::Cursor; /// Returns `None` under any of the following conditions: /// - `list_size == 0` /// - `index >= list_size` -/// - `list_size >= 2**24` -/// - `list_size >= usize::max_value() / 2` +/// - `list_size > 2**24` +/// - `list_size > usize::max_value() / 2` pub fn get_permutated_index( index: usize, list_size: usize, @@ -23,8 +23,8 @@ pub fn get_permutated_index( ) -> Option { if list_size == 0 || index >= list_size - || list_size >= usize::max_value() / 2 - || list_size >= 2_usize.pow(24) + || list_size > usize::max_value() / 2 + || list_size > 2_usize.pow(24) { return None; } @@ -67,10 +67,48 @@ fn bytes_to_int64(bytes: &[u8]) -> u64 { #[cfg(test)] mod tests { use super::*; + use ethereum_types::H256 as Hash256; use hex; use std::{fs::File, io::prelude::*, path::PathBuf}; use yaml_rust::yaml; + #[test] + #[ignore] + fn fuzz_test() { + let max_list_size = 2_usize.pow(24); + let test_runs = 1000; + + // Test at max list_size with the end index. + for _ in 0..test_runs { + let index = max_list_size - 1; + let list_size = max_list_size; + let seed = Hash256::random(); + let shuffle_rounds = 90; + + assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some()); + } + + // Test at max list_size low indices. + for i in 0..test_runs { + let index = i; + let list_size = max_list_size; + let seed = Hash256::random(); + let shuffle_rounds = 90; + + assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some()); + } + + // Test at max list_size high indices. + for i in 0..test_runs { + let index = max_list_size - 1 - i; + let list_size = max_list_size; + let seed = Hash256::random(); + let shuffle_rounds = 90; + + assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some()); + } + } + #[test] fn returns_none_for_zero_length_list() { assert_eq!(None, get_permutated_index(100, 0, &[42, 42], 90)); From 2cab2952a8d8ba38a35e82c6ae830f46ba2eb2c6 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Fri, 15 Feb 2019 13:29:09 +1100 Subject: [PATCH 14/30] Removes duplication of macros in types. --- eth2/fork_choice/Cargo.toml | 2 + eth2/types/src/lib.rs | 2 + eth2/types/src/slot_epoch.rs | 249 ------------------------ eth2/types/src/slot_epoch_macros.rs | 265 ++++++++++++++++++++++++++ eth2/types/src/slot_height.rs | 281 +--------------------------- 5 files changed, 271 insertions(+), 528 deletions(-) create mode 100644 eth2/types/src/slot_epoch_macros.rs diff --git a/eth2/fork_choice/Cargo.toml b/eth2/fork_choice/Cargo.toml index 82d5785b9..72a653032 100644 --- a/eth2/fork_choice/Cargo.toml +++ b/eth2/fork_choice/Cargo.toml @@ -14,3 +14,5 @@ byteorder = "1.3.1" [dev-dependencies] yaml-rust = "0.4.2" bls = { path = "../utils/bls" } +slot_clock = { path = "../utils/slot_clock" } +beacon_chain = { path = "../../beacon_node/beacon_chain" } diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index 589593482..233d1cc3e 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -24,6 +24,8 @@ pub mod readers; pub mod shard_reassignment_record; pub mod slashable_attestation; pub mod slashable_vote_data; +#[macro_use] +pub mod slot_epoch_macros; pub mod slot_epoch; pub mod slot_height; pub mod spec; diff --git a/eth2/types/src/slot_epoch.rs b/eth2/types/src/slot_epoch.rs index b7a8e8f8d..c1554034a 100644 --- a/eth2/types/src/slot_epoch.rs +++ b/eth2/types/src/slot_epoch.rs @@ -21,255 +21,6 @@ use std::hash::{Hash, Hasher}; use std::iter::Iterator; use std::ops::{Add, AddAssign, Div, DivAssign, Mul, MulAssign, Rem, Sub, SubAssign}; -macro_rules! impl_from_into_u64 { - ($main: ident) => { - impl From for $main { - fn from(n: u64) -> $main { - $main(n) - } - } - - impl Into for $main { - fn into(self) -> u64 { - self.0 - } - } - - impl $main { - pub fn as_u64(&self) -> u64 { - self.0 - } - } - }; -} - -macro_rules! impl_from_into_usize { - ($main: ident) => { - impl From for $main { - fn from(n: usize) -> $main { - $main(n as u64) - } - } - - impl Into for $main { - fn into(self) -> usize { - self.0 as usize - } - } - - impl $main { - pub fn as_usize(&self) -> usize { - self.0 as usize - } - } - }; -} - -macro_rules! impl_math_between { - ($main: ident, $other: ident) => { - impl PartialOrd<$other> for $main { - /// Utilizes `partial_cmp` on the underlying `u64`. - fn partial_cmp(&self, other: &$other) -> Option { - Some(self.0.cmp(&(*other).into())) - } - } - - impl PartialEq<$other> for $main { - fn eq(&self, other: &$other) -> bool { - let other: u64 = (*other).into(); - self.0 == other - } - } - - impl Add<$other> for $main { - type Output = $main; - - fn add(self, other: $other) -> $main { - $main::from(self.0.saturating_add(other.into())) - } - } - - impl AddAssign<$other> for $main { - fn add_assign(&mut self, other: $other) { - self.0 = self.0.saturating_add(other.into()); - } - } - - impl Sub<$other> for $main { - type Output = $main; - - fn sub(self, other: $other) -> $main { - $main::from(self.0.saturating_sub(other.into())) - } - } - - impl SubAssign<$other> for $main { - fn sub_assign(&mut self, other: $other) { - self.0 = self.0.saturating_sub(other.into()); - } - } - - impl Mul<$other> for $main { - type Output = $main; - - fn mul(self, rhs: $other) -> $main { - let rhs: u64 = rhs.into(); - $main::from(self.0.saturating_mul(rhs)) - } - } - - impl MulAssign<$other> for $main { - fn mul_assign(&mut self, rhs: $other) { - let rhs: u64 = rhs.into(); - self.0 = self.0.saturating_mul(rhs) - } - } - - impl Div<$other> for $main { - type Output = $main; - - fn div(self, rhs: $other) -> $main { - let rhs: u64 = rhs.into(); - if rhs == 0 { - panic!("Cannot divide by zero-valued Slot/Epoch") - } - $main::from(self.0 / rhs) - } - } - - impl DivAssign<$other> for $main { - fn div_assign(&mut self, rhs: $other) { - let rhs: u64 = rhs.into(); - if rhs == 0 { - panic!("Cannot divide by zero-valued Slot/Epoch") - } - self.0 = self.0 / rhs - } - } - - impl Rem<$other> for $main { - type Output = $main; - - fn rem(self, modulus: $other) -> $main { - let modulus: u64 = modulus.into(); - $main::from(self.0 % modulus) - } - } - }; -} - -macro_rules! impl_math { - ($type: ident) => { - impl $type { - pub fn saturating_sub>(&self, other: T) -> $type { - *self - other.into() - } - - pub fn saturating_add>(&self, other: T) -> $type { - *self + other.into() - } - - pub fn checked_div>(&self, rhs: T) -> Option<$type> { - let rhs: $type = rhs.into(); - if rhs == 0 { - None - } else { - Some(*self / rhs) - } - } - - pub fn is_power_of_two(&self) -> bool { - self.0.is_power_of_two() - } - } - - impl Ord for $type { - fn cmp(&self, other: &$type) -> Ordering { - let other: u64 = (*other).into(); - self.0.cmp(&other) - } - } - }; -} - -macro_rules! impl_display { - ($type: ident) => { - impl fmt::Display for $type { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}", self.0) - } - } - - impl slog::Value for $type { - fn serialize( - &self, - record: &slog::Record, - key: slog::Key, - serializer: &mut slog::Serializer, - ) -> slog::Result { - self.0.serialize(record, key, serializer) - } - } - }; -} - -macro_rules! impl_ssz { - ($type: ident) => { - impl Encodable for $type { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.0); - } - } - - impl Decodable for $type { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (value, i) = <_>::ssz_decode(bytes, i)?; - - Ok(($type(value), i)) - } - } - - impl TreeHash for $type { - fn hash_tree_root(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.0.hash_tree_root()); - hash(&result) - } - } - - impl TestRandom for $type { - fn random_for_test(rng: &mut T) -> Self { - $type::from(u64::random_for_test(rng)) - } - } - }; -} - -macro_rules! impl_hash { - ($type: ident) => { - // Implemented to stop clippy lint: - // https://rust-lang.github.io/rust-clippy/master/index.html#derive_hash_xor_eq - impl Hash for $type { - fn hash(&self, state: &mut H) { - ssz_encode(self).hash(state) - } - } - }; -} - -macro_rules! impl_common { - ($type: ident) => { - impl_from_into_u64!($type); - impl_from_into_usize!($type); - impl_math_between!($type, $type); - impl_math_between!($type, u64); - impl_math!($type); - impl_display!($type); - impl_ssz!($type); - impl_hash!($type); - }; -} - #[derive(Eq, Debug, Clone, Copy, Default, Serialize)] pub struct Slot(u64); diff --git a/eth2/types/src/slot_epoch_macros.rs b/eth2/types/src/slot_epoch_macros.rs new file mode 100644 index 000000000..6813bfeaf --- /dev/null +++ b/eth2/types/src/slot_epoch_macros.rs @@ -0,0 +1,265 @@ +macro_rules! impl_from_into_u64 { + ($main: ident) => { + impl From for $main { + fn from(n: u64) -> $main { + $main(n) + } + } + + impl Into for $main { + fn into(self) -> u64 { + self.0 + } + } + + impl $main { + pub fn as_u64(&self) -> u64 { + self.0 + } + } + }; +} + +// need to truncate for some fork-choice algorithms +macro_rules! impl_into_u32 { + ($main: ident) => { + impl Into for $main { + fn into(self) -> u32 { + self.0 as u32 + } + } + + impl $main { + pub fn as_u32(&self) -> u32 { + self.0 as u32 + } + } + }; +} + +macro_rules! impl_from_into_usize { + ($main: ident) => { + impl From for $main { + fn from(n: usize) -> $main { + $main(n as u64) + } + } + + impl Into for $main { + fn into(self) -> usize { + self.0 as usize + } + } + + impl $main { + pub fn as_usize(&self) -> usize { + self.0 as usize + } + } + }; +} + +macro_rules! impl_math_between { + ($main: ident, $other: ident) => { + impl PartialOrd<$other> for $main { + /// Utilizes `partial_cmp` on the underlying `u64`. + fn partial_cmp(&self, other: &$other) -> Option { + Some(self.0.cmp(&(*other).into())) + } + } + + impl PartialEq<$other> for $main { + fn eq(&self, other: &$other) -> bool { + let other: u64 = (*other).into(); + self.0 == other + } + } + + impl Add<$other> for $main { + type Output = $main; + + fn add(self, other: $other) -> $main { + $main::from(self.0.saturating_add(other.into())) + } + } + + impl AddAssign<$other> for $main { + fn add_assign(&mut self, other: $other) { + self.0 = self.0.saturating_add(other.into()); + } + } + + impl Sub<$other> for $main { + type Output = $main; + + fn sub(self, other: $other) -> $main { + $main::from(self.0.saturating_sub(other.into())) + } + } + + impl SubAssign<$other> for $main { + fn sub_assign(&mut self, other: $other) { + self.0 = self.0.saturating_sub(other.into()); + } + } + + impl Mul<$other> for $main { + type Output = $main; + + fn mul(self, rhs: $other) -> $main { + let rhs: u64 = rhs.into(); + $main::from(self.0.saturating_mul(rhs)) + } + } + + impl MulAssign<$other> for $main { + fn mul_assign(&mut self, rhs: $other) { + let rhs: u64 = rhs.into(); + self.0 = self.0.saturating_mul(rhs) + } + } + + impl Div<$other> for $main { + type Output = $main; + + fn div(self, rhs: $other) -> $main { + let rhs: u64 = rhs.into(); + if rhs == 0 { + panic!("Cannot divide by zero-valued Slot/Epoch") + } + $main::from(self.0 / rhs) + } + } + + impl DivAssign<$other> for $main { + fn div_assign(&mut self, rhs: $other) { + let rhs: u64 = rhs.into(); + if rhs == 0 { + panic!("Cannot divide by zero-valued Slot/Epoch") + } + self.0 = self.0 / rhs + } + } + + impl Rem<$other> for $main { + type Output = $main; + + fn rem(self, modulus: $other) -> $main { + let modulus: u64 = modulus.into(); + $main::from(self.0 % modulus) + } + } + }; +} + +macro_rules! impl_math { + ($type: ident) => { + impl $type { + pub fn saturating_sub>(&self, other: T) -> $type { + *self - other.into() + } + + pub fn saturating_add>(&self, other: T) -> $type { + *self + other.into() + } + + pub fn checked_div>(&self, rhs: T) -> Option<$type> { + let rhs: $type = rhs.into(); + if rhs == 0 { + None + } else { + Some(*self / rhs) + } + } + + pub fn is_power_of_two(&self) -> bool { + self.0.is_power_of_two() + } + } + + impl Ord for $type { + fn cmp(&self, other: &$type) -> Ordering { + let other: u64 = (*other).into(); + self.0.cmp(&other) + } + } + }; +} + +macro_rules! impl_display { + ($type: ident) => { + impl fmt::Display for $type { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.0) + } + } + + impl slog::Value for $type { + fn serialize( + &self, + record: &slog::Record, + key: slog::Key, + serializer: &mut slog::Serializer, + ) -> slog::Result { + self.0.serialize(record, key, serializer) + } + } + }; +} + +macro_rules! impl_ssz { + ($type: ident) => { + impl Encodable for $type { + fn ssz_append(&self, s: &mut SszStream) { + s.append(&self.0); + } + } + + impl Decodable for $type { + fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { + let (value, i) = <_>::ssz_decode(bytes, i)?; + + Ok(($type(value), i)) + } + } + + impl TreeHash for $type { + fn hash_tree_root(&self) -> Vec { + let mut result: Vec = vec![]; + result.append(&mut self.0.hash_tree_root()); + hash(&result) + } + } + + impl TestRandom for $type { + fn random_for_test(rng: &mut T) -> Self { + $type::from(u64::random_for_test(rng)) + } + } + }; +} + +macro_rules! impl_hash { + ($type: ident) => { + // Implemented to stop clippy lint: + // https://rust-lang.github.io/rust-clippy/master/index.html#derive_hash_xor_eq + impl Hash for $type { + fn hash(&self, state: &mut H) { + ssz_encode(self).hash(state) + } + } + }; +} + +macro_rules! impl_common { + ($type: ident) => { + impl_from_into_u64!($type); + impl_from_into_usize!($type); + impl_math_between!($type, $type); + impl_math_between!($type, u64); + impl_math!($type); + impl_display!($type); + impl_ssz!($type); + impl_hash!($type); + }; +} diff --git a/eth2/types/src/slot_height.rs b/eth2/types/src/slot_height.rs index 77dd17ad9..3efd00cae 100644 --- a/eth2/types/src/slot_height.rs +++ b/eth2/types/src/slot_height.rs @@ -1,290 +1,13 @@ -// Copyright 2019 Sigma Prime Pty Ltd. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - use crate::slot_epoch::{Epoch, Slot}; +use crate::test_utils::TestRandom; +use rand::RngCore; use serde_derive::Serialize; -use slog; use ssz::{hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash}; use std::cmp::{Ord, Ordering}; use std::fmt; use std::hash::{Hash, Hasher}; use std::ops::{Add, AddAssign, Div, DivAssign, Mul, MulAssign, Rem, Sub, SubAssign}; -macro_rules! impl_from_into_u64 { - ($main: ident) => { - impl From for $main { - fn from(n: u64) -> $main { - $main(n) - } - } - - impl Into for $main { - fn into(self) -> u64 { - self.0 - } - } - - impl $main { - pub fn as_u64(&self) -> u64 { - self.0 - } - } - }; -} - -// need to truncate for some fork-choice algorithms -macro_rules! impl_into_u32 { - ($main: ident) => { - impl Into for $main { - fn into(self) -> u32 { - self.0 as u32 - } - } - - impl $main { - pub fn as_u32(&self) -> u32 { - self.0 as u32 - } - } - }; -} -macro_rules! impl_from_into_usize { - ($main: ident) => { - impl From for $main { - fn from(n: usize) -> $main { - $main(n as u64) - } - } - - impl Into for $main { - fn into(self) -> usize { - self.0 as usize - } - } - - impl $main { - pub fn as_usize(&self) -> usize { - self.0 as usize - } - } - }; -} - -macro_rules! impl_math_between { - ($main: ident, $other: ident) => { - impl PartialOrd<$other> for $main { - /// Utilizes `partial_cmp` on the underlying `u64`. - fn partial_cmp(&self, other: &$other) -> Option { - Some(self.0.cmp(&(*other).into())) - } - } - - impl PartialEq<$other> for $main { - fn eq(&self, other: &$other) -> bool { - let other: u64 = (*other).into(); - self.0 == other - } - } - - impl Add<$other> for $main { - type Output = $main; - - fn add(self, other: $other) -> $main { - $main::from(self.0.saturating_add(other.into())) - } - } - - impl AddAssign<$other> for $main { - fn add_assign(&mut self, other: $other) { - self.0 = self.0.saturating_add(other.into()); - } - } - - impl Sub<$other> for $main { - type Output = $main; - - fn sub(self, other: $other) -> $main { - $main::from(self.0.saturating_sub(other.into())) - } - } - - impl SubAssign<$other> for $main { - fn sub_assign(&mut self, other: $other) { - self.0 = self.0.saturating_sub(other.into()); - } - } - - impl Mul<$other> for $main { - type Output = $main; - - fn mul(self, rhs: $other) -> $main { - let rhs: u64 = rhs.into(); - $main::from(self.0.saturating_mul(rhs)) - } - } - - impl MulAssign<$other> for $main { - fn mul_assign(&mut self, rhs: $other) { - let rhs: u64 = rhs.into(); - self.0 = self.0.saturating_mul(rhs) - } - } - - impl Div<$other> for $main { - type Output = $main; - - fn div(self, rhs: $other) -> $main { - let rhs: u64 = rhs.into(); - if rhs == 0 { - panic!("Cannot divide by zero-valued Slot/Epoch") - } - $main::from(self.0 / rhs) - } - } - - impl DivAssign<$other> for $main { - fn div_assign(&mut self, rhs: $other) { - let rhs: u64 = rhs.into(); - if rhs == 0 { - panic!("Cannot divide by zero-valued Slot/Epoch") - } - self.0 = self.0 / rhs - } - } - - impl Rem<$other> for $main { - type Output = $main; - - fn rem(self, modulus: $other) -> $main { - let modulus: u64 = modulus.into(); - $main::from(self.0 % modulus) - } - } - }; -} - -macro_rules! impl_math { - ($type: ident) => { - impl $type { - pub fn saturating_sub>(&self, other: T) -> $type { - *self - other.into() - } - - pub fn saturating_add>(&self, other: T) -> $type { - *self + other.into() - } - - pub fn checked_div>(&self, rhs: T) -> Option<$type> { - let rhs: $type = rhs.into(); - if rhs == 0 { - None - } else { - Some(*self / rhs) - } - } - - pub fn is_power_of_two(&self) -> bool { - self.0.is_power_of_two() - } - } - - impl Ord for $type { - fn cmp(&self, other: &$type) -> Ordering { - let other: u64 = (*other).into(); - self.0.cmp(&other) - } - } - }; -} - -macro_rules! impl_display { - ($type: ident) => { - impl fmt::Display for $type { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}", self.0) - } - } - - impl slog::Value for $type { - fn serialize( - &self, - record: &slog::Record, - key: slog::Key, - serializer: &mut slog::Serializer, - ) -> slog::Result { - self.0.serialize(record, key, serializer) - } - } - }; -} - -macro_rules! impl_ssz { - ($type: ident) => { - impl Encodable for $type { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.0); - } - } - - impl Decodable for $type { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (value, i) = <_>::ssz_decode(bytes, i)?; - - Ok(($type(value), i)) - } - } - - impl TreeHash for $type { - fn hash_tree_root(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.0.hash_tree_root()); - hash(&result) - } - } - }; -} - -macro_rules! impl_hash { - ($type: ident) => { - // Implemented to stop clippy lint: - // https://rust-lang.github.io/rust-clippy/master/index.html#derive_hash_xor_eq - impl Hash for $type { - fn hash(&self, state: &mut H) { - ssz_encode(self).hash(state) - } - } - }; -} - -macro_rules! impl_common { - ($type: ident) => { - impl_from_into_u64!($type); - impl_from_into_usize!($type); - impl_math_between!($type, $type); - impl_math_between!($type, u64); - impl_math!($type); - impl_display!($type); - impl_ssz!($type); - impl_hash!($type); - }; -} /// Beacon block height, effectively `Slot/GENESIS_START_BLOCK`. #[derive(Eq, Debug, Clone, Copy, Default, Serialize)] pub struct SlotHeight(u64); From 5031ee5505924071f46029a944794cf7c0782862 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Fri, 15 Feb 2019 13:32:37 +1100 Subject: [PATCH 15/30] Remove comments from fork choice. --- eth2/fork_choice/src/lib.rs | 5 ----- eth2/fork_choice/src/longest_chain.rs | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/eth2/fork_choice/src/lib.rs b/eth2/fork_choice/src/lib.rs index e876ed60f..c0df820c6 100644 --- a/eth2/fork_choice/src/lib.rs +++ b/eth2/fork_choice/src/lib.rs @@ -44,14 +44,12 @@ extern crate types; pub mod longest_chain; pub mod optimised_lmd_ghost; -//pub mod protolambda_lmd_ghost; pub mod slow_lmd_ghost; use db::stores::BeaconBlockAtSlotError; use db::DBError; use types::{BeaconBlock, Hash256}; -// export the main structs pub use longest_chain::LongestChain; pub use optimised_lmd_ghost::OptimisedLMDGhost; @@ -118,7 +116,4 @@ pub enum ForkChoiceAlgorithms { SlowLMDGhost, /// An optimised version of LMD-GHOST by Vitalik. OptimisedLMDGhost, - // Protolambda currently not implemented - // /// An optimised version of LMD-GHOST by Protolambda. - //ProtoLMDGhost, } diff --git a/eth2/fork_choice/src/longest_chain.rs b/eth2/fork_choice/src/longest_chain.rs index c25cd43a1..8056c11f2 100644 --- a/eth2/fork_choice/src/longest_chain.rs +++ b/eth2/fork_choice/src/longest_chain.rs @@ -52,7 +52,7 @@ impl ForkChoice for LongestChain { let block = self .block_store .get_deserialized(&block_hash)? - .ok_or(ForkChoiceError::MissingBeaconBlock(*block_hash))?; + .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*block_hash))?; head_blocks.push((index, block)); } From b16ac40fd51da6aac00418bb4547e03d0dbc6baf Mon Sep 17 00:00:00 2001 From: Age Manning Date: Fri, 15 Feb 2019 14:16:45 +1100 Subject: [PATCH 16/30] Add tests to SlotHeight. --- eth2/types/src/slot_epoch.rs | 380 +--------------------------- eth2/types/src/slot_epoch_macros.rs | 356 ++++++++++++++++++++++++++ eth2/types/src/slot_height.rs | 10 + 3 files changed, 379 insertions(+), 367 deletions(-) diff --git a/eth2/types/src/slot_epoch.rs b/eth2/types/src/slot_epoch.rs index c1554034a..eb5a8dced 100644 --- a/eth2/types/src/slot_epoch.rs +++ b/eth2/types/src/slot_epoch.rs @@ -100,373 +100,19 @@ impl<'a> Iterator for SlotIter<'a> { } #[cfg(test)] -mod tests { +mod slot_tests { use super::*; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; - macro_rules! new_tests { - ($type: ident) => { - #[test] - fn new() { - assert_eq!($type(0), $type::new(0)); - assert_eq!($type(3), $type::new(3)); - assert_eq!($type(u64::max_value()), $type::new(u64::max_value())); - } - }; - } - - macro_rules! from_into_tests { - ($type: ident, $other: ident) => { - #[test] - fn into() { - let x: $other = $type(0).into(); - assert_eq!(x, 0); - - let x: $other = $type(3).into(); - assert_eq!(x, 3); - - let x: $other = $type(u64::max_value()).into(); - // Note: this will fail on 32 bit systems. This is expected as we don't have a proper - // 32-bit system strategy in place. - assert_eq!(x, $other::max_value()); - } - - #[test] - fn from() { - assert_eq!($type(0), $type::from(0_u64)); - assert_eq!($type(3), $type::from(3_u64)); - assert_eq!($type(u64::max_value()), $type::from($other::max_value())); - } - }; - } - - macro_rules! math_between_tests { - ($type: ident, $other: ident) => { - #[test] - fn partial_ord() { - let assert_partial_ord = |a: u64, partial_ord: Ordering, b: u64| { - let other: $other = $type(b).into(); - assert_eq!($type(a).partial_cmp(&other), Some(partial_ord)); - }; - - assert_partial_ord(1, Ordering::Less, 2); - assert_partial_ord(2, Ordering::Greater, 1); - assert_partial_ord(0, Ordering::Less, u64::max_value()); - assert_partial_ord(u64::max_value(), Ordering::Greater, 0); - } - - #[test] - fn partial_eq() { - let assert_partial_eq = |a: u64, b: u64, is_equal: bool| { - let other: $other = $type(b).into(); - assert_eq!($type(a).eq(&other), is_equal); - }; - - assert_partial_eq(0, 0, true); - assert_partial_eq(0, 1, false); - assert_partial_eq(1, 0, false); - assert_partial_eq(1, 1, true); - - assert_partial_eq(u64::max_value(), u64::max_value(), true); - assert_partial_eq(0, u64::max_value(), false); - assert_partial_eq(u64::max_value(), 0, false); - } - - #[test] - fn add_and_add_assign() { - let assert_add = |a: u64, b: u64, result: u64| { - let other: $other = $type(b).into(); - assert_eq!($type(a) + other, $type(result)); - - let mut add_assigned = $type(a); - add_assigned += other; - - assert_eq!(add_assigned, $type(result)); - }; - - assert_add(0, 1, 1); - assert_add(1, 0, 1); - assert_add(1, 2, 3); - assert_add(2, 1, 3); - assert_add(7, 7, 14); - - // Addition should be saturating. - assert_add(u64::max_value(), 1, u64::max_value()); - assert_add(u64::max_value(), u64::max_value(), u64::max_value()); - } - - #[test] - fn sub_and_sub_assign() { - let assert_sub = |a: u64, b: u64, result: u64| { - let other: $other = $type(b).into(); - assert_eq!($type(a) - other, $type(result)); - - let mut sub_assigned = $type(a); - sub_assigned -= other; - - assert_eq!(sub_assigned, $type(result)); - }; - - assert_sub(1, 0, 1); - assert_sub(2, 1, 1); - assert_sub(14, 7, 7); - assert_sub(u64::max_value(), 1, u64::max_value() - 1); - assert_sub(u64::max_value(), u64::max_value(), 0); - - // Subtraction should be saturating - assert_sub(0, 1, 0); - assert_sub(1, 2, 0); - } - - #[test] - fn mul_and_mul_assign() { - let assert_mul = |a: u64, b: u64, result: u64| { - let other: $other = $type(b).into(); - assert_eq!($type(a) * other, $type(result)); - - let mut mul_assigned = $type(a); - mul_assigned *= other; - - assert_eq!(mul_assigned, $type(result)); - }; - - assert_mul(2, 2, 4); - assert_mul(1, 2, 2); - assert_mul(0, 2, 0); - - // Multiplication should be saturating. - assert_mul(u64::max_value(), 2, u64::max_value()); - } - - #[test] - fn div_and_div_assign() { - let assert_div = |a: u64, b: u64, result: u64| { - let other: $other = $type(b).into(); - assert_eq!($type(a) / other, $type(result)); - - let mut div_assigned = $type(a); - div_assigned /= other; - - assert_eq!(div_assigned, $type(result)); - }; - - assert_div(0, 2, 0); - assert_div(2, 2, 1); - assert_div(100, 50, 2); - assert_div(128, 2, 64); - assert_div(u64::max_value(), 2, 2_u64.pow(63) - 1); - } - - #[test] - #[should_panic] - fn div_panics_with_divide_by_zero() { - let other: $other = $type(0).into(); - let _ = $type(2) / other; - } - - #[test] - #[should_panic] - fn div_assign_panics_with_divide_by_zero() { - let other: $other = $type(0).into(); - let mut assigned = $type(2); - assigned /= other; - } - - #[test] - fn rem() { - let assert_rem = |a: u64, b: u64, result: u64| { - let other: $other = $type(b).into(); - assert_eq!($type(a) % other, $type(result)); - }; - - assert_rem(3, 2, 1); - assert_rem(40, 2, 0); - assert_rem(10, 100, 10); - assert_rem(302042, 3293, 2379); - } - }; - } - - macro_rules! math_tests { - ($type: ident) => { - #[test] - fn saturating_sub() { - let assert_saturating_sub = |a: u64, b: u64, result: u64| { - assert_eq!($type(a).saturating_sub($type(b)), $type(result)); - }; - - assert_saturating_sub(1, 0, 1); - assert_saturating_sub(2, 1, 1); - assert_saturating_sub(14, 7, 7); - assert_saturating_sub(u64::max_value(), 1, u64::max_value() - 1); - assert_saturating_sub(u64::max_value(), u64::max_value(), 0); - - // Subtraction should be saturating - assert_saturating_sub(0, 1, 0); - assert_saturating_sub(1, 2, 0); - } - - #[test] - fn saturating_add() { - let assert_saturating_add = |a: u64, b: u64, result: u64| { - assert_eq!($type(a).saturating_add($type(b)), $type(result)); - }; - - assert_saturating_add(0, 1, 1); - assert_saturating_add(1, 0, 1); - assert_saturating_add(1, 2, 3); - assert_saturating_add(2, 1, 3); - assert_saturating_add(7, 7, 14); - - // Addition should be saturating. - assert_saturating_add(u64::max_value(), 1, u64::max_value()); - assert_saturating_add(u64::max_value(), u64::max_value(), u64::max_value()); - } - - #[test] - fn checked_div() { - let assert_checked_div = |a: u64, b: u64, result: Option| { - let division_result_as_u64 = match $type(a).checked_div($type(b)) { - None => None, - Some(val) => Some(val.as_u64()), - }; - assert_eq!(division_result_as_u64, result); - }; - - assert_checked_div(0, 2, Some(0)); - assert_checked_div(2, 2, Some(1)); - assert_checked_div(100, 50, Some(2)); - assert_checked_div(128, 2, Some(64)); - assert_checked_div(u64::max_value(), 2, Some(2_u64.pow(63) - 1)); - - assert_checked_div(2, 0, None); - assert_checked_div(0, 0, None); - assert_checked_div(u64::max_value(), 0, None); - } - - #[test] - fn is_power_of_two() { - let assert_is_power_of_two = |a: u64, result: bool| { - assert_eq!( - $type(a).is_power_of_two(), - result, - "{}.is_power_of_two() != {}", - a, - result - ); - }; - - assert_is_power_of_two(0, false); - assert_is_power_of_two(1, true); - assert_is_power_of_two(2, true); - assert_is_power_of_two(3, false); - assert_is_power_of_two(4, true); - - assert_is_power_of_two(2_u64.pow(4), true); - assert_is_power_of_two(u64::max_value(), false); - } - - #[test] - fn ord() { - let assert_ord = |a: u64, ord: Ordering, b: u64| { - assert_eq!($type(a).cmp(&$type(b)), ord); - }; - - assert_ord(1, Ordering::Less, 2); - assert_ord(2, Ordering::Greater, 1); - assert_ord(0, Ordering::Less, u64::max_value()); - assert_ord(u64::max_value(), Ordering::Greater, 0); - } - }; - } - - macro_rules! ssz_tests { - ($type: ident) => { - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = $type::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = $type::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = $type::random_for_test(&mut rng); - - let result = original.hash_tree_root(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } - }; - } - - macro_rules! all_tests { - ($type: ident) => { - new_tests!($type); - math_between_tests!($type, $type); - math_tests!($type); - ssz_tests!($type); - - mod u64_tests { - use super::*; - - from_into_tests!($type, u64); - math_between_tests!($type, u64); - - #[test] - pub fn as_64() { - let x = $type(0).as_u64(); - assert_eq!(x, 0); - - let x = $type(3).as_u64(); - assert_eq!(x, 3); - - let x = $type(u64::max_value()).as_u64(); - assert_eq!(x, u64::max_value()); - } - } - - mod usize_tests { - use super::*; - - from_into_tests!($type, usize); - - #[test] - pub fn as_usize() { - let x = $type(0).as_usize(); - assert_eq!(x, 0); - - let x = $type(3).as_usize(); - assert_eq!(x, 3); - - let x = $type(u64::max_value()).as_usize(); - assert_eq!(x, usize::max_value()); - } - } - }; - } - - #[cfg(test)] - mod slot_tests { - use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; - - all_tests!(Slot); - } - - #[cfg(test)] - mod epoch_tests { - use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; - - all_tests!(Epoch); - } + all_tests!(Slot); +} + +#[cfg(test)] +mod epoch_tests { + use super::*; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; + + all_tests!(Epoch); } diff --git a/eth2/types/src/slot_epoch_macros.rs b/eth2/types/src/slot_epoch_macros.rs index 6813bfeaf..48bc219da 100644 --- a/eth2/types/src/slot_epoch_macros.rs +++ b/eth2/types/src/slot_epoch_macros.rs @@ -263,3 +263,359 @@ macro_rules! impl_common { impl_hash!($type); }; } + +// test macros +#[allow(unused_macros)] +macro_rules! new_tests { + ($type: ident) => { + #[test] + fn new() { + assert_eq!($type(0), $type::new(0)); + assert_eq!($type(3), $type::new(3)); + assert_eq!($type(u64::max_value()), $type::new(u64::max_value())); + } + }; +} + +#[allow(unused_macros)] +macro_rules! from_into_tests { + ($type: ident, $other: ident) => { + #[test] + fn into() { + let x: $other = $type(0).into(); + assert_eq!(x, 0); + + let x: $other = $type(3).into(); + assert_eq!(x, 3); + + let x: $other = $type(u64::max_value()).into(); + // Note: this will fail on 32 bit systems. This is expected as we don't have a proper + // 32-bit system strategy in place. + assert_eq!(x, $other::max_value()); + } + + #[test] + fn from() { + assert_eq!($type(0), $type::from(0_u64)); + assert_eq!($type(3), $type::from(3_u64)); + assert_eq!($type(u64::max_value()), $type::from($other::max_value())); + } + }; +} + +#[allow(unused_macros)] +macro_rules! math_between_tests { + ($type: ident, $other: ident) => { + #[test] + fn partial_ord() { + let assert_partial_ord = |a: u64, partial_ord: Ordering, b: u64| { + let other: $other = $type(b).into(); + assert_eq!($type(a).partial_cmp(&other), Some(partial_ord)); + }; + + assert_partial_ord(1, Ordering::Less, 2); + assert_partial_ord(2, Ordering::Greater, 1); + assert_partial_ord(0, Ordering::Less, u64::max_value()); + assert_partial_ord(u64::max_value(), Ordering::Greater, 0); + } + + #[test] + fn partial_eq() { + let assert_partial_eq = |a: u64, b: u64, is_equal: bool| { + let other: $other = $type(b).into(); + assert_eq!($type(a).eq(&other), is_equal); + }; + + assert_partial_eq(0, 0, true); + assert_partial_eq(0, 1, false); + assert_partial_eq(1, 0, false); + assert_partial_eq(1, 1, true); + + assert_partial_eq(u64::max_value(), u64::max_value(), true); + assert_partial_eq(0, u64::max_value(), false); + assert_partial_eq(u64::max_value(), 0, false); + } + + #[test] + fn add_and_add_assign() { + let assert_add = |a: u64, b: u64, result: u64| { + let other: $other = $type(b).into(); + assert_eq!($type(a) + other, $type(result)); + + let mut add_assigned = $type(a); + add_assigned += other; + + assert_eq!(add_assigned, $type(result)); + }; + + assert_add(0, 1, 1); + assert_add(1, 0, 1); + assert_add(1, 2, 3); + assert_add(2, 1, 3); + assert_add(7, 7, 14); + + // Addition should be saturating. + assert_add(u64::max_value(), 1, u64::max_value()); + assert_add(u64::max_value(), u64::max_value(), u64::max_value()); + } + + #[test] + fn sub_and_sub_assign() { + let assert_sub = |a: u64, b: u64, result: u64| { + let other: $other = $type(b).into(); + assert_eq!($type(a) - other, $type(result)); + + let mut sub_assigned = $type(a); + sub_assigned -= other; + + assert_eq!(sub_assigned, $type(result)); + }; + + assert_sub(1, 0, 1); + assert_sub(2, 1, 1); + assert_sub(14, 7, 7); + assert_sub(u64::max_value(), 1, u64::max_value() - 1); + assert_sub(u64::max_value(), u64::max_value(), 0); + + // Subtraction should be saturating + assert_sub(0, 1, 0); + assert_sub(1, 2, 0); + } + + #[test] + fn mul_and_mul_assign() { + let assert_mul = |a: u64, b: u64, result: u64| { + let other: $other = $type(b).into(); + assert_eq!($type(a) * other, $type(result)); + + let mut mul_assigned = $type(a); + mul_assigned *= other; + + assert_eq!(mul_assigned, $type(result)); + }; + + assert_mul(2, 2, 4); + assert_mul(1, 2, 2); + assert_mul(0, 2, 0); + + // Multiplication should be saturating. + assert_mul(u64::max_value(), 2, u64::max_value()); + } + + #[test] + fn div_and_div_assign() { + let assert_div = |a: u64, b: u64, result: u64| { + let other: $other = $type(b).into(); + assert_eq!($type(a) / other, $type(result)); + + let mut div_assigned = $type(a); + div_assigned /= other; + + assert_eq!(div_assigned, $type(result)); + }; + + assert_div(0, 2, 0); + assert_div(2, 2, 1); + assert_div(100, 50, 2); + assert_div(128, 2, 64); + assert_div(u64::max_value(), 2, 2_u64.pow(63) - 1); + } + + #[test] + #[should_panic] + fn div_panics_with_divide_by_zero() { + let other: $other = $type(0).into(); + let _ = $type(2) / other; + } + + #[test] + #[should_panic] + fn div_assign_panics_with_divide_by_zero() { + let other: $other = $type(0).into(); + let mut assigned = $type(2); + assigned /= other; + } + + #[test] + fn rem() { + let assert_rem = |a: u64, b: u64, result: u64| { + let other: $other = $type(b).into(); + assert_eq!($type(a) % other, $type(result)); + }; + + assert_rem(3, 2, 1); + assert_rem(40, 2, 0); + assert_rem(10, 100, 10); + assert_rem(302042, 3293, 2379); + } + }; +} + +#[allow(unused_macros)] +macro_rules! math_tests { + ($type: ident) => { + #[test] + fn saturating_sub() { + let assert_saturating_sub = |a: u64, b: u64, result: u64| { + assert_eq!($type(a).saturating_sub($type(b)), $type(result)); + }; + + assert_saturating_sub(1, 0, 1); + assert_saturating_sub(2, 1, 1); + assert_saturating_sub(14, 7, 7); + assert_saturating_sub(u64::max_value(), 1, u64::max_value() - 1); + assert_saturating_sub(u64::max_value(), u64::max_value(), 0); + + // Subtraction should be saturating + assert_saturating_sub(0, 1, 0); + assert_saturating_sub(1, 2, 0); + } + + #[test] + fn saturating_add() { + let assert_saturating_add = |a: u64, b: u64, result: u64| { + assert_eq!($type(a).saturating_add($type(b)), $type(result)); + }; + + assert_saturating_add(0, 1, 1); + assert_saturating_add(1, 0, 1); + assert_saturating_add(1, 2, 3); + assert_saturating_add(2, 1, 3); + assert_saturating_add(7, 7, 14); + + // Addition should be saturating. + assert_saturating_add(u64::max_value(), 1, u64::max_value()); + assert_saturating_add(u64::max_value(), u64::max_value(), u64::max_value()); + } + + #[test] + fn checked_div() { + let assert_checked_div = |a: u64, b: u64, result: Option| { + let division_result_as_u64 = match $type(a).checked_div($type(b)) { + None => None, + Some(val) => Some(val.as_u64()), + }; + assert_eq!(division_result_as_u64, result); + }; + + assert_checked_div(0, 2, Some(0)); + assert_checked_div(2, 2, Some(1)); + assert_checked_div(100, 50, Some(2)); + assert_checked_div(128, 2, Some(64)); + assert_checked_div(u64::max_value(), 2, Some(2_u64.pow(63) - 1)); + + assert_checked_div(2, 0, None); + assert_checked_div(0, 0, None); + assert_checked_div(u64::max_value(), 0, None); + } + + #[test] + fn is_power_of_two() { + let assert_is_power_of_two = |a: u64, result: bool| { + assert_eq!( + $type(a).is_power_of_two(), + result, + "{}.is_power_of_two() != {}", + a, + result + ); + }; + + assert_is_power_of_two(0, false); + assert_is_power_of_two(1, true); + assert_is_power_of_two(2, true); + assert_is_power_of_two(3, false); + assert_is_power_of_two(4, true); + + assert_is_power_of_two(2_u64.pow(4), true); + assert_is_power_of_two(u64::max_value(), false); + } + + #[test] + fn ord() { + let assert_ord = |a: u64, ord: Ordering, b: u64| { + assert_eq!($type(a).cmp(&$type(b)), ord); + }; + + assert_ord(1, Ordering::Less, 2); + assert_ord(2, Ordering::Greater, 1); + assert_ord(0, Ordering::Less, u64::max_value()); + assert_ord(u64::max_value(), Ordering::Greater, 0); + } + }; +} + +#[allow(unused_macros)] +macro_rules! ssz_tests { + ($type: ident) => { + #[test] + pub fn test_ssz_round_trip() { + let mut rng = XorShiftRng::from_seed([42; 16]); + let original = $type::random_for_test(&mut rng); + + let bytes = ssz_encode(&original); + let (decoded, _) = $type::ssz_decode(&bytes, 0).unwrap(); + + assert_eq!(original, decoded); + } + + #[test] + pub fn test_hash_tree_root() { + let mut rng = XorShiftRng::from_seed([42; 16]); + let original = $type::random_for_test(&mut rng); + + let result = original.hash_tree_root(); + + assert_eq!(result.len(), 32); + // TODO: Add further tests + // https://github.com/sigp/lighthouse/issues/170 + } + }; +} + +#[allow(unused_macros)] +macro_rules! all_tests { + ($type: ident) => { + new_tests!($type); + math_between_tests!($type, $type); + math_tests!($type); + ssz_tests!($type); + + mod u64_tests { + use super::*; + + from_into_tests!($type, u64); + math_between_tests!($type, u64); + + #[test] + pub fn as_64() { + let x = $type(0).as_u64(); + assert_eq!(x, 0); + + let x = $type(3).as_u64(); + assert_eq!(x, 3); + + let x = $type(u64::max_value()).as_u64(); + assert_eq!(x, u64::max_value()); + } + } + + mod usize_tests { + use super::*; + + from_into_tests!($type, usize); + + #[test] + pub fn as_usize() { + let x = $type(0).as_usize(); + assert_eq!(x, 0); + + let x = $type(3).as_usize(); + assert_eq!(x, 3); + + let x = $type(u64::max_value()).as_usize(); + assert_eq!(x, usize::max_value()); + } + } + }; +} diff --git a/eth2/types/src/slot_height.rs b/eth2/types/src/slot_height.rs index 3efd00cae..afa0ff775 100644 --- a/eth2/types/src/slot_height.rs +++ b/eth2/types/src/slot_height.rs @@ -32,3 +32,13 @@ impl SlotHeight { SlotHeight(u64::max_value()) } } + +#[cfg(test)] + +mod slot_height_tests { + use super::*; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; + + all_tests!(SlotHeight); +} From 94c87845236383e8506f257b4a2853b3e87e63b1 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Fri, 15 Feb 2019 14:17:22 +1100 Subject: [PATCH 17/30] Remove Slow LMD Ghost from test harness. --- .../beacon_chain/test_harness/src/beacon_chain_harness.rs | 2 +- .../beacon_chain/test_harness/src/validator_harness/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index 09621268c..acba2e015 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -6,7 +6,7 @@ use db::{ stores::{BeaconBlockStore, BeaconStateStore}, MemoryDB, }; -use fork_choice::{optimised_lmd_ghost::OptimisedLMDGhost, slow_lmd_ghost::SlowLMDGhost}; // import all the algorithms +use fork_choice::OptimisedLMDGhost; use log::debug; use rayon::prelude::*; use slot_clock::TestingSlotClock; diff --git a/beacon_node/beacon_chain/test_harness/src/validator_harness/mod.rs b/beacon_node/beacon_chain/test_harness/src/validator_harness/mod.rs index e22ea1a2e..3df32fa64 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator_harness/mod.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator_harness/mod.rs @@ -10,7 +10,7 @@ use block_producer::{BlockProducer, Error as BlockPollError}; use db::MemoryDB; use direct_beacon_node::DirectBeaconNode; use direct_duties::DirectDuties; -use fork_choice::{optimised_lmd_ghost::OptimisedLMDGhost, slow_lmd_ghost::SlowLMDGhost}; +use fork_choice::OptimisedLMDGhost; use local_signer::LocalSigner; use slot_clock::TestingSlotClock; use std::sync::Arc; From e06c4796e41ce67fb2136bcbf48de7f2486c08c8 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Fri, 15 Feb 2019 14:21:33 +1100 Subject: [PATCH 18/30] Import Slot directly from types in fork-choice. --- eth2/fork_choice/src/optimised_lmd_ghost.rs | 4 ++-- eth2/fork_choice/src/slow_lmd_ghost.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eth2/fork_choice/src/optimised_lmd_ghost.rs b/eth2/fork_choice/src/optimised_lmd_ghost.rs index 40dc029be..6b21e39f8 100644 --- a/eth2/fork_choice/src/optimised_lmd_ghost.rs +++ b/eth2/fork_choice/src/optimised_lmd_ghost.rs @@ -30,8 +30,8 @@ use fast_math::log2_raw; use std::collections::HashMap; use std::sync::Arc; use types::{ - readers::BeaconBlockReader, slot_epoch::Slot, slot_height::SlotHeight, - validator_registry::get_active_validator_indices, BeaconBlock, Hash256, + readers::BeaconBlockReader, validator_registry::get_active_validator_indices, BeaconBlock, + Hash256, Slot, SlotHeight, }; //TODO: Pruning - Children diff --git a/eth2/fork_choice/src/slow_lmd_ghost.rs b/eth2/fork_choice/src/slow_lmd_ghost.rs index 609d28ab2..3184150fd 100644 --- a/eth2/fork_choice/src/slow_lmd_ghost.rs +++ b/eth2/fork_choice/src/slow_lmd_ghost.rs @@ -28,8 +28,8 @@ use db::{ use std::collections::HashMap; use std::sync::Arc; use types::{ - readers::BeaconBlockReader, slot_epoch::Slot, validator_registry::get_active_validator_indices, - BeaconBlock, Hash256, + readers::BeaconBlockReader, validator_registry::get_active_validator_indices, BeaconBlock, + Hash256, Slot, }; //TODO: Pruning and syncing From ec4a658fe77afdd20e0556a3b1a1f697bbbb34e7 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 15 Feb 2019 16:12:24 +1100 Subject: [PATCH 19/30] Add beacon state test builder, tidy errors --- .../src/attestation_aggregator.rs | 4 +- beacon_node/beacon_chain/src/beacon_chain.rs | 16 +-- .../state_processing/src/block_processable.rs | 29 ++++- .../state_processing/src/epoch_processable.rs | 19 ++- eth2/state_processing/src/slot_processable.rs | 10 +- eth2/types/Cargo.toml | 3 + eth2/types/src/beacon_state.rs | 121 +++++++----------- eth2/types/src/beacon_state_tests.rs | 74 +++++++++++ eth2/types/src/lib.rs | 1 + 9 files changed, 173 insertions(+), 104 deletions(-) create mode 100644 eth2/types/src/beacon_state_tests.rs diff --git a/beacon_node/beacon_chain/src/attestation_aggregator.rs b/beacon_node/beacon_chain/src/attestation_aggregator.rs index 149f0d60d..6fbc11612 100644 --- a/beacon_node/beacon_chain/src/attestation_aggregator.rs +++ b/beacon_node/beacon_chain/src/attestation_aggregator.rs @@ -1,7 +1,7 @@ use state_processing::validate_attestation_without_signature; use std::collections::{HashMap, HashSet}; use types::{ - beacon_state::CommitteesError, AggregateSignature, Attestation, AttestationData, BeaconState, + beacon_state::BeaconStateError, AggregateSignature, Attestation, AttestationData, BeaconState, Bitfield, ChainSpec, FreeAttestation, Signature, }; @@ -79,7 +79,7 @@ impl AttestationAggregator { state: &BeaconState, free_attestation: &FreeAttestation, spec: &ChainSpec, - ) -> Result { + ) -> Result { let (slot, shard, committee_index) = some_or_invalid!( state.attestation_slot_and_shard_for_validator( free_attestation.validator_index as usize, diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 41ceb4e29..c16337fd4 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -14,7 +14,7 @@ use state_processing::{ }; use std::sync::Arc; use types::{ - beacon_state::CommitteesError, + beacon_state::BeaconStateError, readers::{BeaconBlockReader, BeaconStateReader}, AttestationData, BeaconBlock, BeaconBlockBody, BeaconState, ChainSpec, Crosslink, Deposit, Epoch, Eth1Data, FreeAttestation, Hash256, PublicKey, Signature, Slot, @@ -24,7 +24,7 @@ use types::{ pub enum Error { InsufficientValidators, BadRecentBlockRoots, - CommitteesError(CommitteesError), + BeaconStateError(BeaconStateError), DBInconsistent(String), DBError(String), ForkChoiceError(ForkChoiceError), @@ -99,7 +99,7 @@ where initial_validator_deposits, latest_eth1_data, &spec, - ); + )?; let state_root = genesis_state.canonical_root(); state_store.put(&state_root, &ssz_encode(&genesis_state)[..])?; @@ -252,7 +252,7 @@ where /// /// Information is read from the present `beacon_state` shuffling, so only information from the /// present and prior epoch is available. - pub fn block_proposer(&self, slot: Slot) -> Result { + pub fn block_proposer(&self, slot: Slot) -> Result { let index = self .state .read() @@ -273,7 +273,7 @@ where pub fn validator_attestion_slot_and_shard( &self, validator_index: usize, - ) -> Result, CommitteesError> { + ) -> Result, BeaconStateError> { if let Some((slot, shard, _committee)) = self .state .read() @@ -588,8 +588,8 @@ impl From for Error { } } -impl From for Error { - fn from(e: CommitteesError) -> Error { - Error::CommitteesError(e) +impl From for Error { + fn from(e: BeaconStateError) -> Error { + Error::BeaconStateError(e) } } diff --git a/eth2/state_processing/src/block_processable.rs b/eth2/state_processing/src/block_processable.rs index f043a723d..3ae3c1b67 100644 --- a/eth2/state_processing/src/block_processable.rs +++ b/eth2/state_processing/src/block_processable.rs @@ -3,7 +3,7 @@ use hashing::hash; use log::debug; use ssz::{ssz_encode, TreeHash}; use types::{ - beacon_state::{AttestationValidationError, CommitteesError}, + beacon_state::{AttestationParticipantsError, BeaconStateError}, AggregatePublicKey, Attestation, BeaconBlock, BeaconState, ChainSpec, Crosslink, Epoch, Exit, Fork, Hash256, PendingAttestation, PublicKey, Signature, }; @@ -41,10 +41,23 @@ pub enum Error { BadCustodyReseeds, BadCustodyChallenges, BadCustodyResponses, - CommitteesError(CommitteesError), + BeaconStateError(BeaconStateError), SlotProcessingError(SlotProcessingError), } +#[derive(Debug, PartialEq)] +pub enum AttestationValidationError { + IncludedTooEarly, + IncludedTooLate, + WrongJustifiedSlot, + WrongJustifiedRoot, + BadLatestCrosslinkRoot, + BadSignature, + ShardBlockRootNotZero, + NoBlockRoot, + AttestationParticipantsError(AttestationParticipantsError), +} + macro_rules! ensure { ($condition: expr, $result: expr) => { if !$condition { @@ -390,9 +403,9 @@ impl From for Error { } } -impl From for Error { - fn from(e: CommitteesError) -> Error { - Error::CommitteesError(e) +impl From for Error { + fn from(e: BeaconStateError) -> Error { + Error::BeaconStateError(e) } } @@ -401,3 +414,9 @@ impl From for Error { Error::SlotProcessingError(e) } } + +impl From for AttestationValidationError { + fn from(e: AttestationParticipantsError) -> AttestationValidationError { + AttestationValidationError::AttestationParticipantsError(e) + } +} diff --git a/eth2/state_processing/src/epoch_processable.rs b/eth2/state_processing/src/epoch_processable.rs index aece61184..658449d6f 100644 --- a/eth2/state_processing/src/epoch_processable.rs +++ b/eth2/state_processing/src/epoch_processable.rs @@ -5,7 +5,7 @@ use ssz::TreeHash; use std::collections::{HashMap, HashSet}; use std::iter::FromIterator; use types::{ - beacon_state::{AttestationParticipantsError, CommitteesError, InclusionError}, + beacon_state::{AttestationParticipantsError, BeaconStateError, InclusionError}, validator_registry::get_active_validator_indices, BeaconState, ChainSpec, Crosslink, Epoch, Hash256, PendingAttestation, }; @@ -27,7 +27,7 @@ pub enum Error { NoBlockRoots, BaseRewardQuotientIsZero, NoRandaoSeed, - CommitteesError(CommitteesError), + BeaconStateError(BeaconStateError), AttestationParticipantsError(AttestationParticipantsError), InclusionError(InclusionError), WinningRootError(WinningRootError), @@ -559,9 +559,7 @@ impl EpochProcessable for BeaconState { self.current_epoch_start_shard = (self.current_epoch_start_shard + self.get_current_epoch_committee_count(spec) as u64) % spec.shard_count; - self.current_epoch_seed = self - .generate_seed(self.current_calculation_epoch, spec) - .ok_or_else(|| Error::NoRandaoSeed)?; + self.current_epoch_seed = self.generate_seed(self.current_calculation_epoch, spec)? } else { let epochs_since_last_registry_update = current_epoch - self.validator_registry_update_epoch; @@ -569,9 +567,8 @@ impl EpochProcessable for BeaconState { & epochs_since_last_registry_update.is_power_of_two() { self.current_calculation_epoch = next_epoch; - self.current_epoch_seed = self - .generate_seed(self.current_calculation_epoch, spec) - .ok_or_else(|| Error::NoRandaoSeed)?; + self.current_epoch_seed = + self.generate_seed(self.current_calculation_epoch, spec)? } } @@ -689,9 +686,9 @@ impl From for Error { } } -impl From for Error { - fn from(e: CommitteesError) -> Error { - Error::CommitteesError(e) +impl From for Error { + fn from(e: BeaconStateError) -> Error { + Error::BeaconStateError(e) } } diff --git a/eth2/state_processing/src/slot_processable.rs b/eth2/state_processing/src/slot_processable.rs index 7726c5071..9e3b611fd 100644 --- a/eth2/state_processing/src/slot_processable.rs +++ b/eth2/state_processing/src/slot_processable.rs @@ -1,9 +1,9 @@ use crate::{EpochProcessable, EpochProcessingError}; -use types::{beacon_state::CommitteesError, BeaconState, ChainSpec, Hash256}; +use types::{beacon_state::BeaconStateError, BeaconState, ChainSpec, Hash256}; #[derive(Debug, PartialEq)] pub enum Error { - CommitteesError(CommitteesError), + BeaconStateError(BeaconStateError), EpochProcessingError(EpochProcessingError), } @@ -49,9 +49,9 @@ fn merkle_root(_input: &[Hash256]) -> Hash256 { Hash256::zero() } -impl From for Error { - fn from(e: CommitteesError) -> Error { - Error::CommitteesError(e) +impl From for Error { + fn from(e: BeaconStateError) -> Error { + Error::BeaconStateError(e) } } diff --git a/eth2/types/Cargo.toml b/eth2/types/Cargo.toml index 5c5672297..099717a2c 100644 --- a/eth2/types/Cargo.toml +++ b/eth2/types/Cargo.toml @@ -19,3 +19,6 @@ serde_json = "1.0" slog = "^2.2.3" ssz = { path = "../utils/ssz" } vec_shuffle = { path = "../utils/vec_shuffle" } + +[dev-dependencies] +env_logger = "0.6.0" diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index ed53bfea9..112639792 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -9,48 +9,31 @@ use honey_badger_split::SplitExt; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; -use std::ops::Range; use vec_shuffle::shuffle; -pub enum Error { - InsufficientValidators, - BadBlockSignature, - InvalidEpoch(Slot, Range), - CommitteesError(CommitteesError), -} - #[derive(Debug, PartialEq)] -pub enum CommitteesError { - InvalidEpoch, - InsufficientNumberOfValidators, - BadRandao, +pub enum BeaconStateError { + EpochOutOfBounds, + InsufficientRandaoMixes, + InsufficientValidators, + InsufficientBlockRoots, + InsufficientIndexRoots, + InsufficientAttestations, + InsufficientCommittees, } #[derive(Debug, PartialEq)] pub enum InclusionError { - NoIncludedAttestations, + /// The validator did not participate in an attestation in this period. + NoAttestationsForValidator, AttestationParticipantsError(AttestationParticipantsError), } #[derive(Debug, PartialEq)] pub enum AttestationParticipantsError { + /// There is no committee for the given shard in the given epoch. NoCommitteeForShard, - NoCommittees, - BadBitfieldLength, - CommitteesError(CommitteesError), -} - -#[derive(Debug, PartialEq)] -pub enum AttestationValidationError { - IncludedTooEarly, - IncludedTooLate, - WrongJustifiedSlot, - WrongJustifiedRoot, - BadLatestCrosslinkRoot, - BadSignature, - ShardBlockRootNotZero, - NoBlockRoot, - AttestationParticipantsError(AttestationParticipantsError), + BeaconStateError(BeaconStateError), } macro_rules! safe_add_assign { @@ -111,7 +94,7 @@ impl BeaconState { initial_validator_deposits: Vec, latest_eth1_data: Eth1Data, spec: &ChainSpec, - ) -> BeaconState { + ) -> Result { let initial_crosslink = Crosslink { epoch: spec.genesis_epoch, shard_block_root: spec.zero_hash, @@ -195,11 +178,9 @@ impl BeaconState { )); genesis_state.latest_index_roots = vec![genesis_active_index_root; spec.latest_index_roots_length]; - genesis_state.current_epoch_seed = genesis_state - .generate_seed(spec.genesis_epoch, spec) - .expect("Unable to generate seed."); + genesis_state.current_epoch_seed = genesis_state.generate_seed(spec.genesis_epoch, spec)?; - genesis_state + Ok(genesis_state) } /// Return the tree hash root for this `BeaconState`. @@ -322,7 +303,7 @@ impl BeaconState { + 1; let latest_index_root = current_epoch + spec.entry_exit_delay; - if (epoch <= earliest_index_root) & (epoch >= latest_index_root) { + if (epoch >= earliest_index_root) & (epoch <= latest_index_root) { Some(self.latest_index_roots[epoch.as_usize() % spec.latest_index_roots_length]) } else { None @@ -332,12 +313,27 @@ impl BeaconState { /// Generate a seed for the given ``epoch``. /// /// Spec v0.2.0 - pub fn generate_seed(&self, epoch: Epoch, spec: &ChainSpec) -> Option { - let mut input = self.get_randao_mix(epoch, spec)?.to_vec(); - input.append(&mut self.get_active_index_root(epoch, spec)?.to_vec()); + pub fn generate_seed( + &self, + epoch: Epoch, + spec: &ChainSpec, + ) -> Result { + let mut input = self + .get_randao_mix(epoch, spec) + .ok_or_else(|| BeaconStateError::InsufficientRandaoMixes)? + .to_vec(); + + input.append( + &mut self + .get_active_index_root(epoch, spec) + .ok_or_else(|| BeaconStateError::InsufficientIndexRoots)? + .to_vec(), + ); + // TODO: ensure `Hash256::from(u64)` == `int_to_bytes32`. input.append(&mut Hash256::from(epoch.as_u64()).to_vec()); - Some(Hash256::from(&hash(&input[..])[..])) + + Ok(Hash256::from(&hash(&input[..])[..])) } /// Return the list of ``(committee, shard)`` tuples for the ``slot``. @@ -351,7 +347,7 @@ impl BeaconState { slot: Slot, registry_change: bool, spec: &ChainSpec, - ) -> Result, u64)>, CommitteesError> { + ) -> Result, u64)>, BeaconStateError> { let epoch = slot.epoch(spec.epoch_length); let current_epoch = self.current_epoch(spec); let previous_epoch = if current_epoch == spec.genesis_epoch { @@ -381,9 +377,7 @@ impl BeaconState { let epochs_since_last_registry_update = current_epoch - self.validator_registry_update_epoch; let (seed, shuffling_start_shard) = if registry_change { - let next_seed = self - .generate_seed(next_epoch, spec) - .ok_or_else(|| CommitteesError::BadRandao)?; + let next_seed = self.generate_seed(next_epoch, spec)?; ( next_seed, (self.current_epoch_start_shard + current_committees_per_epoch) @@ -392,9 +386,7 @@ impl BeaconState { } else if (epochs_since_last_registry_update > 1) & epochs_since_last_registry_update.is_power_of_two() { - let next_seed = self - .generate_seed(next_epoch, spec) - .ok_or_else(|| CommitteesError::BadRandao)?; + let next_seed = self.generate_seed(next_epoch, spec)?; (next_seed, self.current_epoch_start_shard) } else { (self.current_epoch_seed, self.current_epoch_start_shard) @@ -406,7 +398,7 @@ impl BeaconState { shuffling_start_shard, ) } else { - panic!("Epoch out-of-bounds.") + return Err(BeaconStateError::EpochOutOfBounds); }; let shuffling = self.get_shuffling(seed, shuffling_epoch, spec); @@ -434,7 +426,7 @@ impl BeaconState { &self, validator_index: usize, spec: &ChainSpec, - ) -> Result, CommitteesError> { + ) -> Result, BeaconStateError> { let mut result = None; for slot in self.current_epoch(spec).slot_iter(spec.epoch_length) { for (committee, shard) in self.get_crosslink_committees_at_slot(slot, false, spec)? { @@ -464,16 +456,15 @@ impl BeaconState { &self, slot: Slot, spec: &ChainSpec, - ) -> Result { + ) -> Result { let committees = self.get_crosslink_committees_at_slot(slot, false, spec)?; committees .first() - .ok_or(CommitteesError::InsufficientNumberOfValidators) + .ok_or(BeaconStateError::InsufficientValidators) .and_then(|(first_committee, _)| { let index = (slot.as_usize()) .checked_rem(first_committee.len()) - .ok_or(CommitteesError::InsufficientNumberOfValidators)?; - // NOTE: next index will not panic as we have already returned if this is the case. + .ok_or(BeaconStateError::InsufficientValidators)?; Ok(first_committee[index]) }) } @@ -682,7 +673,7 @@ impl BeaconState { &mut self, validator_index: usize, spec: &ChainSpec, - ) -> Result<(), CommitteesError> { + ) -> Result<(), BeaconStateError> { self.exit_validator(validator_index, spec); let current_epoch = self.current_epoch(spec); @@ -802,7 +793,7 @@ impl BeaconState { let earliest_attestation_index = included_attestations .iter() .min_by_key(|i| attestations[**i].inclusion_slot) - .ok_or_else(|| InclusionError::NoIncludedAttestations)?; + .ok_or_else(|| InclusionError::NoAttestationsForValidator)?; Ok(attestations[*earliest_attestation_index].clone()) } @@ -907,34 +898,18 @@ fn hash_tree_root(input: Vec) -> Hash256 { Hash256::from(&input.hash_tree_root()[..]) } -impl From for AttestationValidationError { - fn from(e: AttestationParticipantsError) -> AttestationValidationError { - AttestationValidationError::AttestationParticipantsError(e) +impl From for AttestationParticipantsError { + fn from(e: BeaconStateError) -> AttestationParticipantsError { + AttestationParticipantsError::BeaconStateError(e) } } -impl From for AttestationParticipantsError { - fn from(e: CommitteesError) -> AttestationParticipantsError { - AttestationParticipantsError::CommitteesError(e) - } -} - -/* - -*/ - impl From for InclusionError { fn from(e: AttestationParticipantsError) -> InclusionError { InclusionError::AttestationParticipantsError(e) } } -impl From for Error { - fn from(e: CommitteesError) -> Error { - Error::CommitteesError(e) - } -} - impl Encodable for BeaconState { fn ssz_append(&self, s: &mut SszStream) { s.append(&self.slot); diff --git a/eth2/types/src/beacon_state_tests.rs b/eth2/types/src/beacon_state_tests.rs new file mode 100644 index 000000000..bba99b2d2 --- /dev/null +++ b/eth2/types/src/beacon_state_tests.rs @@ -0,0 +1,74 @@ +use crate::{ + beacon_state::BeaconStateError, BeaconState, ChainSpec, Deposit, DepositData, DepositInput, + Eth1Data, Hash256, Keypair, +}; +use bls::create_proof_of_possession; + +struct BeaconStateTestBuilder { + pub genesis_time: u64, + pub initial_validator_deposits: Vec, + pub latest_eth1_data: Eth1Data, + pub spec: ChainSpec, + pub keypairs: Vec, +} + +impl BeaconStateTestBuilder { + pub fn with_random_validators(validator_count: usize) -> Self { + let genesis_time = 10_000_000; + let keypairs: Vec = (0..validator_count) + .collect::>() + .iter() + .map(|_| Keypair::random()) + .collect(); + let initial_validator_deposits = keypairs + .iter() + .map(|keypair| Deposit { + branch: vec![], // branch verification is not specified. + index: 0, // index verification is not specified. + deposit_data: DepositData { + amount: 32_000_000_000, // 32 ETH (in Gwei) + timestamp: genesis_time - 1, + deposit_input: DepositInput { + pubkey: keypair.pk.clone(), + withdrawal_credentials: Hash256::zero(), // Withdrawal not possible. + proof_of_possession: create_proof_of_possession(&keypair), + }, + }, + }) + .collect(); + let latest_eth1_data = Eth1Data { + deposit_root: Hash256::zero(), + block_hash: Hash256::zero(), + }; + let spec = ChainSpec::foundation(); + + Self { + genesis_time, + initial_validator_deposits, + latest_eth1_data, + spec, + keypairs, + } + } + + pub fn build(&self) -> Result { + BeaconState::genesis( + self.genesis_time, + self.initial_validator_deposits.clone(), + self.latest_eth1_data.clone(), + &self.spec, + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + pub fn can_produce_genesis_block() { + let builder = BeaconStateTestBuilder::with_random_validators(2); + + builder.build().unwrap(); + } +} diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index 233d1cc3e..d33c48532 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -7,6 +7,7 @@ pub mod attester_slashing; pub mod beacon_block; pub mod beacon_block_body; pub mod beacon_state; +pub mod beacon_state_tests; pub mod casper_slashing; pub mod crosslink; pub mod deposit; From 054c1f90476c99671ec1c1217d47b0eb5606d096 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 15 Feb 2019 16:55:52 +1100 Subject: [PATCH 20/30] Swap over to v0.2.0 shuffling algo --- eth2/types/Cargo.toml | 2 +- eth2/types/src/beacon_state.rs | 40 ++++++++++++++++++++++++---------- eth2/types/src/spec/mod.rs | 2 +- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/eth2/types/Cargo.toml b/eth2/types/Cargo.toml index fefd431a3..4c5be65b5 100644 --- a/eth2/types/Cargo.toml +++ b/eth2/types/Cargo.toml @@ -18,7 +18,7 @@ serde_derive = "1.0" serde_json = "1.0" slog = "^2.2.3" ssz = { path = "../utils/ssz" } -fisher_yates_shuffle = { path = "../utils/fisher_yates_shuffle" } +swap_or_not_shuffle = { path = "../utils/swap_or_not_shuffle" } [dev-dependencies] env_logger = "0.6.0" diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index d826344de..af815cbe2 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -5,15 +5,16 @@ use crate::{ PendingAttestation, PublicKey, Signature, Slot, Validator, }; use bls::verify_proof_of_possession; -use fisher_yates_shuffle::shuffle; use honey_badger_split::SplitExt; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use swap_or_not_shuffle::get_permutated_index; #[derive(Debug, PartialEq)] pub enum BeaconStateError { EpochOutOfBounds, + UnableToShuffle, InsufficientRandaoMixes, InsufficientValidators, InsufficientBlockRoots, @@ -249,23 +250,36 @@ impl BeaconState { /// committee is itself a list of validator indices. /// /// Spec v0.1 - pub fn get_shuffling(&self, seed: Hash256, epoch: Epoch, spec: &ChainSpec) -> Vec> { + pub fn get_shuffling( + &self, + seed: Hash256, + epoch: Epoch, + spec: &ChainSpec, + ) -> Option>> { let active_validator_indices = get_active_validator_indices(&self.validator_registry, epoch); let committees_per_epoch = self.get_epoch_committee_count(active_validator_indices.len(), spec); - // TODO: check that Hash256::from(u64) matches 'int_to_bytes32'. - let seed = seed ^ Hash256::from(epoch.as_u64()); - // TODO: fix `expect` assert. - let shuffled_active_validator_indices = - shuffle(&seed, active_validator_indices).expect("Max validator count exceed!"); + let mut shuffled_active_validator_indices = + Vec::with_capacity(active_validator_indices.len()); + for &i in &active_validator_indices { + let shuffled_i = get_permutated_index( + i, + active_validator_indices.len(), + &seed[..], + spec.shuffle_round_count, + )?; + shuffled_active_validator_indices[i] = active_validator_indices[shuffled_i] + } - shuffled_active_validator_indices - .honey_badger_split(committees_per_epoch as usize) - .map(|slice: &[usize]| slice.to_vec()) - .collect() + Some( + shuffled_active_validator_indices + .honey_badger_split(committees_per_epoch as usize) + .map(|slice: &[usize]| slice.to_vec()) + .collect(), + ) } /// Return the number of committees in the previous epoch. @@ -401,7 +415,9 @@ impl BeaconState { return Err(BeaconStateError::EpochOutOfBounds); }; - let shuffling = self.get_shuffling(seed, shuffling_epoch, spec); + let shuffling = self + .get_shuffling(seed, shuffling_epoch, spec) + .ok_or_else(|| BeaconStateError::UnableToShuffle)?; let offset = slot.as_u64() % spec.epoch_length; let committees_per_slot = committees_per_epoch / spec.epoch_length; let slot_start_shard = diff --git a/eth2/types/src/spec/mod.rs b/eth2/types/src/spec/mod.rs index 53c78a2c2..3632108ca 100644 --- a/eth2/types/src/spec/mod.rs +++ b/eth2/types/src/spec/mod.rs @@ -17,7 +17,7 @@ pub struct ChainSpec { pub beacon_chain_shard_number: u64, pub max_indices_per_slashable_vote: u64, pub max_withdrawals_per_epoch: u64, - pub shuffle_round_count: u64, + pub shuffle_round_count: u8, /* * Deposit contract From 1c57eb47f9aa8916b6ef6a677aee5a345d6e0af1 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 15 Feb 2019 17:07:52 +1100 Subject: [PATCH 21/30] Move `BeaconStateTestBuilder` into test mod --- eth2/types/src/beacon_state_tests.rs | 126 +++++++++++++-------------- 1 file changed, 62 insertions(+), 64 deletions(-) diff --git a/eth2/types/src/beacon_state_tests.rs b/eth2/types/src/beacon_state_tests.rs index bba99b2d2..cae815f33 100644 --- a/eth2/types/src/beacon_state_tests.rs +++ b/eth2/types/src/beacon_state_tests.rs @@ -1,69 +1,67 @@ -use crate::{ - beacon_state::BeaconStateError, BeaconState, ChainSpec, Deposit, DepositData, DepositInput, - Eth1Data, Hash256, Keypair, -}; -use bls::create_proof_of_possession; - -struct BeaconStateTestBuilder { - pub genesis_time: u64, - pub initial_validator_deposits: Vec, - pub latest_eth1_data: Eth1Data, - pub spec: ChainSpec, - pub keypairs: Vec, -} - -impl BeaconStateTestBuilder { - pub fn with_random_validators(validator_count: usize) -> Self { - let genesis_time = 10_000_000; - let keypairs: Vec = (0..validator_count) - .collect::>() - .iter() - .map(|_| Keypair::random()) - .collect(); - let initial_validator_deposits = keypairs - .iter() - .map(|keypair| Deposit { - branch: vec![], // branch verification is not specified. - index: 0, // index verification is not specified. - deposit_data: DepositData { - amount: 32_000_000_000, // 32 ETH (in Gwei) - timestamp: genesis_time - 1, - deposit_input: DepositInput { - pubkey: keypair.pk.clone(), - withdrawal_credentials: Hash256::zero(), // Withdrawal not possible. - proof_of_possession: create_proof_of_possession(&keypair), - }, - }, - }) - .collect(); - let latest_eth1_data = Eth1Data { - deposit_root: Hash256::zero(), - block_hash: Hash256::zero(), - }; - let spec = ChainSpec::foundation(); - - Self { - genesis_time, - initial_validator_deposits, - latest_eth1_data, - spec, - keypairs, - } - } - - pub fn build(&self) -> Result { - BeaconState::genesis( - self.genesis_time, - self.initial_validator_deposits.clone(), - self.latest_eth1_data.clone(), - &self.spec, - ) - } -} - #[cfg(test)] mod tests { - use super::*; + use crate::{ + beacon_state::BeaconStateError, BeaconState, ChainSpec, Deposit, DepositData, DepositInput, + Eth1Data, Hash256, Keypair, + }; + use bls::create_proof_of_possession; + + struct BeaconStateTestBuilder { + pub genesis_time: u64, + pub initial_validator_deposits: Vec, + pub latest_eth1_data: Eth1Data, + pub spec: ChainSpec, + pub keypairs: Vec, + } + + impl BeaconStateTestBuilder { + pub fn with_random_validators(validator_count: usize) -> Self { + let genesis_time = 10_000_000; + let keypairs: Vec = (0..validator_count) + .collect::>() + .iter() + .map(|_| Keypair::random()) + .collect(); + let initial_validator_deposits = keypairs + .iter() + .map(|keypair| Deposit { + branch: vec![], // branch verification is not specified. + index: 0, // index verification is not specified. + deposit_data: DepositData { + amount: 32_000_000_000, // 32 ETH (in Gwei) + timestamp: genesis_time - 1, + deposit_input: DepositInput { + pubkey: keypair.pk.clone(), + withdrawal_credentials: Hash256::zero(), // Withdrawal not possible. + proof_of_possession: create_proof_of_possession(&keypair), + }, + }, + }) + .collect(); + let latest_eth1_data = Eth1Data { + deposit_root: Hash256::zero(), + block_hash: Hash256::zero(), + }; + let spec = ChainSpec::foundation(); + + Self { + genesis_time, + initial_validator_deposits, + latest_eth1_data, + spec, + keypairs, + } + } + + pub fn build(&self) -> Result { + BeaconState::genesis( + self.genesis_time, + self.initial_validator_deposits.clone(), + self.latest_eth1_data.clone(), + &self.spec, + ) + } + } #[test] pub fn can_produce_genesis_block() { From 7c920cfb96136105771bac06b518c8c9efc03f01 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 15 Feb 2019 19:23:22 +1100 Subject: [PATCH 22/30] Add incomplete progress on fixing test harness --- beacon_node/beacon_chain/src/beacon_chain.rs | 6 +++ .../beacon_chain/src/cached_beacon_state.rs | 49 +++++++++++++++++++ beacon_node/beacon_chain/src/lib.rs | 1 + .../test_harness/src/beacon_chain_harness.rs | 1 + .../beacon_chain/test_harness/tests/chain.rs | 17 +++---- eth2/types/src/beacon_state.rs | 45 ++++++++++++++++- eth2/types/src/spec/few_validators.rs | 21 ++++++++ eth2/types/src/spec/mod.rs | 1 + 8 files changed, 128 insertions(+), 13 deletions(-) create mode 100644 beacon_node/beacon_chain/src/cached_beacon_state.rs create mode 100644 eth2/types/src/spec/few_validators.rs diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index c16337fd4..6c7a8c751 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -253,6 +253,7 @@ where /// Information is read from the present `beacon_state` shuffling, so only information from the /// present and prior epoch is available. pub fn block_proposer(&self, slot: Slot) -> Result { + trace!("BeaconChain::block_proposer: slot: {}", slot); let index = self .state .read() @@ -274,6 +275,10 @@ where &self, validator_index: usize, ) -> Result, BeaconStateError> { + trace!( + "BeaconChain::validator_attestion_slot_and_shard: validator_index: {}", + validator_index + ); if let Some((slot, shard, _committee)) = self .state .read() @@ -287,6 +292,7 @@ where /// Produce an `AttestationData` that is valid for the present `slot` and given `shard`. pub fn produce_attestation_data(&self, shard: u64) -> Result { + trace!("BeaconChain::produce_attestation_data: shard: {}", shard); let justified_epoch = self.justified_epoch(); let justified_block_root = *self .state diff --git a/beacon_node/beacon_chain/src/cached_beacon_state.rs b/beacon_node/beacon_chain/src/cached_beacon_state.rs new file mode 100644 index 000000000..4717d1744 --- /dev/null +++ b/beacon_node/beacon_chain/src/cached_beacon_state.rs @@ -0,0 +1,49 @@ +use types::{beacon_state::BeaconStateError, BeaconState, ChainSpec, Epoch, Slot}; + +pub const CACHED_EPOCHS: usize = 3; // previous, current, next. + +pub type CrosslinkCommittees = Vec<(Vec, u64)>; + +pub struct CachedBeaconState<'a> { + state: BeaconState, + crosslinks: Vec>, + spec: &'a ChainSpec, +} + +impl<'a> CachedBeaconState<'a> { + pub fn from_beacon_state( + state: BeaconState, + spec: &'a ChainSpec, + ) -> Result { + let current_epoch = state.current_epoch(spec); + let previous_epoch = if current_epoch == spec.genesis_epoch { + current_epoch + } else { + current_epoch.saturating_sub(1_u64) + }; + let next_epoch = state.next_epoch(spec); + + let mut crosslinks: Vec> = Vec::with_capacity(3); + crosslinks.push(committees_for_all_slots(&state, previous_epoch, spec)?); + crosslinks.push(committees_for_all_slots(&state, current_epoch, spec)?); + crosslinks.push(committees_for_all_slots(&state, next_epoch, spec)?); + + Ok(Self { + state, + crosslinks, + spec, + }) + } +} + +fn committees_for_all_slots( + state: &BeaconState, + epoch: Epoch, + spec: &ChainSpec, +) -> Result, BeaconStateError> { + let mut crosslinks: Vec = Vec::with_capacity(spec.epoch_length as usize); + for slot in epoch.slot_iter(spec.epoch_length) { + crosslinks.push(state.get_crosslink_committees_at_slot(slot, false, spec)?) + } + Ok(crosslinks) +} diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 4dac0b672..bc9085fbe 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -1,5 +1,6 @@ mod attestation_aggregator; mod beacon_chain; +mod cached_beacon_state; mod checkpoint; pub use self::beacon_chain::{BeaconChain, Error}; diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index acba2e015..5ea681ca3 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -209,6 +209,7 @@ impl BeaconChainHarness { self.increment_beacon_chain_slot(); // Produce a new block. + debug!("Producing block..."); let block = self.produce_block(); debug!("Submitting block for processing..."); self.beacon_chain.process_block(block).unwrap(); diff --git a/beacon_node/beacon_chain/test_harness/tests/chain.rs b/beacon_node/beacon_chain/test_harness/tests/chain.rs index 8be6f2a26..13e276abb 100644 --- a/beacon_node/beacon_chain/test_harness/tests/chain.rs +++ b/beacon_node/beacon_chain/test_harness/tests/chain.rs @@ -1,19 +1,14 @@ use env_logger::{Builder, Env}; use log::debug; use test_harness::BeaconChainHarness; -use types::{ChainSpec, Slot}; +use types::ChainSpec; #[test] -#[ignore] fn it_can_build_on_genesis_block() { - let mut spec = ChainSpec::foundation(); - spec.genesis_slot = Slot::new(spec.epoch_length * 8); + Builder::from_env(Env::default().default_filter_or("trace")).init(); + let spec = ChainSpec::few_validators(); - /* - spec.shard_count = spec.shard_count / 8; - spec.target_committee_size = spec.target_committee_size / 8; - */ - let validator_count = 1000; + let validator_count = 8; let mut harness = BeaconChainHarness::new(spec, validator_count as usize); @@ -23,7 +18,7 @@ fn it_can_build_on_genesis_block() { #[test] #[ignore] fn it_can_produce_past_first_epoch_boundary() { - Builder::from_env(Env::default().default_filter_or("debug")).init(); + Builder::from_env(Env::default().default_filter_or("trace")).init(); let validator_count = 100; @@ -33,7 +28,7 @@ fn it_can_produce_past_first_epoch_boundary() { debug!("Harness built, tests starting.."); - let blocks = harness.spec.epoch_length * 3 + 1; + let blocks = harness.spec.epoch_length * 2 + 1; for i in 0..blocks { harness.advance_chain_with_block(); diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index af815cbe2..aa6690705 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -6,6 +6,7 @@ use crate::{ }; use bls::verify_proof_of_possession; use honey_badger_split::SplitExt; +use log::trace; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; @@ -259,11 +260,24 @@ impl BeaconState { let active_validator_indices = get_active_validator_indices(&self.validator_registry, epoch); + if active_validator_indices.is_empty() { + return None; + } + + trace!( + "BeaconState::get_shuffling: active_validator_indices.len() == {}", + active_validator_indices.len() + ); + let committees_per_epoch = self.get_epoch_committee_count(active_validator_indices.len(), spec); - let mut shuffled_active_validator_indices = - Vec::with_capacity(active_validator_indices.len()); + trace!( + "BeaconState::get_shuffling: active_validator_indices.len() == {}, committees_per_epoch: {}", + active_validator_indices.len(), committees_per_epoch + ); + + let mut shuffled_active_validator_indices = vec![0; active_validator_indices.len()]; for &i in &active_validator_indices { let shuffled_i = get_permutated_index( i, @@ -317,9 +331,17 @@ impl BeaconState { + 1; let latest_index_root = current_epoch + spec.entry_exit_delay; + trace!( + "BeaconState::get_active_index_root: epoch: {}, earliest: {}, latest: {}", + epoch, + earliest_index_root, + latest_index_root + ); + if (epoch >= earliest_index_root) & (epoch <= latest_index_root) { Some(self.latest_index_roots[epoch.as_usize() % spec.latest_index_roots_length]) } else { + trace!("BeaconState::get_active_index_root: epoch out of range."); None } } @@ -371,8 +393,14 @@ impl BeaconState { }; let next_epoch = self.next_epoch(spec); + trace!( + "BeaconState::get_crosslink_committees_at_slot: epoch: {}", + epoch + ); + let (committees_per_epoch, seed, shuffling_epoch, shuffling_start_shard) = if epoch == previous_epoch { + trace!("BeaconState::get_crosslink_committees_at_slot: epoch == previous_epoch"); ( self.get_previous_epoch_committee_count(spec), self.previous_epoch_seed, @@ -380,6 +408,7 @@ impl BeaconState { self.previous_epoch_start_shard, ) } else if epoch == current_epoch { + trace!("BeaconState::get_crosslink_committees_at_slot: epoch == current_epoch"); ( self.get_current_epoch_committee_count(spec), self.current_epoch_seed, @@ -387,6 +416,7 @@ impl BeaconState { self.current_epoch_start_shard, ) } else if epoch == next_epoch { + trace!("BeaconState::get_crosslink_committees_at_slot: epoch == next_epoch"); let current_committees_per_epoch = self.get_current_epoch_committee_count(spec); let epochs_since_last_registry_update = current_epoch - self.validator_registry_update_epoch; @@ -423,6 +453,12 @@ impl BeaconState { let slot_start_shard = (shuffling_start_shard + committees_per_slot * offset) % spec.shard_count; + trace!( + "BeaconState::get_crosslink_committees_at_slot: committees_per_slot: {}, slot_start_shard: {}", + committees_per_slot, + slot_start_shard + ); + let mut crosslinks_at_slot = vec![]; for i in 0..committees_per_slot { let tuple = ( @@ -474,6 +510,11 @@ impl BeaconState { spec: &ChainSpec, ) -> Result { let committees = self.get_crosslink_committees_at_slot(slot, false, spec)?; + trace!( + "get_beacon_proposer_index: slot: {}, committees_count: {}", + slot, + committees.len() + ); committees .first() .ok_or(BeaconStateError::InsufficientValidators) diff --git a/eth2/types/src/spec/few_validators.rs b/eth2/types/src/spec/few_validators.rs new file mode 100644 index 000000000..05fe4c6cd --- /dev/null +++ b/eth2/types/src/spec/few_validators.rs @@ -0,0 +1,21 @@ +use crate::{ChainSpec, Slot}; + +impl ChainSpec { + /// Returns a `ChainSpec` compatible with the specification suitable for 8 validators. + /// + /// Spec v0.2.0 + pub fn few_validators() -> Self { + let genesis_slot = Slot::new(2_u64.pow(19)); + let epoch_length = 8; + let genesis_epoch = genesis_slot.epoch(epoch_length); + + Self { + shard_count: 1, + target_committee_size: 1, + genesis_slot, + genesis_epoch, + epoch_length, + ..ChainSpec::foundation() + } + } +} diff --git a/eth2/types/src/spec/mod.rs b/eth2/types/src/spec/mod.rs index 3632108ca..3e983a148 100644 --- a/eth2/types/src/spec/mod.rs +++ b/eth2/types/src/spec/mod.rs @@ -1,3 +1,4 @@ +mod few_validators; mod foundation; use crate::{Address, Epoch, Hash256, Slot}; From 3b92b69028021537ba98340c9552bf183957c822 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 16 Feb 2019 11:04:12 +1100 Subject: [PATCH 23/30] Apply CachedBeaconState to some functions - Attestation aggregation - Getting attestation duties --- .../src/attestation_aggregator.rs | 8 +- beacon_node/beacon_chain/src/beacon_chain.rs | 29 +++- .../beacon_chain/src/cached_beacon_state.rs | 138 +++++++++++++++--- .../beacon_chain/test_harness/tests/chain.rs | 9 +- 4 files changed, 151 insertions(+), 33 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_aggregator.rs b/beacon_node/beacon_chain/src/attestation_aggregator.rs index 6fbc11612..fa2ec87ab 100644 --- a/beacon_node/beacon_chain/src/attestation_aggregator.rs +++ b/beacon_node/beacon_chain/src/attestation_aggregator.rs @@ -1,3 +1,4 @@ +use crate::cached_beacon_state::CachedBeaconState; use state_processing::validate_attestation_without_signature; use std::collections::{HashMap, HashSet}; use types::{ @@ -76,12 +77,12 @@ impl AttestationAggregator { /// - The signature is verified against that of the validator at `validator_index`. pub fn process_free_attestation( &mut self, - state: &BeaconState, + cached_state: &CachedBeaconState, free_attestation: &FreeAttestation, spec: &ChainSpec, ) -> Result { let (slot, shard, committee_index) = some_or_invalid!( - state.attestation_slot_and_shard_for_validator( + cached_state.attestation_slot_and_shard_for_validator( free_attestation.validator_index as usize, spec, )?, @@ -104,7 +105,8 @@ impl AttestationAggregator { let signable_message = free_attestation.data.signable_message(PHASE_0_CUSTODY_BIT); let validator_record = some_or_invalid!( - state + cached_state + .state .validator_registry .get(free_attestation.validator_index as usize), Message::BadValidatorIndex diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 6c7a8c751..b2d041654 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1,4 +1,5 @@ use crate::attestation_aggregator::{AttestationAggregator, Outcome as AggregationOutcome}; +use crate::cached_beacon_state::CachedBeaconState; use crate::checkpoint::CheckPoint; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, @@ -69,6 +70,7 @@ pub struct BeaconChain { canonical_head: RwLock, finalized_head: RwLock, pub state: RwLock, + pub cached_state: RwLock, pub spec: ChainSpec, pub fork_choice: RwLock, } @@ -107,6 +109,11 @@ where let block_root = genesis_block.canonical_root(); block_store.put(&block_root, &ssz_encode(&genesis_block)[..])?; + let cached_state = RwLock::new(CachedBeaconState::from_beacon_state( + genesis_state.clone(), + spec.clone(), + )?); + let finalized_head = RwLock::new(CheckPoint::new( genesis_block.clone(), block_root, @@ -127,6 +134,7 @@ where slot_clock, attestation_aggregator, state: RwLock::new(genesis_state.clone()), + cached_state, finalized_head, canonical_head, spec, @@ -280,7 +288,7 @@ where validator_index ); if let Some((slot, shard, _committee)) = self - .state + .cached_state .read() .attestation_slot_and_shard_for_validator(validator_index, &self.spec)? { @@ -338,9 +346,7 @@ where let aggregation_outcome = self .attestation_aggregator .write() - .process_free_attestation(&self.state.read(), &free_attestation, &self.spec)?; - // TODO: Check this comment - //.map_err(|e| e.into())?; + .process_free_attestation(&self.cached_state.read(), &free_attestation, &self.spec)?; // return if the attestation is invalid if !aggregation_outcome.valid { @@ -495,6 +501,9 @@ where ); // Update the local state variable. *self.state.write() = state.clone(); + // Update the cached state variable. + *self.cached_state.write() = + CachedBeaconState::from_beacon_state(state.clone(), self.spec.clone())?; } Ok(BlockProcessingOutcome::ValidBlock(ValidBlock::Processed)) @@ -543,9 +552,15 @@ where }, }; - state - .per_block_processing_without_verifying_block_signature(&block, &self.spec) - .ok()?; + trace!("BeaconChain::produce_block: updating state for new block.",); + + let result = + state.per_block_processing_without_verifying_block_signature(&block, &self.spec); + trace!( + "BeaconNode::produce_block: state processing result: {:?}", + result + ); + result.ok()?; let state_root = state.canonical_root(); diff --git a/beacon_node/beacon_chain/src/cached_beacon_state.rs b/beacon_node/beacon_chain/src/cached_beacon_state.rs index 4717d1744..fec1d7c06 100644 --- a/beacon_node/beacon_chain/src/cached_beacon_state.rs +++ b/beacon_node/beacon_chain/src/cached_beacon_state.rs @@ -1,49 +1,149 @@ +use log::debug; +use std::collections::HashMap; use types::{beacon_state::BeaconStateError, BeaconState, ChainSpec, Epoch, Slot}; -pub const CACHED_EPOCHS: usize = 3; // previous, current, next. +pub const CACHE_PREVIOUS: bool = false; +pub const CACHE_CURRENT: bool = true; +pub const CACHE_NEXT: bool = false; pub type CrosslinkCommittees = Vec<(Vec, u64)>; +pub type Shard = u64; +pub type CommitteeIndex = u64; +pub type AttestationDuty = (Slot, Shard, CommitteeIndex); +pub type AttestationDutyMap = HashMap; -pub struct CachedBeaconState<'a> { - state: BeaconState, - crosslinks: Vec>, - spec: &'a ChainSpec, +// TODO: CachedBeaconState is presently duplicating `BeaconState` and `ChainSpec`. This is a +// massive memory waste, switch them to references. + +pub struct CachedBeaconState { + pub state: BeaconState, + committees: Vec>, + attestation_duties: Vec, + next_epoch: Epoch, + current_epoch: Epoch, + previous_epoch: Epoch, + spec: ChainSpec, } -impl<'a> CachedBeaconState<'a> { +impl CachedBeaconState { pub fn from_beacon_state( state: BeaconState, - spec: &'a ChainSpec, + spec: ChainSpec, ) -> Result { - let current_epoch = state.current_epoch(spec); + let current_epoch = state.current_epoch(&spec); let previous_epoch = if current_epoch == spec.genesis_epoch { current_epoch } else { current_epoch.saturating_sub(1_u64) }; - let next_epoch = state.next_epoch(spec); + let next_epoch = state.next_epoch(&spec); - let mut crosslinks: Vec> = Vec::with_capacity(3); - crosslinks.push(committees_for_all_slots(&state, previous_epoch, spec)?); - crosslinks.push(committees_for_all_slots(&state, current_epoch, spec)?); - crosslinks.push(committees_for_all_slots(&state, next_epoch, spec)?); + let mut committees: Vec> = Vec::with_capacity(3); + let mut attestation_duties: Vec = Vec::with_capacity(3); + + if CACHE_PREVIOUS { + debug!("CachedBeaconState::from_beacon_state: building previous epoch cache."); + let cache = build_epoch_cache(&state, previous_epoch, &spec)?; + committees.push(cache.committees); + attestation_duties.push(cache.attestation_duty_map); + } else { + committees.push(vec![]); + attestation_duties.push(HashMap::new()); + } + if CACHE_CURRENT { + debug!("CachedBeaconState::from_beacon_state: building current epoch cache."); + let cache = build_epoch_cache(&state, current_epoch, &spec)?; + committees.push(cache.committees); + attestation_duties.push(cache.attestation_duty_map); + } else { + committees.push(vec![]); + attestation_duties.push(HashMap::new()); + } + if CACHE_NEXT { + debug!("CachedBeaconState::from_beacon_state: building next epoch cache."); + let cache = build_epoch_cache(&state, next_epoch, &spec)?; + committees.push(cache.committees); + attestation_duties.push(cache.attestation_duty_map); + } else { + committees.push(vec![]); + attestation_duties.push(HashMap::new()); + } Ok(Self { state, - crosslinks, + committees, + attestation_duties, + next_epoch, + current_epoch, + previous_epoch, spec, }) } + + fn slot_to_cache_index(&self, slot: Slot) -> Option { + match slot.epoch(self.spec.epoch_length) { + epoch if (epoch == self.previous_epoch) & CACHE_PREVIOUS => Some(0), + epoch if (epoch == self.current_epoch) & CACHE_CURRENT => Some(1), + epoch if (epoch == self.next_epoch) & CACHE_NEXT => Some(2), + _ => None, + } + } + + /// Returns the `slot`, `shard` and `committee_index` for which a validator must produce an + /// attestation. + /// + /// Cached method. + /// + /// Spec v0.2.0 + pub fn attestation_slot_and_shard_for_validator( + &self, + validator_index: usize, + _spec: &ChainSpec, + ) -> Result, BeaconStateError> { + // Get the result for this epoch. + let cache_index = self + .slot_to_cache_index(self.state.slot) + .expect("Current epoch should always have a cache index."); + + let duties = self.attestation_duties[cache_index] + .get(&(validator_index as u64)) + .and_then(|tuple| Some(*tuple)); + + Ok(duties) + } } -fn committees_for_all_slots( +struct EpochCacheResult { + committees: Vec, + attestation_duty_map: AttestationDutyMap, +} + +fn build_epoch_cache( state: &BeaconState, epoch: Epoch, spec: &ChainSpec, -) -> Result, BeaconStateError> { - let mut crosslinks: Vec = Vec::with_capacity(spec.epoch_length as usize); +) -> Result { + let mut epoch_committees: Vec = + Vec::with_capacity(spec.epoch_length as usize); + let mut attestation_duty_map: AttestationDutyMap = HashMap::new(); + for slot in epoch.slot_iter(spec.epoch_length) { - crosslinks.push(state.get_crosslink_committees_at_slot(slot, false, spec)?) + let slot_committees = state.get_crosslink_committees_at_slot(slot, false, spec)?; + + for (committee, shard) in slot_committees { + for (committee_index, validator_index) in committee.iter().enumerate() { + attestation_duty_map.insert( + *validator_index as u64, + (slot, shard, committee_index as u64), + ); + } + } + + epoch_committees.push(state.get_crosslink_committees_at_slot(slot, false, spec)?) } - Ok(crosslinks) + + Ok(EpochCacheResult { + committees: epoch_committees, + attestation_duty_map, + }) } diff --git a/beacon_node/beacon_chain/test_harness/tests/chain.rs b/beacon_node/beacon_chain/test_harness/tests/chain.rs index 13e276abb..a57a0161f 100644 --- a/beacon_node/beacon_chain/test_harness/tests/chain.rs +++ b/beacon_node/beacon_chain/test_harness/tests/chain.rs @@ -6,8 +6,8 @@ use types::ChainSpec; #[test] fn it_can_build_on_genesis_block() { Builder::from_env(Env::default().default_filter_or("trace")).init(); - let spec = ChainSpec::few_validators(); + let spec = ChainSpec::few_validators(); let validator_count = 8; let mut harness = BeaconChainHarness::new(spec, validator_count as usize); @@ -18,13 +18,14 @@ fn it_can_build_on_genesis_block() { #[test] #[ignore] fn it_can_produce_past_first_epoch_boundary() { - Builder::from_env(Env::default().default_filter_or("trace")).init(); + Builder::from_env(Env::default().default_filter_or("debug")).init(); - let validator_count = 100; + let spec = ChainSpec::few_validators(); + let validator_count = 8; debug!("Starting harness build..."); - let mut harness = BeaconChainHarness::new(ChainSpec::foundation(), validator_count); + let mut harness = BeaconChainHarness::new(spec, validator_count); debug!("Harness built, tests starting.."); From 5e6acb8f39bf72e2754cb8a0f75f5d8e1799cb75 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 16 Feb 2019 11:19:47 +1100 Subject: [PATCH 24/30] Break `BeaconState` tests into separate file --- eth2/types/src/beacon_state.rs | 32 +-------- eth2/types/src/beacon_state/tests.rs | 97 ++++++++++++++++++++++++++++ eth2/types/src/beacon_state_tests.rs | 72 --------------------- eth2/types/src/lib.rs | 1 - 4 files changed, 99 insertions(+), 103 deletions(-) create mode 100644 eth2/types/src/beacon_state/tests.rs delete mode 100644 eth2/types/src/beacon_state_tests.rs diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index af815cbe2..22885adfe 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -11,6 +11,8 @@ use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; use swap_or_not_shuffle::get_permutated_index; +mod tests; + #[derive(Debug, PartialEq)] pub enum BeaconStateError { EpochOutOfBounds, @@ -1080,33 +1082,3 @@ impl TestRandom for BeaconState { } } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; - - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = BeaconState::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = BeaconState::random_for_test(&mut rng); - - let result = original.hash_tree_root(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } -} diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs new file mode 100644 index 000000000..e069e462e --- /dev/null +++ b/eth2/types/src/beacon_state/tests.rs @@ -0,0 +1,97 @@ +#![cfg(test)] + +use super::*; +use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; +use crate::{ + beacon_state::BeaconStateError, BeaconState, ChainSpec, Deposit, DepositData, DepositInput, + Eth1Data, Hash256, Keypair, +}; +use bls::create_proof_of_possession; +use ssz::ssz_encode; + +struct BeaconStateTestBuilder { + pub genesis_time: u64, + pub initial_validator_deposits: Vec, + pub latest_eth1_data: Eth1Data, + pub spec: ChainSpec, + pub keypairs: Vec, +} + +impl BeaconStateTestBuilder { + pub fn with_random_validators(validator_count: usize) -> Self { + let genesis_time = 10_000_000; + let keypairs: Vec = (0..validator_count) + .collect::>() + .iter() + .map(|_| Keypair::random()) + .collect(); + let initial_validator_deposits = keypairs + .iter() + .map(|keypair| Deposit { + branch: vec![], // branch verification is not specified. + index: 0, // index verification is not specified. + deposit_data: DepositData { + amount: 32_000_000_000, // 32 ETH (in Gwei) + timestamp: genesis_time - 1, + deposit_input: DepositInput { + pubkey: keypair.pk.clone(), + withdrawal_credentials: Hash256::zero(), // Withdrawal not possible. + proof_of_possession: create_proof_of_possession(&keypair), + }, + }, + }) + .collect(); + let latest_eth1_data = Eth1Data { + deposit_root: Hash256::zero(), + block_hash: Hash256::zero(), + }; + let spec = ChainSpec::foundation(); + + Self { + genesis_time, + initial_validator_deposits, + latest_eth1_data, + spec, + keypairs, + } + } + + pub fn build(&self) -> Result { + BeaconState::genesis( + self.genesis_time, + self.initial_validator_deposits.clone(), + self.latest_eth1_data.clone(), + &self.spec, + ) + } +} + +#[test] +pub fn can_produce_genesis_block() { + let builder = BeaconStateTestBuilder::with_random_validators(2); + + builder.build().unwrap(); +} + +#[test] +pub fn test_ssz_round_trip() { + let mut rng = XorShiftRng::from_seed([42; 16]); + let original = BeaconState::random_for_test(&mut rng); + + let bytes = ssz_encode(&original); + let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); + + assert_eq!(original, decoded); +} + +#[test] +pub fn test_hash_tree_root() { + let mut rng = XorShiftRng::from_seed([42; 16]); + let original = BeaconState::random_for_test(&mut rng); + + let result = original.hash_tree_root(); + + assert_eq!(result.len(), 32); + // TODO: Add further tests + // https://github.com/sigp/lighthouse/issues/170 +} diff --git a/eth2/types/src/beacon_state_tests.rs b/eth2/types/src/beacon_state_tests.rs deleted file mode 100644 index cae815f33..000000000 --- a/eth2/types/src/beacon_state_tests.rs +++ /dev/null @@ -1,72 +0,0 @@ -#[cfg(test)] -mod tests { - use crate::{ - beacon_state::BeaconStateError, BeaconState, ChainSpec, Deposit, DepositData, DepositInput, - Eth1Data, Hash256, Keypair, - }; - use bls::create_proof_of_possession; - - struct BeaconStateTestBuilder { - pub genesis_time: u64, - pub initial_validator_deposits: Vec, - pub latest_eth1_data: Eth1Data, - pub spec: ChainSpec, - pub keypairs: Vec, - } - - impl BeaconStateTestBuilder { - pub fn with_random_validators(validator_count: usize) -> Self { - let genesis_time = 10_000_000; - let keypairs: Vec = (0..validator_count) - .collect::>() - .iter() - .map(|_| Keypair::random()) - .collect(); - let initial_validator_deposits = keypairs - .iter() - .map(|keypair| Deposit { - branch: vec![], // branch verification is not specified. - index: 0, // index verification is not specified. - deposit_data: DepositData { - amount: 32_000_000_000, // 32 ETH (in Gwei) - timestamp: genesis_time - 1, - deposit_input: DepositInput { - pubkey: keypair.pk.clone(), - withdrawal_credentials: Hash256::zero(), // Withdrawal not possible. - proof_of_possession: create_proof_of_possession(&keypair), - }, - }, - }) - .collect(); - let latest_eth1_data = Eth1Data { - deposit_root: Hash256::zero(), - block_hash: Hash256::zero(), - }; - let spec = ChainSpec::foundation(); - - Self { - genesis_time, - initial_validator_deposits, - latest_eth1_data, - spec, - keypairs, - } - } - - pub fn build(&self) -> Result { - BeaconState::genesis( - self.genesis_time, - self.initial_validator_deposits.clone(), - self.latest_eth1_data.clone(), - &self.spec, - ) - } - } - - #[test] - pub fn can_produce_genesis_block() { - let builder = BeaconStateTestBuilder::with_random_validators(2); - - builder.build().unwrap(); - } -} diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index d33c48532..233d1cc3e 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -7,7 +7,6 @@ pub mod attester_slashing; pub mod beacon_block; pub mod beacon_block_body; pub mod beacon_state; -pub mod beacon_state_tests; pub mod casper_slashing; pub mod crosslink; pub mod deposit; From b0513b1ec1839826ad64704b5457d47adcc8d257 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 16 Feb 2019 15:08:33 +1100 Subject: [PATCH 25/30] Add and update logs --- .../beacon_chain/src/cached_beacon_state.rs | 9 ++-- .../test_harness/src/beacon_chain_harness.rs | 13 ++++- .../beacon_chain/test_harness/tests/chain.rs | 2 +- .../state_processing/src/block_processable.rs | 13 ++--- .../state_processing/src/epoch_processable.rs | 13 +++++ eth2/types/src/beacon_state.rs | 49 ++++++++----------- 6 files changed, 59 insertions(+), 40 deletions(-) diff --git a/beacon_node/beacon_chain/src/cached_beacon_state.rs b/beacon_node/beacon_chain/src/cached_beacon_state.rs index fec1d7c06..e14e9fe99 100644 --- a/beacon_node/beacon_chain/src/cached_beacon_state.rs +++ b/beacon_node/beacon_chain/src/cached_beacon_state.rs @@ -1,4 +1,4 @@ -use log::debug; +use log::{debug, trace}; use std::collections::HashMap; use types::{beacon_state::BeaconStateError, BeaconState, ChainSpec, Epoch, Slot}; @@ -42,7 +42,7 @@ impl CachedBeaconState { let mut attestation_duties: Vec = Vec::with_capacity(3); if CACHE_PREVIOUS { - debug!("CachedBeaconState::from_beacon_state: building previous epoch cache."); + debug!("from_beacon_state: building previous epoch cache."); let cache = build_epoch_cache(&state, previous_epoch, &spec)?; committees.push(cache.committees); attestation_duties.push(cache.attestation_duty_map); @@ -51,7 +51,7 @@ impl CachedBeaconState { attestation_duties.push(HashMap::new()); } if CACHE_CURRENT { - debug!("CachedBeaconState::from_beacon_state: building current epoch cache."); + debug!("from_beacon_state: building current epoch cache."); let cache = build_epoch_cache(&state, current_epoch, &spec)?; committees.push(cache.committees); attestation_duties.push(cache.attestation_duty_map); @@ -60,7 +60,7 @@ impl CachedBeaconState { attestation_duties.push(HashMap::new()); } if CACHE_NEXT { - debug!("CachedBeaconState::from_beacon_state: building next epoch cache."); + debug!("from_beacon_state: building next epoch cache."); let cache = build_epoch_cache(&state, next_epoch, &spec)?; committees.push(cache.committees); attestation_duties.push(cache.attestation_duty_map); @@ -81,6 +81,7 @@ impl CachedBeaconState { } fn slot_to_cache_index(&self, slot: Slot) -> Option { + trace!("slot_to_cache_index: cache lookup"); match slot.epoch(self.spec.epoch_length) { epoch if (epoch == self.previous_epoch) & CACHE_PREVIOUS => Some(0), epoch if (epoch == self.current_epoch) & CACHE_CURRENT => Some(1), diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index 5ea681ca3..9d61952f0 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -128,7 +128,18 @@ impl BeaconChainHarness { pub fn increment_beacon_chain_slot(&mut self) -> Slot { let slot = self.beacon_chain.present_slot() + 1; - debug!("Incrementing BeaconChain slot to {}.", slot); + let nth_slot = slot + - slot + .epoch(self.spec.epoch_length) + .start_slot(self.spec.epoch_length); + let nth_epoch = slot.epoch(self.spec.epoch_length) - self.spec.genesis_epoch; + debug!( + "Advancing BeaconChain to slot {}, epoch {} (epoch height: {}, slot {} in epoch.).", + slot, + slot.epoch(self.spec.epoch_length), + nth_epoch, + nth_slot + ); self.beacon_chain.slot_clock.set_slot(slot.as_u64()); self.beacon_chain.advance_state(slot).unwrap(); diff --git a/beacon_node/beacon_chain/test_harness/tests/chain.rs b/beacon_node/beacon_chain/test_harness/tests/chain.rs index a57a0161f..b3354784f 100644 --- a/beacon_node/beacon_chain/test_harness/tests/chain.rs +++ b/beacon_node/beacon_chain/test_harness/tests/chain.rs @@ -33,7 +33,7 @@ fn it_can_produce_past_first_epoch_boundary() { for i in 0..blocks { harness.advance_chain_with_block(); - debug!("Produced block {}/{}.", i, blocks); + debug!("Produced block {}/{}.", i + 1, blocks); } let dump = harness.chain_dump().expect("Chain dump failed."); diff --git a/eth2/state_processing/src/block_processable.rs b/eth2/state_processing/src/block_processable.rs index 904d2fac5..539711c69 100644 --- a/eth2/state_processing/src/block_processable.rs +++ b/eth2/state_processing/src/block_processable.rs @@ -1,7 +1,7 @@ use crate::SlotProcessingError; use hashing::hash; use int_to_bytes::int_to_bytes32; -use log::debug; +use log::{debug, trace}; use ssz::{ssz_encode, TreeHash}; use types::{ beacon_state::{AttestationParticipantsError, BeaconStateError}, @@ -219,6 +219,8 @@ fn per_block_processing_signature_optional( Error::MaxAttestationsExceeded ); + debug!("Verifying {} attestations.", block.body.attestations.len()); + for attestation in &block.body.attestations { validate_attestation(&state, attestation, spec)?; @@ -231,11 +233,6 @@ fn per_block_processing_signature_optional( state.latest_attestations.push(pending_attestation); } - debug!( - "{} attestations verified & processed.", - block.body.attestations.len() - ); - /* * Deposits */ @@ -312,6 +309,10 @@ fn validate_attestation_signature_optional( spec: &ChainSpec, verify_signature: bool, ) -> Result<(), AttestationValidationError> { + trace!( + "validate_attestation_signature_optional: attestation epoch: {}", + attestation.data.slot.epoch(spec.epoch_length) + ); ensure!( attestation.data.slot + spec.min_attestation_inclusion_delay <= state.slot, AttestationValidationError::IncludedTooEarly diff --git a/eth2/state_processing/src/epoch_processable.rs b/eth2/state_processing/src/epoch_processable.rs index 658449d6f..80318914c 100644 --- a/eth2/state_processing/src/epoch_processable.rs +++ b/eth2/state_processing/src/epoch_processable.rs @@ -315,6 +315,11 @@ impl EpochProcessable for BeaconState { // for slot in self.slot.saturating_sub(2 * spec.epoch_length)..self.slot { for slot in self.previous_epoch(spec).slot_iter(spec.epoch_length) { + trace!( + "Finding winning root for slot: {} (epoch: {})", + slot, + slot.epoch(spec.epoch_length) + ); let crosslink_committees_at_slot = self.get_crosslink_committees_at_slot(slot, false, spec)?; @@ -539,6 +544,12 @@ impl EpochProcessable for BeaconState { */ self.previous_calculation_epoch = self.current_calculation_epoch; self.previous_epoch_start_shard = self.current_epoch_start_shard; + + debug!( + "setting previous_epoch_seed to : {}", + self.current_epoch_seed + ); + self.previous_epoch_seed = self.current_epoch_seed; let should_update_validator_registy = if self.finalized_epoch @@ -553,6 +564,7 @@ impl EpochProcessable for BeaconState { }; if should_update_validator_registy { + trace!("updating validator registry."); self.update_validator_registry(spec); self.current_calculation_epoch = next_epoch; @@ -561,6 +573,7 @@ impl EpochProcessable for BeaconState { % spec.shard_count; self.current_epoch_seed = self.generate_seed(self.current_calculation_epoch, spec)? } else { + trace!("not updating validator registry."); let epochs_since_last_registry_update = current_epoch - self.validator_registry_update_epoch; if (epochs_since_last_registry_update > 1) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index aa6690705..6a4033d5c 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -265,7 +265,7 @@ impl BeaconState { } trace!( - "BeaconState::get_shuffling: active_validator_indices.len() == {}", + "get_shuffling: active_validator_indices.len() == {}", active_validator_indices.len() ); @@ -273,8 +273,9 @@ impl BeaconState { self.get_epoch_committee_count(active_validator_indices.len(), spec); trace!( - "BeaconState::get_shuffling: active_validator_indices.len() == {}, committees_per_epoch: {}", - active_validator_indices.len(), committees_per_epoch + "get_shuffling: active_validator_indices.len() == {}, committees_per_epoch: {}", + active_validator_indices.len(), + committees_per_epoch ); let mut shuffled_active_validator_indices = vec![0; active_validator_indices.len()]; @@ -332,7 +333,7 @@ impl BeaconState { let latest_index_root = current_epoch + spec.entry_exit_delay; trace!( - "BeaconState::get_active_index_root: epoch: {}, earliest: {}, latest: {}", + "get_active_index_root: epoch: {}, earliest: {}, latest: {}", epoch, earliest_index_root, latest_index_root @@ -341,7 +342,7 @@ impl BeaconState { if (epoch >= earliest_index_root) & (epoch <= latest_index_root) { Some(self.latest_index_roots[epoch.as_usize() % spec.latest_index_roots_length]) } else { - trace!("BeaconState::get_active_index_root: epoch out of range."); + trace!("get_active_index_root: epoch out of range."); None } } @@ -386,37 +387,28 @@ impl BeaconState { ) -> Result, u64)>, BeaconStateError> { let epoch = slot.epoch(spec.epoch_length); let current_epoch = self.current_epoch(spec); - let previous_epoch = if current_epoch == spec.genesis_epoch { - current_epoch - } else { - current_epoch.saturating_sub(1_u64) - }; + let previous_epoch = self.previous_epoch(spec); let next_epoch = self.next_epoch(spec); - trace!( - "BeaconState::get_crosslink_committees_at_slot: epoch: {}", - epoch - ); - let (committees_per_epoch, seed, shuffling_epoch, shuffling_start_shard) = - if epoch == previous_epoch { - trace!("BeaconState::get_crosslink_committees_at_slot: epoch == previous_epoch"); - ( - self.get_previous_epoch_committee_count(spec), - self.previous_epoch_seed, - self.previous_calculation_epoch, - self.previous_epoch_start_shard, - ) - } else if epoch == current_epoch { - trace!("BeaconState::get_crosslink_committees_at_slot: epoch == current_epoch"); + if epoch == current_epoch { + trace!("get_crosslink_committees_at_slot: current_epoch"); ( self.get_current_epoch_committee_count(spec), self.current_epoch_seed, self.current_calculation_epoch, self.current_epoch_start_shard, ) + } else if epoch == previous_epoch { + trace!("get_crosslink_committees_at_slot: previous_epoch"); + ( + self.get_previous_epoch_committee_count(spec), + self.previous_epoch_seed, + self.previous_calculation_epoch, + self.previous_epoch_start_shard, + ) } else if epoch == next_epoch { - trace!("BeaconState::get_crosslink_committees_at_slot: epoch == next_epoch"); + trace!("get_crosslink_committees_at_slot: next_epoch"); let current_committees_per_epoch = self.get_current_epoch_committee_count(spec); let epochs_since_last_registry_update = current_epoch - self.validator_registry_update_epoch; @@ -454,9 +446,10 @@ impl BeaconState { (shuffling_start_shard + committees_per_slot * offset) % spec.shard_count; trace!( - "BeaconState::get_crosslink_committees_at_slot: committees_per_slot: {}, slot_start_shard: {}", + "get_crosslink_committees_at_slot: committees_per_slot: {}, slot_start_shard: {}, seed: {}", committees_per_slot, - slot_start_shard + slot_start_shard, + seed ); let mut crosslinks_at_slot = vec![]; From c5158e297494dcae1c992fe592121698bd1c2176 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 16 Feb 2019 15:08:57 +1100 Subject: [PATCH 26/30] Fix bug with total_balance in epoch processing --- eth2/state_processing/src/epoch_processable.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eth2/state_processing/src/epoch_processable.rs b/eth2/state_processing/src/epoch_processable.rs index 80318914c..a930cd750 100644 --- a/eth2/state_processing/src/epoch_processable.rs +++ b/eth2/state_processing/src/epoch_processable.rs @@ -144,8 +144,10 @@ impl EpochProcessable for BeaconState { let previous_epoch_attester_indices = self.get_attestation_participants_union(&previous_epoch_attestations[..], spec)?; - let previous_total_balance = - self.get_total_balance(&previous_epoch_attester_indices[..], spec); + let previous_total_balance = self.get_total_balance( + &get_active_validator_indices(&self.validator_registry, previous_epoch), + spec, + ); /* * Validators targetting the previous justified slot From b79f0cdf68a26666bbd2bac6e578bf59627ff76c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 16 Feb 2019 15:09:14 +1100 Subject: [PATCH 27/30] Fix bug with reward quotient in epoch processing --- eth2/state_processing/src/epoch_processable.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eth2/state_processing/src/epoch_processable.rs b/eth2/state_processing/src/epoch_processable.rs index a930cd750..11b2b224d 100644 --- a/eth2/state_processing/src/epoch_processable.rs +++ b/eth2/state_processing/src/epoch_processable.rs @@ -359,7 +359,8 @@ impl EpochProcessable for BeaconState { /* * Rewards and Penalities */ - let base_reward_quotient = previous_total_balance.integer_sqrt(); + let base_reward_quotient = + previous_total_balance.integer_sqrt() / spec.base_reward_quotient; if base_reward_quotient == 0 { return Err(Error::BaseRewardQuotientIsZero); } From f83d02b394d7cc40e5ce5647c286d061bb2f9599 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 16 Feb 2019 15:09:43 +1100 Subject: [PATCH 28/30] Update previous epoch function --- eth2/types/src/beacon_state.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 6a4033d5c..2e1df45f2 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -203,7 +203,12 @@ impl BeaconState { /// /// Spec v0.2.0 pub fn previous_epoch(&self, spec: &ChainSpec) -> Epoch { - self.current_epoch(spec).saturating_sub(1_u64) + let current_epoch = self.current_epoch(&spec); + if current_epoch == spec.genesis_epoch { + current_epoch + } else { + current_epoch - 1 + } } /// The epoch following `self.current_epoch()`. From c4bedc03a8f704cb10e6c8da689e3f9b06362c1a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 17 Feb 2019 20:21:13 +1100 Subject: [PATCH 29/30] Fix file org. inconsistency in `types` --- .../src/{spec/foundation.rs => chain_spec.rs} | 111 +++++++++++++++++- eth2/types/src/lib.rs | 4 +- eth2/types/src/spec/few_validators.rs | 21 ---- eth2/types/src/spec/mod.rs | 93 --------------- 4 files changed, 112 insertions(+), 117 deletions(-) rename eth2/types/src/{spec/foundation.rs => chain_spec.rs} (54%) delete mode 100644 eth2/types/src/spec/few_validators.rs delete mode 100644 eth2/types/src/spec/mod.rs diff --git a/eth2/types/src/spec/foundation.rs b/eth2/types/src/chain_spec.rs similarity index 54% rename from eth2/types/src/spec/foundation.rs rename to eth2/types/src/chain_spec.rs index 79abe4061..b5d5689e3 100644 --- a/eth2/types/src/spec/foundation.rs +++ b/eth2/types/src/chain_spec.rs @@ -1,7 +1,96 @@ -use crate::{Address, ChainSpec, Epoch, Hash256, Signature, Slot}; +use crate::{Address, Epoch, Hash256, Slot}; +use bls::Signature; const GWEI: u64 = 1_000_000_000; +/// Holds all the "constants" for a BeaconChain. +/// +/// Spec v0.2.0 +#[derive(PartialEq, Debug, Clone)] +pub struct ChainSpec { + /* + * Misc + */ + pub shard_count: u64, + pub target_committee_size: u64, + pub max_balance_churn_quotient: u64, + pub beacon_chain_shard_number: u64, + pub max_indices_per_slashable_vote: u64, + pub max_withdrawals_per_epoch: u64, + pub shuffle_round_count: u8, + + /* + * Deposit contract + */ + pub deposit_contract_address: Address, + pub deposit_contract_tree_depth: u64, + + /* + * Gwei values + */ + pub min_deposit_amount: u64, + pub max_deposit_amount: u64, + pub fork_choice_balance_increment: u64, + pub ejection_balance: u64, + + /* + * Initial Values + */ + pub genesis_fork_version: u64, + pub genesis_slot: Slot, + pub genesis_epoch: Epoch, + pub genesis_start_shard: u64, + pub far_future_epoch: Epoch, + pub zero_hash: Hash256, + pub empty_signature: Signature, + pub bls_withdrawal_prefix_byte: u8, + + /* + * Time parameters + */ + pub slot_duration: u64, + pub min_attestation_inclusion_delay: u64, + pub epoch_length: u64, + pub seed_lookahead: Epoch, + pub entry_exit_delay: u64, + pub eth1_data_voting_period: u64, + pub min_validator_withdrawal_epochs: Epoch, + + /* + * State list lengths + */ + pub latest_block_roots_length: usize, + pub latest_randao_mixes_length: usize, + pub latest_index_roots_length: usize, + pub latest_penalized_exit_length: usize, + + /* + * Reward and penalty quotients + */ + pub base_reward_quotient: u64, + pub whistleblower_reward_quotient: u64, + pub includer_reward_quotient: u64, + pub inactivity_penalty_quotient: u64, + + /* + * Max operations per block + */ + pub max_proposer_slashings: u64, + pub max_attester_slashings: u64, + pub max_attestations: u64, + pub max_deposits: u64, + pub max_exits: u64, + + /* + * Signature domains + */ + pub domain_deposit: u64, + pub domain_attestation: u64, + pub domain_proposal: u64, + pub domain_exit: u64, + pub domain_randao: u64, +} + impl ChainSpec { /// Returns a `ChainSpec` compatible with the specification from Ethereum Foundation. /// @@ -100,6 +189,26 @@ impl ChainSpec { } } +impl ChainSpec { + /// Returns a `ChainSpec` compatible with the specification suitable for 8 validators. + /// + /// Spec v0.2.0 + pub fn few_validators() -> Self { + let genesis_slot = Slot::new(2_u64.pow(19)); + let epoch_length = 8; + let genesis_epoch = genesis_slot.epoch(epoch_length); + + Self { + shard_count: 1, + target_committee_size: 1, + genesis_slot, + genesis_epoch, + epoch_length, + ..ChainSpec::foundation() + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index 233d1cc3e..f2c128440 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -8,6 +8,7 @@ pub mod beacon_block; pub mod beacon_block_body; pub mod beacon_state; pub mod casper_slashing; +pub mod chain_spec; pub mod crosslink; pub mod deposit; pub mod deposit_data; @@ -28,7 +29,6 @@ pub mod slashable_vote_data; pub mod slot_epoch_macros; pub mod slot_epoch; pub mod slot_height; -pub mod spec; pub mod validator; pub mod validator_registry; pub mod validator_registry_delta_block; @@ -44,6 +44,7 @@ pub use crate::beacon_block::BeaconBlock; pub use crate::beacon_block_body::BeaconBlockBody; pub use crate::beacon_state::BeaconState; pub use crate::casper_slashing::CasperSlashing; +pub use crate::chain_spec::ChainSpec; pub use crate::crosslink::Crosslink; pub use crate::deposit::Deposit; pub use crate::deposit_data::DepositData; @@ -60,7 +61,6 @@ pub use crate::slashable_attestation::SlashableAttestation; pub use crate::slashable_vote_data::SlashableVoteData; pub use crate::slot_epoch::{Epoch, Slot}; pub use crate::slot_height::SlotHeight; -pub use crate::spec::ChainSpec; pub use crate::validator::{StatusFlags as ValidatorStatusFlags, Validator}; pub use crate::validator_registry_delta_block::ValidatorRegistryDeltaBlock; diff --git a/eth2/types/src/spec/few_validators.rs b/eth2/types/src/spec/few_validators.rs deleted file mode 100644 index 05fe4c6cd..000000000 --- a/eth2/types/src/spec/few_validators.rs +++ /dev/null @@ -1,21 +0,0 @@ -use crate::{ChainSpec, Slot}; - -impl ChainSpec { - /// Returns a `ChainSpec` compatible with the specification suitable for 8 validators. - /// - /// Spec v0.2.0 - pub fn few_validators() -> Self { - let genesis_slot = Slot::new(2_u64.pow(19)); - let epoch_length = 8; - let genesis_epoch = genesis_slot.epoch(epoch_length); - - Self { - shard_count: 1, - target_committee_size: 1, - genesis_slot, - genesis_epoch, - epoch_length, - ..ChainSpec::foundation() - } - } -} diff --git a/eth2/types/src/spec/mod.rs b/eth2/types/src/spec/mod.rs deleted file mode 100644 index 3e983a148..000000000 --- a/eth2/types/src/spec/mod.rs +++ /dev/null @@ -1,93 +0,0 @@ -mod few_validators; -mod foundation; - -use crate::{Address, Epoch, Hash256, Slot}; -use bls::Signature; - -/// Holds all the "constants" for a BeaconChain. -/// -/// Spec v0.2.0 -#[derive(PartialEq, Debug, Clone)] -pub struct ChainSpec { - /* - * Misc - */ - pub shard_count: u64, - pub target_committee_size: u64, - pub max_balance_churn_quotient: u64, - pub beacon_chain_shard_number: u64, - pub max_indices_per_slashable_vote: u64, - pub max_withdrawals_per_epoch: u64, - pub shuffle_round_count: u8, - - /* - * Deposit contract - */ - pub deposit_contract_address: Address, - pub deposit_contract_tree_depth: u64, - - /* - * Gwei values - */ - pub min_deposit_amount: u64, - pub max_deposit_amount: u64, - pub fork_choice_balance_increment: u64, - pub ejection_balance: u64, - - /* - * Initial Values - */ - pub genesis_fork_version: u64, - pub genesis_slot: Slot, - pub genesis_epoch: Epoch, - pub genesis_start_shard: u64, - pub far_future_epoch: Epoch, - pub zero_hash: Hash256, - pub empty_signature: Signature, - pub bls_withdrawal_prefix_byte: u8, - - /* - * Time parameters - */ - pub slot_duration: u64, - pub min_attestation_inclusion_delay: u64, - pub epoch_length: u64, - pub seed_lookahead: Epoch, - pub entry_exit_delay: u64, - pub eth1_data_voting_period: u64, - pub min_validator_withdrawal_epochs: Epoch, - - /* - * State list lengths - */ - pub latest_block_roots_length: usize, - pub latest_randao_mixes_length: usize, - pub latest_index_roots_length: usize, - pub latest_penalized_exit_length: usize, - - /* - * Reward and penalty quotients - */ - pub base_reward_quotient: u64, - pub whistleblower_reward_quotient: u64, - pub includer_reward_quotient: u64, - pub inactivity_penalty_quotient: u64, - - /* - * Max operations per block - */ - pub max_proposer_slashings: u64, - pub max_attester_slashings: u64, - pub max_attestations: u64, - pub max_deposits: u64, - pub max_exits: u64, - - /* - * Signature domains - */ - pub domain_deposit: u64, - pub domain_attestation: u64, - pub domain_proposal: u64, - pub domain_exit: u64, - pub domain_randao: u64, -} From dc0696754be06da921b2536af1771f9bdbc9d200 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 17 Feb 2019 20:23:31 +1100 Subject: [PATCH 30/30] Raise log level on test_harness tests --- beacon_node/beacon_chain/test_harness/tests/chain.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/tests/chain.rs b/beacon_node/beacon_chain/test_harness/tests/chain.rs index b3354784f..1a08ffcf1 100644 --- a/beacon_node/beacon_chain/test_harness/tests/chain.rs +++ b/beacon_node/beacon_chain/test_harness/tests/chain.rs @@ -5,7 +5,7 @@ use types::ChainSpec; #[test] fn it_can_build_on_genesis_block() { - Builder::from_env(Env::default().default_filter_or("trace")).init(); + Builder::from_env(Env::default().default_filter_or("info")).init(); let spec = ChainSpec::few_validators(); let validator_count = 8; @@ -18,7 +18,7 @@ fn it_can_build_on_genesis_block() { #[test] #[ignore] fn it_can_produce_past_first_epoch_boundary() { - Builder::from_env(Env::default().default_filter_or("debug")).init(); + Builder::from_env(Env::default().default_filter_or("info")).init(); let spec = ChainSpec::few_validators(); let validator_count = 8;