diff --git a/README.md b/README.md index 7759c1166..6da6732ad 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,19 @@ If you'd like some background on Sigma Prime, please see the [Lighthouse Update \#00](https://lighthouse.sigmaprime.io/update-00.html) blog post or the [company website](https://sigmaprime.io). +### Directory Structure + +- [`beacon_node/`](beacon_node/): the "Beacon Node" binary and crates exclusively + associated with it. +- [`docs/`](docs/): documentation related to the repository. This includes contributor + guides, etc. (It does not include code documentation, which can be produced with `cargo doc`). +- [`eth2/`](eth2/): Crates containing common logic across the Lighthouse project. For + example: Ethereum 2.0 types ([`BeaconBlock`](eth2/types/src/beacon_block.rs), [`BeaconState`](eth2/types/src/beacon_state.rs), etc) and + SimpleSerialize (SSZ). +- [`protos/`](protos/): protobuf/gRPC definitions that are common across the Lighthouse project. +- [`validator_client/`](validator_client/): the "Validator Client" binary and crates exclusively + associated with it. + ### Components The following list describes some of the components actively under development @@ -61,7 +74,7 @@ by the team: from the Ethereum Foundation to develop *simpleserialize* (SSZ), a purpose-built serialization format for sending information across a network. Check out the [SSZ -implementation](https://github.com/sigp/lighthouse/tree/master/beacon_chain/utils/ssz) +implementation](https://github.com/ethereum/eth2.0-specs/blob/00aa553fee95963b74fbec84dbd274d7247b8a0e/specs/simple-serialize.md) and this [research](https://github.com/sigp/serialization_sandbox/blob/report/report/serialization_report.md) on serialization formats for more information. @@ -79,16 +92,6 @@ In addition to these components we are also working on database schemas, RPC frameworks, specification development, database optimizations (e.g., bloom-filters), and tons of other interesting stuff (at least we think so). -### Directory Structure - -Here we provide an overview of the directory structure: - -- `beacon_chain/`: contains logic derived directly from the specification. - E.g., shuffling algorithms, state transition logic and structs, block -validation, BLS crypto, etc. -- `lighthouse/`: contains logic specific to this client implementation. E.g., - CLI parsing, RPC end-points, databases, etc. - ### Running **NOTE: The cryptography libraries used in this implementation are diff --git a/eth2/README.md b/eth2/README.md new file mode 100644 index 000000000..cf041e987 --- /dev/null +++ b/eth2/README.md @@ -0,0 +1,37 @@ +# Ethereum 2.0 Common Crates + +Rust crates containing logic common across the Lighthouse project. + +## Per-Crate Summary + +- [`attester/`](attester/): Core logic for attesting to beacon and shard blocks. +- [`block_proposer/`](block_proposer/): Core logic for proposing beacon blocks. +- [`fork_choice/`](fork_choice/): A collection of fork-choice algorithms for + the Beacon Chain. +- [`state_processing/`](state_processing/): Provides per-slot, per-block, and + per-epoch state processing. +- [`types/`](types/): Defines base Ethereum 2.0 types (e.g., `BeaconBlock`, + `BeaconState`, etc). +- [`utils/`](utils/): + - [`bls`](utils/bls/): A wrapper for an external BLS encryption library. + - [`boolean-bitfield`](utils/boolean-bitfield/): Provides an expandable vector + of bools, specifically for use in Eth2. + - [`fisher-yates-shuffle`](utils/fisher-yates-shuffle/): shuffles a list + pseudo-randomly. + - [`hashing`](utils/hashing/): A wrapper for external hashing libraries. + - [`honey-badger-split`](utils/honey-badger-split/): Splits a list in `n` + parts without giving AF about the length of the list, `n`, or anything + else. + - [`int-to-bytes`](utils/int-to-bytes/): Simple library which converts ints + into byte-strings of various lengths. + - [`slot_clock`](utils/slot_clock/): translates the system time into Beacon + Chain "slots". (Also provides another slot clock that's useful during + testing.) + - [`ssz`](utils/ssz/): an implementation of the SimpleSerialize + serialization/deserialization protocol used by Eth 2.0. + - [`ssz_derive`](utils/ssz_derive/): provides procedural macros for + deriving SSZ `Encodable`, `Decodable`, and `TreeHash` methods. + - [`swap_or_not_shuffle`](utils/swap_or_not_shuffle/): a list-shuffling + method which is slow, but allows for a subset of indices to be shuffled. + - [`test_random_derive`](utils/test_random_derive/): provides procedural + macros for deriving the `TestRandom` trait defined in `types`. diff --git a/eth2/fork_choice/src/bitwise_lmd_ghost.rs b/eth2/fork_choice/src/bitwise_lmd_ghost.rs index e1d246e92..1e66de079 100644 --- a/eth2/fork_choice/src/bitwise_lmd_ghost.rs +++ b/eth2/fork_choice/src/bitwise_lmd_ghost.rs @@ -192,7 +192,7 @@ where } // Check if there is a clear block winner at this height. If so return it. for (hash, votes) in current_votes.iter() { - if *votes >= total_vote_count / 2 { + if *votes > total_vote_count / 2 { // we have a clear winner, return it return Some(*hash); } @@ -371,7 +371,10 @@ impl ForkChoice for BitwiseLMDGhost { // if there are no children, we are done, return the current_head let children = match self.children.get(¤t_head) { Some(children) => children.clone(), - None => return Ok(current_head), + None => { + debug!("Head found: {}", current_head); + return Ok(current_head); + } }; // logarithmic lookup blocks to see if there are obvious winners, if so, @@ -391,7 +394,7 @@ impl ForkChoice for BitwiseLMDGhost { step /= 2; } if step > 0 { - trace!("Found clear winner in log lookup"); + trace!("Found clear winner: {}", current_head); } // if our skip lookup failed and we only have one child, progress to that child else if children.len() == 1 { diff --git a/eth2/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml b/eth2/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml index 1578673cd..3233137ab 100644 --- a/eth2/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml +++ b/eth2/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml @@ -35,3 +35,31 @@ test_cases: - b3: 3 heads: - id: 'b2' +- blocks: + - id: 'b0' + parent: 'b0' + - id: 'b1' + parent: 'b0' + - id: 'b2' + parent: 'b0' + - id: 'b3' + parent: 'b1' + - id: 'b4' + parent: 'b1' + - id: 'b5' + parent: 'b1' + - id: 'b6' + parent: 'b2' + - id: 'b7' + parent: 'b6' + weights: + - b0: 0 + - b1: 3 + - b2: 2 + - b3: 1 + - b4: 1 + - b5: 1 + - b6: 2 + - b7: 2 + heads: + - id: 'b4' diff --git a/eth2/state_processing/src/epoch_processable.rs b/eth2/state_processing/src/epoch_processable.rs index 2cee455a3..0ecd1bbd1 100644 --- a/eth2/state_processing/src/epoch_processable.rs +++ b/eth2/state_processing/src/epoch_processable.rs @@ -619,12 +619,6 @@ impl EpochProcessable for BeaconState { .cloned() .collect(); - /* - * Manage the beacon state caches - */ - self.advance_caches(); - self.build_epoch_cache(RelativeEpoch::Next, spec)?; - debug!("Epoch transition complete."); Ok(()) diff --git a/eth2/state_processing/src/slot_processable.rs b/eth2/state_processing/src/slot_processable.rs index 0bbc79ab0..6017f4c0a 100644 --- a/eth2/state_processing/src/slot_processable.rs +++ b/eth2/state_processing/src/slot_processable.rs @@ -26,6 +26,7 @@ where ) -> Result<(), Error> { if (self.slot + 1) % spec.epoch_length == 0 { self.per_epoch_processing(spec)?; + self.advance_caches(); } self.slot += 1; diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 1c486e9b6..6dfbf78ee 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -13,7 +13,7 @@ use rayon::prelude::*; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; use std::collections::HashMap; -use swap_or_not_shuffle::get_permutated_index; +use swap_or_not_shuffle::shuffle_list; pub use builder::BeaconStateBuilder; @@ -423,17 +423,16 @@ impl BeaconState { committees_per_epoch ); - let mut shuffled_active_validator_indices = vec![0; active_validator_indices.len()]; - for (i, _) in active_validator_indices.iter().enumerate() { - let shuffled_i = get_permutated_index( - i, - active_validator_indices.len(), - &seed[..], - spec.shuffle_round_count, - ) - .ok_or_else(|| Error::UnableToShuffle)?; - shuffled_active_validator_indices[i] = active_validator_indices[shuffled_i] - } + let active_validator_indices: Vec = + active_validator_indices.iter().cloned().collect(); + + let shuffled_active_validator_indices = shuffle_list( + active_validator_indices, + spec.shuffle_round_count, + &seed[..], + true, + ) + .ok_or_else(|| Error::UnableToShuffle)?; Ok(shuffled_active_validator_indices .honey_badger_split(committees_per_epoch as usize) diff --git a/eth2/utils/swap_or_not_shuffle/benches/benches.rs b/eth2/utils/swap_or_not_shuffle/benches/benches.rs index 1d5b5476c..0502e6fc4 100644 --- a/eth2/utils/swap_or_not_shuffle/benches/benches.rs +++ b/eth2/utils/swap_or_not_shuffle/benches/benches.rs @@ -1,6 +1,6 @@ use criterion::Criterion; use criterion::{black_box, criterion_group, criterion_main, Benchmark}; -use swap_or_not_shuffle::get_permutated_index; +use swap_or_not_shuffle::{get_permutated_index, shuffle_list as fast_shuffle}; const SHUFFLE_ROUND_COUNT: u8 = 90; @@ -48,6 +48,16 @@ fn shuffles(c: &mut Criterion) { .sample_size(10), ); + c.bench( + "_fast_ whole list shuffle", + Benchmark::new("512 elements", move |b| { + let seed = vec![42; 32]; + let list: Vec = (0..512).collect(); + b.iter(|| black_box(fast_shuffle(list.clone(), SHUFFLE_ROUND_COUNT, &seed, true))) + }) + .sample_size(10), + ); + c.bench( "whole list shuffle", Benchmark::new("16384 elements", move |b| { @@ -56,6 +66,26 @@ fn shuffles(c: &mut Criterion) { }) .sample_size(10), ); + + c.bench( + "_fast_ whole list shuffle", + Benchmark::new("16384 elements", move |b| { + let seed = vec![42; 32]; + let list: Vec = (0..16384).collect(); + b.iter(|| black_box(fast_shuffle(list.clone(), SHUFFLE_ROUND_COUNT, &seed, true))) + }) + .sample_size(10), + ); + + c.bench( + "_fast_ whole list shuffle", + Benchmark::new("4m elements", move |b| { + let seed = vec![42; 32]; + let list: Vec = (0..4_000_000).collect(); + b.iter(|| black_box(fast_shuffle(list.clone(), SHUFFLE_ROUND_COUNT, &seed, true))) + }) + .sample_size(10), + ); } criterion_group!(benches, shuffles,); diff --git a/eth2/utils/swap_or_not_shuffle/src/get_permutated_index.rs b/eth2/utils/swap_or_not_shuffle/src/get_permutated_index.rs new file mode 100644 index 000000000..37a82341e --- /dev/null +++ b/eth2/utils/swap_or_not_shuffle/src/get_permutated_index.rs @@ -0,0 +1,187 @@ +use bytes::Buf; +use hashing::hash; +use int_to_bytes::{int_to_bytes1, int_to_bytes4}; +use std::cmp::max; +use std::io::Cursor; + +/// Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy. +/// +/// Utilizes 'swap or not' shuffling found in +/// https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf +/// See the 'generalized domain' algorithm on page 3. +/// +/// Note: this function is significantly slower than the `shuffle_list` function in this crate. +/// Using `get_permutated_list` to shuffle an entire list, index by index, has been observed to be +/// 250x slower than `shuffle_list`. Therefore, this function is only useful when shuffling a small +/// portion of a much larger list. +/// +/// Returns `None` under any of the following conditions: +/// - `list_size == 0` +/// - `index >= list_size` +/// - `list_size > 2**24` +/// - `list_size > usize::max_value() / 2` +pub fn get_permutated_index( + index: usize, + list_size: usize, + seed: &[u8], + shuffle_round_count: u8, +) -> Option { + if list_size == 0 + || index >= list_size + || list_size > usize::max_value() / 2 + || list_size > 2_usize.pow(24) + { + return None; + } + + let mut index = index; + for round in 0..shuffle_round_count { + let pivot = bytes_to_int64(&hash_with_round(seed, round)[..]) as usize % list_size; + index = do_round(seed, index, pivot, round, list_size)?; + } + Some(index) +} + +fn do_round(seed: &[u8], index: usize, pivot: usize, round: u8, list_size: usize) -> Option { + let flip = (pivot + list_size - index) % list_size; + let position = max(index, flip); + let source = hash_with_round_and_position(seed, round, position)?; + let byte = source[(position % 256) / 8]; + let bit = (byte >> (position % 8)) % 2; + Some(if bit == 1 { flip } else { index }) +} + +fn hash_with_round_and_position(seed: &[u8], round: u8, position: usize) -> Option> { + let mut seed = seed.to_vec(); + seed.append(&mut int_to_bytes1(round)); + /* + * Note: the specification has an implicit assertion in `int_to_bytes4` that `position / 256 < + * 2**24`. For efficiency, we do not check for that here as it is checked in `get_permutated_index`. + */ + seed.append(&mut int_to_bytes4((position / 256) as u32)); + Some(hash(&seed[..])) +} + +fn hash_with_round(seed: &[u8], round: u8) -> Vec { + let mut seed = seed.to_vec(); + seed.append(&mut int_to_bytes1(round)); + hash(&seed[..]) +} + +fn bytes_to_int64(bytes: &[u8]) -> u64 { + let mut cursor = Cursor::new(bytes); + cursor.get_u64_le() +} + +#[cfg(test)] +mod tests { + use super::*; + use ethereum_types::H256 as Hash256; + use hex; + use std::{fs::File, io::prelude::*, path::PathBuf}; + use yaml_rust::yaml; + + #[test] + #[ignore] + fn fuzz_test() { + let max_list_size = 2_usize.pow(24); + let test_runs = 1000; + + // Test at max list_size with the end index. + for _ in 0..test_runs { + let index = max_list_size - 1; + let list_size = max_list_size; + let seed = Hash256::random(); + let shuffle_rounds = 90; + + assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some()); + } + + // Test at max list_size low indices. + for i in 0..test_runs { + let index = i; + let list_size = max_list_size; + let seed = Hash256::random(); + let shuffle_rounds = 90; + + assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some()); + } + + // Test at max list_size high indices. + for i in 0..test_runs { + let index = max_list_size - 1 - i; + let list_size = max_list_size; + let seed = Hash256::random(); + let shuffle_rounds = 90; + + assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some()); + } + } + + #[test] + fn returns_none_for_zero_length_list() { + assert_eq!(None, get_permutated_index(100, 0, &[42, 42], 90)); + } + + #[test] + fn returns_none_for_out_of_bounds_index() { + assert_eq!(None, get_permutated_index(100, 100, &[42, 42], 90)); + } + + #[test] + fn returns_none_for_too_large_list() { + assert_eq!( + None, + get_permutated_index(100, usize::max_value() / 2, &[42, 42], 90) + ); + } + + #[test] + fn test_vectors() { + /* + * Test vectors are generated here: + * + * https://github.com/ethereum/eth2.0-test-generators + */ + let mut file = { + let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + file_path_buf.push("src/specs/test_vector_permutated_index.yml"); + + File::open(file_path_buf).unwrap() + }; + + let mut yaml_str = String::new(); + + file.read_to_string(&mut yaml_str).unwrap(); + + let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap(); + let doc = &docs[0]; + let test_cases = doc["test_cases"].as_vec().unwrap(); + + for (i, test_case) in test_cases.iter().enumerate() { + let index = test_case["index"].as_i64().unwrap() as usize; + let list_size = test_case["list_size"].as_i64().unwrap() as usize; + let permutated_index = test_case["permutated_index"].as_i64().unwrap() as usize; + let shuffle_round_count = test_case["shuffle_round_count"].as_i64().unwrap(); + let seed_string = test_case["seed"].clone().into_string().unwrap(); + let seed = hex::decode(seed_string.replace("0x", "")).unwrap(); + + let shuffle_round_count = if shuffle_round_count < (u8::max_value() as i64) { + shuffle_round_count as u8 + } else { + panic!("shuffle_round_count must be a u8") + }; + + assert_eq!( + Some(permutated_index), + get_permutated_index(index, list_size, &seed[..], shuffle_round_count), + "Failure on case #{} index: {}, list_size: {}, round_count: {}, seed: {}", + i, + index, + list_size, + shuffle_round_count, + seed_string, + ); + } + } +} diff --git a/eth2/utils/swap_or_not_shuffle/src/lib.rs b/eth2/utils/swap_or_not_shuffle/src/lib.rs index 753265f3e..57049fbdf 100644 --- a/eth2/utils/swap_or_not_shuffle/src/lib.rs +++ b/eth2/utils/swap_or_not_shuffle/src/lib.rs @@ -1,178 +1,21 @@ -use bytes::Buf; -use hashing::hash; -use int_to_bytes::{int_to_bytes1, int_to_bytes4}; -use std::cmp::max; -use std::io::Cursor; +//! Provides list-shuffling functions matching the Ethereum 2.0 specification. +//! +//! See +//! [get_permutated_index](https://github.com/ethereum/eth2.0-specs/blob/0.4.0/specs/core/0_beacon-chain.md#get_permuted_index) +//! for specifications. +//! +//! There are two functions exported by this crate: +//! +//! - `get_permutated_index`: given a single index, computes the index resulting from a shuffle. +//! Runs in less time than it takes to run `shuffle_list`. +//! - `shuffle_list`: shuffles an entire list in-place. Runs in less time than it takes to run +//! `get_permutated_index` on each index. +//! +//! In general, use `get_permutated_list` to calculate the shuffling of a small subset of a much +//! larger list (~250x larger is a good guide, but solid figures yet to be calculated). -/// Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy. -/// -/// Utilizes 'swap or not' shuffling found in -/// https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf -/// See the 'generalized domain' algorithm on page 3. -/// -/// Returns `None` under any of the following conditions: -/// - `list_size == 0` -/// - `index >= list_size` -/// - `list_size > 2**24` -/// - `list_size > usize::max_value() / 2` -pub fn get_permutated_index( - index: usize, - list_size: usize, - seed: &[u8], - shuffle_round_count: u8, -) -> Option { - if list_size == 0 - || index >= list_size - || list_size > usize::max_value() / 2 - || list_size > 2_usize.pow(24) - { - return None; - } +mod get_permutated_index; +mod shuffle_list; - let mut index = index; - for round in 0..shuffle_round_count { - let pivot = bytes_to_int64(&hash_with_round(seed, round)[..]) as usize % list_size; - let flip = (pivot + list_size - index) % list_size; - let position = max(index, flip); - let source = hash_with_round_and_position(seed, round, position)?; - let byte = source[(position % 256) / 8]; - let bit = (byte >> (position % 8)) % 2; - index = if bit == 1 { flip } else { index } - } - Some(index) -} - -fn hash_with_round_and_position(seed: &[u8], round: u8, position: usize) -> Option> { - let mut seed = seed.to_vec(); - seed.append(&mut int_to_bytes1(round)); - /* - * Note: the specification has an implicit assertion in `int_to_bytes4` that `position / 256 < - * 2**24`. For efficiency, we do not check for that here as it is checked in `get_permutated_index`. - */ - seed.append(&mut int_to_bytes4((position / 256) as u32)); - Some(hash(&seed[..])) -} - -fn hash_with_round(seed: &[u8], round: u8) -> Vec { - let mut seed = seed.to_vec(); - seed.append(&mut int_to_bytes1(round)); - hash(&seed[..]) -} - -fn bytes_to_int64(bytes: &[u8]) -> u64 { - let mut cursor = Cursor::new(bytes); - cursor.get_u64_le() -} - -#[cfg(test)] -mod tests { - use super::*; - use ethereum_types::H256 as Hash256; - use hex; - use std::{fs::File, io::prelude::*, path::PathBuf}; - use yaml_rust::yaml; - - #[test] - #[ignore] - fn fuzz_test() { - let max_list_size = 2_usize.pow(24); - let test_runs = 1000; - - // Test at max list_size with the end index. - for _ in 0..test_runs { - let index = max_list_size - 1; - let list_size = max_list_size; - let seed = Hash256::random(); - let shuffle_rounds = 90; - - assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some()); - } - - // Test at max list_size low indices. - for i in 0..test_runs { - let index = i; - let list_size = max_list_size; - let seed = Hash256::random(); - let shuffle_rounds = 90; - - assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some()); - } - - // Test at max list_size high indices. - for i in 0..test_runs { - let index = max_list_size - 1 - i; - let list_size = max_list_size; - let seed = Hash256::random(); - let shuffle_rounds = 90; - - assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some()); - } - } - - #[test] - fn returns_none_for_zero_length_list() { - assert_eq!(None, get_permutated_index(100, 0, &[42, 42], 90)); - } - - #[test] - fn returns_none_for_out_of_bounds_index() { - assert_eq!(None, get_permutated_index(100, 100, &[42, 42], 90)); - } - - #[test] - fn returns_none_for_too_large_list() { - assert_eq!( - None, - get_permutated_index(100, usize::max_value() / 2, &[42, 42], 90) - ); - } - - #[test] - fn test_vectors() { - /* - * Test vectors are generated here: - * - * https://github.com/ethereum/eth2.0-test-generators - */ - let mut file = { - let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - file_path_buf.push("src/specs/test_vector_permutated_index.yml"); - - File::open(file_path_buf).unwrap() - }; - - let mut yaml_str = String::new(); - - file.read_to_string(&mut yaml_str).unwrap(); - - let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap(); - let doc = &docs[0]; - let test_cases = doc["test_cases"].as_vec().unwrap(); - - for (i, test_case) in test_cases.iter().enumerate() { - let index = test_case["index"].as_i64().unwrap() as usize; - let list_size = test_case["list_size"].as_i64().unwrap() as usize; - let permutated_index = test_case["permutated_index"].as_i64().unwrap() as usize; - let shuffle_round_count = test_case["shuffle_round_count"].as_i64().unwrap(); - let seed_string = test_case["seed"].clone().into_string().unwrap(); - let seed = hex::decode(seed_string.replace("0x", "")).unwrap(); - - let shuffle_round_count = if shuffle_round_count < (u8::max_value() as i64) { - shuffle_round_count as u8 - } else { - panic!("shuffle_round_count must be a u8") - }; - - assert_eq!( - Some(permutated_index), - get_permutated_index(index, list_size, &seed[..], shuffle_round_count), - "Failure on case #{} index: {}, list_size: {}, round_count: {}, seed: {}", - i, - index, - list_size, - shuffle_round_count, - seed_string, - ); - } - } -} +pub use get_permutated_index::get_permutated_index; +pub use shuffle_list::shuffle_list; diff --git a/eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs b/eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs new file mode 100644 index 000000000..e7e1e18e6 --- /dev/null +++ b/eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs @@ -0,0 +1,178 @@ +use bytes::Buf; +use hashing::hash; +use int_to_bytes::int_to_bytes4; +use std::io::Cursor; + +const SEED_SIZE: usize = 32; +const ROUND_SIZE: usize = 1; +const POSITION_WINDOW_SIZE: usize = 4; +const PIVOT_VIEW_SIZE: usize = SEED_SIZE + ROUND_SIZE; +const TOTAL_SIZE: usize = SEED_SIZE + ROUND_SIZE + POSITION_WINDOW_SIZE; + +/// Shuffles an entire list in-place. +/// +/// Note: this is equivalent to the `get_permutated_index` function, except it shuffles an entire +/// list not just a single index. With large lists this function has been observed to be 250x +/// faster than running `get_permutated_index` across an entire list. +/// +/// Credits to [@protolambda](https://github.com/protolambda) for defining this algorithm. +/// +/// Shuffles if `forwards == true`, otherwise un-shuffles. +/// +/// Returns `None` under any of the following conditions: +/// - `list_size == 0` +/// - `list_size > 2**24` +/// - `list_size > usize::max_value() / 2` +pub fn shuffle_list( + mut input: Vec, + rounds: u8, + seed: &[u8], + forwards: bool, +) -> Option> { + let list_size = input.len(); + + if input.is_empty() + || list_size > usize::max_value() / 2 + || list_size > 2_usize.pow(24) + || rounds == 0 + { + return None; + } + + let mut buf: Vec = Vec::with_capacity(TOTAL_SIZE); + + let mut r = if forwards { 0 } else { rounds - 1 }; + + buf.extend_from_slice(seed); + + loop { + buf.splice(SEED_SIZE.., vec![r]); + + let pivot = bytes_to_int64(&hash(&buf[0..PIVOT_VIEW_SIZE])[0..8]) as usize % list_size; + + let mirror = (pivot + 1) >> 1; + + buf.splice(PIVOT_VIEW_SIZE.., int_to_bytes4((pivot >> 8) as u32)); + let mut source = hash(&buf[..]); + let mut byte_v = source[(pivot & 0xff) >> 3]; + + for i in 0..mirror { + let j = pivot - i; + + if j & 0xff == 0xff { + buf.splice(PIVOT_VIEW_SIZE.., int_to_bytes4((j >> 8) as u32)); + source = hash(&buf[..]); + } + + if j & 0x07 == 0x07 { + byte_v = source[(j & 0xff) >> 3]; + } + let bit_v = (byte_v >> (j & 0x07)) & 0x01; + + if bit_v == 1 { + input.swap(i, j); + } + } + + let mirror = (pivot + list_size + 1) >> 1; + let end = list_size - 1; + + buf.splice(PIVOT_VIEW_SIZE.., int_to_bytes4((end >> 8) as u32)); + let mut source = hash(&buf[..]); + let mut byte_v = source[(end & 0xff) >> 3]; + + for (loop_iter, i) in ((pivot + 1)..mirror).enumerate() { + let j = end - loop_iter; + + if j & 0xff == 0xff { + buf.splice(PIVOT_VIEW_SIZE.., int_to_bytes4((j >> 8) as u32)); + source = hash(&buf[..]); + } + + if j & 0x07 == 0x07 { + byte_v = source[(j & 0xff) >> 3]; + } + let bit_v = (byte_v >> (j & 0x07)) & 0x01; + + if bit_v == 1 { + input.swap(i, j); + } + } + + if forwards { + r += 1; + if r == rounds { + break; + } + } else { + if r == 0 { + break; + } + r -= 1; + } + } + + Some(input) +} + +fn bytes_to_int64(bytes: &[u8]) -> u64 { + let mut cursor = Cursor::new(bytes); + cursor.get_u64_le() +} + +#[cfg(test)] +mod tests { + use super::*; + use hex; + use std::{fs::File, io::prelude::*, path::PathBuf}; + use yaml_rust::yaml; + + #[test] + fn returns_none_for_zero_length_list() { + assert_eq!(None, shuffle_list(vec![], 90, &[42, 42], true)); + } + + #[test] + fn test_vectors() { + let mut file = { + let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + file_path_buf.push("src/specs/test_vector_permutated_index.yml"); + + File::open(file_path_buf).unwrap() + }; + + let mut yaml_str = String::new(); + + file.read_to_string(&mut yaml_str).unwrap(); + + let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap(); + let doc = &docs[0]; + let test_cases = doc["test_cases"].as_vec().unwrap(); + + for (i, test_case) in test_cases.iter().enumerate() { + let index = test_case["index"].as_i64().unwrap() as usize; + let list_size = test_case["list_size"].as_i64().unwrap() as usize; + let permutated_index = test_case["permutated_index"].as_i64().unwrap() as usize; + let shuffle_round_count = test_case["shuffle_round_count"].as_i64().unwrap(); + let seed_string = test_case["seed"].clone().into_string().unwrap(); + let seed = hex::decode(seed_string.replace("0x", "")).unwrap(); + + let shuffle_round_count = if shuffle_round_count < (u8::max_value() as i64) { + shuffle_round_count as u8 + } else { + panic!("shuffle_round_count must be a u8") + }; + + let list: Vec = (0..list_size).collect(); + + let shuffled = + shuffle_list(list.clone(), shuffle_round_count, &seed[..], true).unwrap(); + + assert_eq!( + list[index], shuffled[permutated_index], + "Failure on case #{} index: {}, list_size: {}, round_count: {}, seed: {}", + i, index, list_size, shuffle_round_count, seed_string + ); + } + } +}