From 073be906da498171acd3a7b94d773cdded2d3b52 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 28 Feb 2019 16:37:12 +1100 Subject: [PATCH 1/8] Remove cache operations from epoch processing. - Don't build the next cache at all. - Call `advance_caches()` in per-slot processing. --- eth2/state_processing/src/epoch_processable.rs | 6 ------ eth2/state_processing/src/slot_processable.rs | 1 + 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/eth2/state_processing/src/epoch_processable.rs b/eth2/state_processing/src/epoch_processable.rs index 2cee455a3..0ecd1bbd1 100644 --- a/eth2/state_processing/src/epoch_processable.rs +++ b/eth2/state_processing/src/epoch_processable.rs @@ -619,12 +619,6 @@ impl EpochProcessable for BeaconState { .cloned() .collect(); - /* - * Manage the beacon state caches - */ - self.advance_caches(); - self.build_epoch_cache(RelativeEpoch::Next, spec)?; - debug!("Epoch transition complete."); Ok(()) diff --git a/eth2/state_processing/src/slot_processable.rs b/eth2/state_processing/src/slot_processable.rs index 0bbc79ab0..6017f4c0a 100644 --- a/eth2/state_processing/src/slot_processable.rs +++ b/eth2/state_processing/src/slot_processable.rs @@ -26,6 +26,7 @@ where ) -> Result<(), Error> { if (self.slot + 1) % spec.epoch_length == 0 { self.per_epoch_processing(spec)?; + self.advance_caches(); } self.slot += 1; From 6fac35a7c51f3b16c4b9ac240b5ada19ecbb1298 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Mar 2019 00:18:53 +1100 Subject: [PATCH 2/8] Update main readme, add eth2/ readme --- README.md | 13 +++++++++++++ eth2/README.md | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 eth2/README.md diff --git a/README.md b/README.md index 7759c1166..e3df9c19c 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,19 @@ If you'd like some background on Sigma Prime, please see the [Lighthouse Update \#00](https://lighthouse.sigmaprime.io/update-00.html) blog post or the [company website](https://sigmaprime.io). +### Directory Structure + +- [`/beacon_node`](beacon_node/): the "Beacon Node" binary and crates exclusively + associated with it. +- [`/docs`](docs/): documentation related to the repository. This include contributor + guides, etc. Code documentation is produced with `cargo doc`. +- [`/eth2`](eth2/): Crates containing common logic across the Lighthouse project. For + example; Ethereum 2.0 types (`BeaconBlock`, `BeaconState`, etc) and + SimpleSerialize (SSZ). +- [`/protos`](protos/): protobuf/gRPC definitions common across the Lighthouse project. +- [`/validator_client`](validator_client/): the "Validator Client" binary and crates exclusively + associated with it. + ### Components The following list describes some of the components actively under development diff --git a/eth2/README.md b/eth2/README.md new file mode 100644 index 000000000..a96c1b9df --- /dev/null +++ b/eth2/README.md @@ -0,0 +1,40 @@ +# eth2 + +Rust crates containing logic common across the Lighthouse project. + +## Per-Crate Summary + +- [`attester/`](attester/): Core logic for attesting to beacon and shard blocks. +- [`block_proposer/`](block_proposer/): Core logic for proposing beacon blocks. +- [`fork_choice/`](fork_choice/): A collection of fork-choice algorithms for + the Beacon Chain. +- [`state_processing/`](state_processing/): Provides per-slot, per-block and + per-epoch state processing. +- [`types/`](types/): Defines base Ethereum 2.0 types (e.g., `BeaconBlock`, + `BeaconState`, etc). +- [`utils/`](utils/): + - [`bls`](utils/bls/): A wrapper around some external BLS encryption library. + - [`boolean-bitfield`](utils/boolean-bitfield/): Provides an expandable Vec + of bools, specifically for use in Eth2. + - [`fisher-yates-shuffle`](utils/fisher-yates-shuffle/): shuffles a list + pseudo-randomly. + - [`hashing`](utils/hashing/): Provides unified hashing methods, provided + be some external library. + - [`honey-badger-split`](utils/honey-badger-split/): Splits a list in `n` + parts without giving AF about the length of the list, `n` or anything + else. + - [`int-to-bytes`](utils/int-to-bytes/): Simple library which converts ints + into byte-strings of various lengths. + - [`slot_clock`](utils/slot_clock/): translates the system time into Beacon + - Chain "slots". Also + provides another slot clock that's useful during testing. + - [`ssz`](utils/ssz/): an implementation of the SimpleSerialize + - serialization/deserialization + protocol used by + Eth 2.0. + - [`ssz_derive`](utils/ssz_derive/): provides procedural macros for + deriving SSZ `Encodable`, `Decodable` and `TreeHash` methods. + - [`swap_or_not_shuffle`](utils/swap_or_not_shuffle/): a list-shuffling + method which is slow, but allows for shuffling a subset of indices. + - [`test_random_derive`](utils/test_random_derive/): provides procedural + macros for deriving the `TestRandom` trait defined in `types`. From 8842adb53b97c859eac1ea3bf80ee98ade976d3a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Mar 2019 00:25:11 +1100 Subject: [PATCH 3/8] Fix typos in READMEs --- README.md | 24 +++++++----------------- eth2/README.md | 17 +++++++---------- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index e3df9c19c..c26dae219 100644 --- a/README.md +++ b/README.md @@ -31,15 +31,15 @@ If you'd like some background on Sigma Prime, please see the [Lighthouse Update ### Directory Structure -- [`/beacon_node`](beacon_node/): the "Beacon Node" binary and crates exclusively +- [`beacon_node/`](beacon_node/): the "Beacon Node" binary and crates exclusively associated with it. -- [`/docs`](docs/): documentation related to the repository. This include contributor - guides, etc. Code documentation is produced with `cargo doc`. -- [`/eth2`](eth2/): Crates containing common logic across the Lighthouse project. For - example; Ethereum 2.0 types (`BeaconBlock`, `BeaconState`, etc) and +- [`docs/`](docs/): documentation related to the repository. This include contributor + guides, etc. (Code documentation is produced with `cargo doc`). +- [`eth2/`](eth2/): Crates containing common logic across the Lighthouse project. For + example; Ethereum 2.0 types ([`BeaconBlock`](eth2/types/src/beacon_block.rs), [`BeaconState`](eth2/types/src/beacon_state.rs), etc) and SimpleSerialize (SSZ). -- [`/protos`](protos/): protobuf/gRPC definitions common across the Lighthouse project. -- [`/validator_client`](validator_client/): the "Validator Client" binary and crates exclusively +- [`protos/`](protos/): protobuf/gRPC definitions common across the Lighthouse project. +- [`validator_client/`](validator_client/): the "Validator Client" binary and crates exclusively associated with it. ### Components @@ -92,16 +92,6 @@ In addition to these components we are also working on database schemas, RPC frameworks, specification development, database optimizations (e.g., bloom-filters), and tons of other interesting stuff (at least we think so). -### Directory Structure - -Here we provide an overview of the directory structure: - -- `beacon_chain/`: contains logic derived directly from the specification. - E.g., shuffling algorithms, state transition logic and structs, block -validation, BLS crypto, etc. -- `lighthouse/`: contains logic specific to this client implementation. E.g., - CLI parsing, RPC end-points, databases, etc. - ### Running **NOTE: The cryptography libraries used in this implementation are diff --git a/eth2/README.md b/eth2/README.md index a96c1b9df..e7b69635a 100644 --- a/eth2/README.md +++ b/eth2/README.md @@ -1,4 +1,4 @@ -# eth2 +# Ethereum 2.0 Common Crates Rust crates containing logic common across the Lighthouse project. @@ -13,25 +13,22 @@ Rust crates containing logic common across the Lighthouse project. - [`types/`](types/): Defines base Ethereum 2.0 types (e.g., `BeaconBlock`, `BeaconState`, etc). - [`utils/`](utils/): - - [`bls`](utils/bls/): A wrapper around some external BLS encryption library. + - [`bls`](utils/bls/): A wrapper around an external BLS encryption library. - [`boolean-bitfield`](utils/boolean-bitfield/): Provides an expandable Vec of bools, specifically for use in Eth2. - [`fisher-yates-shuffle`](utils/fisher-yates-shuffle/): shuffles a list pseudo-randomly. - - [`hashing`](utils/hashing/): Provides unified hashing methods, provided - be some external library. + - [`hashing`](utils/hashing/): A wrapper around external hashing libraries. - [`honey-badger-split`](utils/honey-badger-split/): Splits a list in `n` parts without giving AF about the length of the list, `n` or anything else. - [`int-to-bytes`](utils/int-to-bytes/): Simple library which converts ints into byte-strings of various lengths. - [`slot_clock`](utils/slot_clock/): translates the system time into Beacon - - Chain "slots". Also - provides another slot clock that's useful during testing. - - [`ssz`](utils/ssz/): an implementation of the SimpleSerialize - - serialization/deserialization - protocol used by - Eth 2.0. + Chain "slots". (Also provides another slot clock that's useful during + testing.) + - [`ssz`](utils/ssz/): an implementation of the SimpleSerialize + serialization/deserialization protocol used by Eth 2.0. - [`ssz_derive`](utils/ssz_derive/): provides procedural macros for deriving SSZ `Encodable`, `Decodable` and `TreeHash` methods. - [`swap_or_not_shuffle`](utils/swap_or_not_shuffle/): a list-shuffling From 8aa7f25bbcb3c2563f2d1596a5814c147135f199 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Mar 2019 01:47:40 +1100 Subject: [PATCH 4/8] Introduce faster swap-or-not whole-list shuffle --- eth2/types/src/beacon_state.rs | 22 ++++----- .../swap_or_not_shuffle/benches/benches.rs | 22 ++++++++- eth2/utils/swap_or_not_shuffle/src/lib.rs | 46 ++++++++++++++++--- 3 files changed, 71 insertions(+), 19 deletions(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 3d94a8e3d..2d4f463b7 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -12,7 +12,7 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; use std::collections::HashMap; -use swap_or_not_shuffle::get_permutated_index; +use swap_or_not_shuffle::get_permutated_list; pub use builder::BeaconStateBuilder; @@ -420,17 +420,15 @@ impl BeaconState { committees_per_epoch ); - let mut shuffled_active_validator_indices = vec![0; active_validator_indices.len()]; - for (i, _) in active_validator_indices.iter().enumerate() { - let shuffled_i = get_permutated_index( - i, - active_validator_indices.len(), - &seed[..], - spec.shuffle_round_count, - ) - .ok_or_else(|| Error::UnableToShuffle)?; - shuffled_active_validator_indices[i] = active_validator_indices[shuffled_i] - } + let active_validator_indices: Vec = + active_validator_indices.iter().cloned().collect(); + + let shuffled_active_validator_indices = get_permutated_list( + &active_validator_indices, + &seed[..], + spec.shuffle_round_count, + ) + .ok_or_else(|| Error::UnableToShuffle)?; Ok(shuffled_active_validator_indices .honey_badger_split(committees_per_epoch as usize) diff --git a/eth2/utils/swap_or_not_shuffle/benches/benches.rs b/eth2/utils/swap_or_not_shuffle/benches/benches.rs index 1d5b5476c..b1311b41e 100644 --- a/eth2/utils/swap_or_not_shuffle/benches/benches.rs +++ b/eth2/utils/swap_or_not_shuffle/benches/benches.rs @@ -1,6 +1,6 @@ use criterion::Criterion; use criterion::{black_box, criterion_group, criterion_main, Benchmark}; -use swap_or_not_shuffle::get_permutated_index; +use swap_or_not_shuffle::{get_permutated_index, get_permutated_list}; const SHUFFLE_ROUND_COUNT: u8 = 90; @@ -48,6 +48,16 @@ fn shuffles(c: &mut Criterion) { .sample_size(10), ); + c.bench( + "_fast_ whole list shuffle", + Benchmark::new("512 elements", move |b| { + let seed = vec![42; 32]; + let list: Vec = (0..512).collect(); + b.iter(|| black_box(get_permutated_list(&list, &seed, SHUFFLE_ROUND_COUNT))) + }) + .sample_size(10), + ); + c.bench( "whole list shuffle", Benchmark::new("16384 elements", move |b| { @@ -56,6 +66,16 @@ fn shuffles(c: &mut Criterion) { }) .sample_size(10), ); + + c.bench( + "_fast_ whole list shuffle", + Benchmark::new("16384 elements", move |b| { + let seed = vec![42; 32]; + let list: Vec = (0..16384).collect(); + b.iter(|| black_box(get_permutated_list(&list, &seed, SHUFFLE_ROUND_COUNT))) + }) + .sample_size(10), + ); } criterion_group!(benches, shuffles,); diff --git a/eth2/utils/swap_or_not_shuffle/src/lib.rs b/eth2/utils/swap_or_not_shuffle/src/lib.rs index 753265f3e..b9140b269 100644 --- a/eth2/utils/swap_or_not_shuffle/src/lib.rs +++ b/eth2/utils/swap_or_not_shuffle/src/lib.rs @@ -4,6 +4,36 @@ use int_to_bytes::{int_to_bytes1, int_to_bytes4}; use std::cmp::max; use std::io::Cursor; +pub fn get_permutated_list( + list: &[usize], + seed: &[u8], + shuffle_round_count: u8, +) -> Option> { + let list_size = list.len(); + + if list_size == 0 || list_size > usize::max_value() / 2 || list_size > 2_usize.pow(24) { + return None; + } + + let mut pivots = Vec::with_capacity(shuffle_round_count as usize); + for round in 0..shuffle_round_count { + pivots.push(bytes_to_int64(&hash_with_round(seed, round)[..]) as usize % list_size); + } + + let mut output = Vec::with_capacity(list_size); + + for i in 0..list_size { + let mut index = i; + for round in 0..shuffle_round_count { + let pivot = pivots[round as usize]; + index = do_round(seed, index, pivot, round, list_size)?; + } + output.push(list[index]) + } + + Some(output) +} + /// Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy. /// /// Utilizes 'swap or not' shuffling found in @@ -32,16 +62,20 @@ pub fn get_permutated_index( let mut index = index; for round in 0..shuffle_round_count { let pivot = bytes_to_int64(&hash_with_round(seed, round)[..]) as usize % list_size; - let flip = (pivot + list_size - index) % list_size; - let position = max(index, flip); - let source = hash_with_round_and_position(seed, round, position)?; - let byte = source[(position % 256) / 8]; - let bit = (byte >> (position % 8)) % 2; - index = if bit == 1 { flip } else { index } + index = do_round(seed, index, pivot, round, list_size)?; } Some(index) } +fn do_round(seed: &[u8], index: usize, pivot: usize, round: u8, list_size: usize) -> Option { + let flip = (pivot + list_size - index) % list_size; + let position = max(index, flip); + let source = hash_with_round_and_position(seed, round, position)?; + let byte = source[(position % 256) / 8]; + let bit = (byte >> (position % 8)) % 2; + Some(if bit == 1 { flip } else { index }) +} + fn hash_with_round_and_position(seed: &[u8], round: u8, position: usize) -> Option> { let mut seed = seed.to_vec(); seed.append(&mut int_to_bytes1(round)); From 056a263f0294acb17e74383910432e7a22ffc2ad Mon Sep 17 00:00:00 2001 From: Micah Dameron Date: Thu, 28 Feb 2019 18:05:11 -0700 Subject: [PATCH 5/8] modified: README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7759c1166..6523e9202 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ by the team: from the Ethereum Foundation to develop *simpleserialize* (SSZ), a purpose-built serialization format for sending information across a network. Check out the [SSZ -implementation](https://github.com/sigp/lighthouse/tree/master/beacon_chain/utils/ssz) +implementation](https://github.com/ethereum/eth2.0-specs/blob/00aa553fee95963b74fbec84dbd274d7247b8a0e/specs/simple-serialize.md) and this [research](https://github.com/sigp/serialization_sandbox/blob/report/report/serialization_report.md) on serialization formats for more information. From c3b2f802a78256f526353f4c6c5322b672df43e8 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Mar 2019 12:19:05 +1100 Subject: [PATCH 6/8] Add fast full-list shuffle for swap-or-not - Passes test vectors - Implemented in beacon state - Added more docs --- eth2/types/src/beacon_state.rs | 9 +- .../swap_or_not_shuffle/benches/benches.rs | 16 +- .../src/get_permutated_index.rs | 187 ++++++++++++++ eth2/utils/swap_or_not_shuffle/src/lib.rs | 229 ++---------------- .../swap_or_not_shuffle/src/shuffle_list.rs | 178 ++++++++++++++ 5 files changed, 402 insertions(+), 217 deletions(-) create mode 100644 eth2/utils/swap_or_not_shuffle/src/get_permutated_index.rs create mode 100644 eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 2d4f463b7..67cf2341a 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -12,7 +12,7 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; use std::collections::HashMap; -use swap_or_not_shuffle::get_permutated_list; +use swap_or_not_shuffle::shuffle_list; pub use builder::BeaconStateBuilder; @@ -423,10 +423,11 @@ impl BeaconState { let active_validator_indices: Vec = active_validator_indices.iter().cloned().collect(); - let shuffled_active_validator_indices = get_permutated_list( - &active_validator_indices, - &seed[..], + let shuffled_active_validator_indices = shuffle_list( + active_validator_indices, spec.shuffle_round_count, + &seed[..], + true, ) .ok_or_else(|| Error::UnableToShuffle)?; diff --git a/eth2/utils/swap_or_not_shuffle/benches/benches.rs b/eth2/utils/swap_or_not_shuffle/benches/benches.rs index b1311b41e..0502e6fc4 100644 --- a/eth2/utils/swap_or_not_shuffle/benches/benches.rs +++ b/eth2/utils/swap_or_not_shuffle/benches/benches.rs @@ -1,6 +1,6 @@ use criterion::Criterion; use criterion::{black_box, criterion_group, criterion_main, Benchmark}; -use swap_or_not_shuffle::{get_permutated_index, get_permutated_list}; +use swap_or_not_shuffle::{get_permutated_index, shuffle_list as fast_shuffle}; const SHUFFLE_ROUND_COUNT: u8 = 90; @@ -53,7 +53,7 @@ fn shuffles(c: &mut Criterion) { Benchmark::new("512 elements", move |b| { let seed = vec![42; 32]; let list: Vec = (0..512).collect(); - b.iter(|| black_box(get_permutated_list(&list, &seed, SHUFFLE_ROUND_COUNT))) + b.iter(|| black_box(fast_shuffle(list.clone(), SHUFFLE_ROUND_COUNT, &seed, true))) }) .sample_size(10), ); @@ -72,7 +72,17 @@ fn shuffles(c: &mut Criterion) { Benchmark::new("16384 elements", move |b| { let seed = vec![42; 32]; let list: Vec = (0..16384).collect(); - b.iter(|| black_box(get_permutated_list(&list, &seed, SHUFFLE_ROUND_COUNT))) + b.iter(|| black_box(fast_shuffle(list.clone(), SHUFFLE_ROUND_COUNT, &seed, true))) + }) + .sample_size(10), + ); + + c.bench( + "_fast_ whole list shuffle", + Benchmark::new("4m elements", move |b| { + let seed = vec![42; 32]; + let list: Vec = (0..4_000_000).collect(); + b.iter(|| black_box(fast_shuffle(list.clone(), SHUFFLE_ROUND_COUNT, &seed, true))) }) .sample_size(10), ); diff --git a/eth2/utils/swap_or_not_shuffle/src/get_permutated_index.rs b/eth2/utils/swap_or_not_shuffle/src/get_permutated_index.rs new file mode 100644 index 000000000..37a82341e --- /dev/null +++ b/eth2/utils/swap_or_not_shuffle/src/get_permutated_index.rs @@ -0,0 +1,187 @@ +use bytes::Buf; +use hashing::hash; +use int_to_bytes::{int_to_bytes1, int_to_bytes4}; +use std::cmp::max; +use std::io::Cursor; + +/// Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy. +/// +/// Utilizes 'swap or not' shuffling found in +/// https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf +/// See the 'generalized domain' algorithm on page 3. +/// +/// Note: this function is significantly slower than the `shuffle_list` function in this crate. +/// Using `get_permutated_list` to shuffle an entire list, index by index, has been observed to be +/// 250x slower than `shuffle_list`. Therefore, this function is only useful when shuffling a small +/// portion of a much larger list. +/// +/// Returns `None` under any of the following conditions: +/// - `list_size == 0` +/// - `index >= list_size` +/// - `list_size > 2**24` +/// - `list_size > usize::max_value() / 2` +pub fn get_permutated_index( + index: usize, + list_size: usize, + seed: &[u8], + shuffle_round_count: u8, +) -> Option { + if list_size == 0 + || index >= list_size + || list_size > usize::max_value() / 2 + || list_size > 2_usize.pow(24) + { + return None; + } + + let mut index = index; + for round in 0..shuffle_round_count { + let pivot = bytes_to_int64(&hash_with_round(seed, round)[..]) as usize % list_size; + index = do_round(seed, index, pivot, round, list_size)?; + } + Some(index) +} + +fn do_round(seed: &[u8], index: usize, pivot: usize, round: u8, list_size: usize) -> Option { + let flip = (pivot + list_size - index) % list_size; + let position = max(index, flip); + let source = hash_with_round_and_position(seed, round, position)?; + let byte = source[(position % 256) / 8]; + let bit = (byte >> (position % 8)) % 2; + Some(if bit == 1 { flip } else { index }) +} + +fn hash_with_round_and_position(seed: &[u8], round: u8, position: usize) -> Option> { + let mut seed = seed.to_vec(); + seed.append(&mut int_to_bytes1(round)); + /* + * Note: the specification has an implicit assertion in `int_to_bytes4` that `position / 256 < + * 2**24`. For efficiency, we do not check for that here as it is checked in `get_permutated_index`. + */ + seed.append(&mut int_to_bytes4((position / 256) as u32)); + Some(hash(&seed[..])) +} + +fn hash_with_round(seed: &[u8], round: u8) -> Vec { + let mut seed = seed.to_vec(); + seed.append(&mut int_to_bytes1(round)); + hash(&seed[..]) +} + +fn bytes_to_int64(bytes: &[u8]) -> u64 { + let mut cursor = Cursor::new(bytes); + cursor.get_u64_le() +} + +#[cfg(test)] +mod tests { + use super::*; + use ethereum_types::H256 as Hash256; + use hex; + use std::{fs::File, io::prelude::*, path::PathBuf}; + use yaml_rust::yaml; + + #[test] + #[ignore] + fn fuzz_test() { + let max_list_size = 2_usize.pow(24); + let test_runs = 1000; + + // Test at max list_size with the end index. + for _ in 0..test_runs { + let index = max_list_size - 1; + let list_size = max_list_size; + let seed = Hash256::random(); + let shuffle_rounds = 90; + + assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some()); + } + + // Test at max list_size low indices. + for i in 0..test_runs { + let index = i; + let list_size = max_list_size; + let seed = Hash256::random(); + let shuffle_rounds = 90; + + assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some()); + } + + // Test at max list_size high indices. + for i in 0..test_runs { + let index = max_list_size - 1 - i; + let list_size = max_list_size; + let seed = Hash256::random(); + let shuffle_rounds = 90; + + assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some()); + } + } + + #[test] + fn returns_none_for_zero_length_list() { + assert_eq!(None, get_permutated_index(100, 0, &[42, 42], 90)); + } + + #[test] + fn returns_none_for_out_of_bounds_index() { + assert_eq!(None, get_permutated_index(100, 100, &[42, 42], 90)); + } + + #[test] + fn returns_none_for_too_large_list() { + assert_eq!( + None, + get_permutated_index(100, usize::max_value() / 2, &[42, 42], 90) + ); + } + + #[test] + fn test_vectors() { + /* + * Test vectors are generated here: + * + * https://github.com/ethereum/eth2.0-test-generators + */ + let mut file = { + let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + file_path_buf.push("src/specs/test_vector_permutated_index.yml"); + + File::open(file_path_buf).unwrap() + }; + + let mut yaml_str = String::new(); + + file.read_to_string(&mut yaml_str).unwrap(); + + let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap(); + let doc = &docs[0]; + let test_cases = doc["test_cases"].as_vec().unwrap(); + + for (i, test_case) in test_cases.iter().enumerate() { + let index = test_case["index"].as_i64().unwrap() as usize; + let list_size = test_case["list_size"].as_i64().unwrap() as usize; + let permutated_index = test_case["permutated_index"].as_i64().unwrap() as usize; + let shuffle_round_count = test_case["shuffle_round_count"].as_i64().unwrap(); + let seed_string = test_case["seed"].clone().into_string().unwrap(); + let seed = hex::decode(seed_string.replace("0x", "")).unwrap(); + + let shuffle_round_count = if shuffle_round_count < (u8::max_value() as i64) { + shuffle_round_count as u8 + } else { + panic!("shuffle_round_count must be a u8") + }; + + assert_eq!( + Some(permutated_index), + get_permutated_index(index, list_size, &seed[..], shuffle_round_count), + "Failure on case #{} index: {}, list_size: {}, round_count: {}, seed: {}", + i, + index, + list_size, + shuffle_round_count, + seed_string, + ); + } + } +} diff --git a/eth2/utils/swap_or_not_shuffle/src/lib.rs b/eth2/utils/swap_or_not_shuffle/src/lib.rs index b9140b269..57049fbdf 100644 --- a/eth2/utils/swap_or_not_shuffle/src/lib.rs +++ b/eth2/utils/swap_or_not_shuffle/src/lib.rs @@ -1,212 +1,21 @@ -use bytes::Buf; -use hashing::hash; -use int_to_bytes::{int_to_bytes1, int_to_bytes4}; -use std::cmp::max; -use std::io::Cursor; +//! Provides list-shuffling functions matching the Ethereum 2.0 specification. +//! +//! See +//! [get_permutated_index](https://github.com/ethereum/eth2.0-specs/blob/0.4.0/specs/core/0_beacon-chain.md#get_permuted_index) +//! for specifications. +//! +//! There are two functions exported by this crate: +//! +//! - `get_permutated_index`: given a single index, computes the index resulting from a shuffle. +//! Runs in less time than it takes to run `shuffle_list`. +//! - `shuffle_list`: shuffles an entire list in-place. Runs in less time than it takes to run +//! `get_permutated_index` on each index. +//! +//! In general, use `get_permutated_list` to calculate the shuffling of a small subset of a much +//! larger list (~250x larger is a good guide, but solid figures yet to be calculated). -pub fn get_permutated_list( - list: &[usize], - seed: &[u8], - shuffle_round_count: u8, -) -> Option> { - let list_size = list.len(); +mod get_permutated_index; +mod shuffle_list; - if list_size == 0 || list_size > usize::max_value() / 2 || list_size > 2_usize.pow(24) { - return None; - } - - let mut pivots = Vec::with_capacity(shuffle_round_count as usize); - for round in 0..shuffle_round_count { - pivots.push(bytes_to_int64(&hash_with_round(seed, round)[..]) as usize % list_size); - } - - let mut output = Vec::with_capacity(list_size); - - for i in 0..list_size { - let mut index = i; - for round in 0..shuffle_round_count { - let pivot = pivots[round as usize]; - index = do_round(seed, index, pivot, round, list_size)?; - } - output.push(list[index]) - } - - Some(output) -} - -/// Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy. -/// -/// Utilizes 'swap or not' shuffling found in -/// https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf -/// See the 'generalized domain' algorithm on page 3. -/// -/// Returns `None` under any of the following conditions: -/// - `list_size == 0` -/// - `index >= list_size` -/// - `list_size > 2**24` -/// - `list_size > usize::max_value() / 2` -pub fn get_permutated_index( - index: usize, - list_size: usize, - seed: &[u8], - shuffle_round_count: u8, -) -> Option { - if list_size == 0 - || index >= list_size - || list_size > usize::max_value() / 2 - || list_size > 2_usize.pow(24) - { - return None; - } - - let mut index = index; - for round in 0..shuffle_round_count { - let pivot = bytes_to_int64(&hash_with_round(seed, round)[..]) as usize % list_size; - index = do_round(seed, index, pivot, round, list_size)?; - } - Some(index) -} - -fn do_round(seed: &[u8], index: usize, pivot: usize, round: u8, list_size: usize) -> Option { - let flip = (pivot + list_size - index) % list_size; - let position = max(index, flip); - let source = hash_with_round_and_position(seed, round, position)?; - let byte = source[(position % 256) / 8]; - let bit = (byte >> (position % 8)) % 2; - Some(if bit == 1 { flip } else { index }) -} - -fn hash_with_round_and_position(seed: &[u8], round: u8, position: usize) -> Option> { - let mut seed = seed.to_vec(); - seed.append(&mut int_to_bytes1(round)); - /* - * Note: the specification has an implicit assertion in `int_to_bytes4` that `position / 256 < - * 2**24`. For efficiency, we do not check for that here as it is checked in `get_permutated_index`. - */ - seed.append(&mut int_to_bytes4((position / 256) as u32)); - Some(hash(&seed[..])) -} - -fn hash_with_round(seed: &[u8], round: u8) -> Vec { - let mut seed = seed.to_vec(); - seed.append(&mut int_to_bytes1(round)); - hash(&seed[..]) -} - -fn bytes_to_int64(bytes: &[u8]) -> u64 { - let mut cursor = Cursor::new(bytes); - cursor.get_u64_le() -} - -#[cfg(test)] -mod tests { - use super::*; - use ethereum_types::H256 as Hash256; - use hex; - use std::{fs::File, io::prelude::*, path::PathBuf}; - use yaml_rust::yaml; - - #[test] - #[ignore] - fn fuzz_test() { - let max_list_size = 2_usize.pow(24); - let test_runs = 1000; - - // Test at max list_size with the end index. - for _ in 0..test_runs { - let index = max_list_size - 1; - let list_size = max_list_size; - let seed = Hash256::random(); - let shuffle_rounds = 90; - - assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some()); - } - - // Test at max list_size low indices. - for i in 0..test_runs { - let index = i; - let list_size = max_list_size; - let seed = Hash256::random(); - let shuffle_rounds = 90; - - assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some()); - } - - // Test at max list_size high indices. - for i in 0..test_runs { - let index = max_list_size - 1 - i; - let list_size = max_list_size; - let seed = Hash256::random(); - let shuffle_rounds = 90; - - assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some()); - } - } - - #[test] - fn returns_none_for_zero_length_list() { - assert_eq!(None, get_permutated_index(100, 0, &[42, 42], 90)); - } - - #[test] - fn returns_none_for_out_of_bounds_index() { - assert_eq!(None, get_permutated_index(100, 100, &[42, 42], 90)); - } - - #[test] - fn returns_none_for_too_large_list() { - assert_eq!( - None, - get_permutated_index(100, usize::max_value() / 2, &[42, 42], 90) - ); - } - - #[test] - fn test_vectors() { - /* - * Test vectors are generated here: - * - * https://github.com/ethereum/eth2.0-test-generators - */ - let mut file = { - let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - file_path_buf.push("src/specs/test_vector_permutated_index.yml"); - - File::open(file_path_buf).unwrap() - }; - - let mut yaml_str = String::new(); - - file.read_to_string(&mut yaml_str).unwrap(); - - let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap(); - let doc = &docs[0]; - let test_cases = doc["test_cases"].as_vec().unwrap(); - - for (i, test_case) in test_cases.iter().enumerate() { - let index = test_case["index"].as_i64().unwrap() as usize; - let list_size = test_case["list_size"].as_i64().unwrap() as usize; - let permutated_index = test_case["permutated_index"].as_i64().unwrap() as usize; - let shuffle_round_count = test_case["shuffle_round_count"].as_i64().unwrap(); - let seed_string = test_case["seed"].clone().into_string().unwrap(); - let seed = hex::decode(seed_string.replace("0x", "")).unwrap(); - - let shuffle_round_count = if shuffle_round_count < (u8::max_value() as i64) { - shuffle_round_count as u8 - } else { - panic!("shuffle_round_count must be a u8") - }; - - assert_eq!( - Some(permutated_index), - get_permutated_index(index, list_size, &seed[..], shuffle_round_count), - "Failure on case #{} index: {}, list_size: {}, round_count: {}, seed: {}", - i, - index, - list_size, - shuffle_round_count, - seed_string, - ); - } - } -} +pub use get_permutated_index::get_permutated_index; +pub use shuffle_list::shuffle_list; diff --git a/eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs b/eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs new file mode 100644 index 000000000..e7e1e18e6 --- /dev/null +++ b/eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs @@ -0,0 +1,178 @@ +use bytes::Buf; +use hashing::hash; +use int_to_bytes::int_to_bytes4; +use std::io::Cursor; + +const SEED_SIZE: usize = 32; +const ROUND_SIZE: usize = 1; +const POSITION_WINDOW_SIZE: usize = 4; +const PIVOT_VIEW_SIZE: usize = SEED_SIZE + ROUND_SIZE; +const TOTAL_SIZE: usize = SEED_SIZE + ROUND_SIZE + POSITION_WINDOW_SIZE; + +/// Shuffles an entire list in-place. +/// +/// Note: this is equivalent to the `get_permutated_index` function, except it shuffles an entire +/// list not just a single index. With large lists this function has been observed to be 250x +/// faster than running `get_permutated_index` across an entire list. +/// +/// Credits to [@protolambda](https://github.com/protolambda) for defining this algorithm. +/// +/// Shuffles if `forwards == true`, otherwise un-shuffles. +/// +/// Returns `None` under any of the following conditions: +/// - `list_size == 0` +/// - `list_size > 2**24` +/// - `list_size > usize::max_value() / 2` +pub fn shuffle_list( + mut input: Vec, + rounds: u8, + seed: &[u8], + forwards: bool, +) -> Option> { + let list_size = input.len(); + + if input.is_empty() + || list_size > usize::max_value() / 2 + || list_size > 2_usize.pow(24) + || rounds == 0 + { + return None; + } + + let mut buf: Vec = Vec::with_capacity(TOTAL_SIZE); + + let mut r = if forwards { 0 } else { rounds - 1 }; + + buf.extend_from_slice(seed); + + loop { + buf.splice(SEED_SIZE.., vec![r]); + + let pivot = bytes_to_int64(&hash(&buf[0..PIVOT_VIEW_SIZE])[0..8]) as usize % list_size; + + let mirror = (pivot + 1) >> 1; + + buf.splice(PIVOT_VIEW_SIZE.., int_to_bytes4((pivot >> 8) as u32)); + let mut source = hash(&buf[..]); + let mut byte_v = source[(pivot & 0xff) >> 3]; + + for i in 0..mirror { + let j = pivot - i; + + if j & 0xff == 0xff { + buf.splice(PIVOT_VIEW_SIZE.., int_to_bytes4((j >> 8) as u32)); + source = hash(&buf[..]); + } + + if j & 0x07 == 0x07 { + byte_v = source[(j & 0xff) >> 3]; + } + let bit_v = (byte_v >> (j & 0x07)) & 0x01; + + if bit_v == 1 { + input.swap(i, j); + } + } + + let mirror = (pivot + list_size + 1) >> 1; + let end = list_size - 1; + + buf.splice(PIVOT_VIEW_SIZE.., int_to_bytes4((end >> 8) as u32)); + let mut source = hash(&buf[..]); + let mut byte_v = source[(end & 0xff) >> 3]; + + for (loop_iter, i) in ((pivot + 1)..mirror).enumerate() { + let j = end - loop_iter; + + if j & 0xff == 0xff { + buf.splice(PIVOT_VIEW_SIZE.., int_to_bytes4((j >> 8) as u32)); + source = hash(&buf[..]); + } + + if j & 0x07 == 0x07 { + byte_v = source[(j & 0xff) >> 3]; + } + let bit_v = (byte_v >> (j & 0x07)) & 0x01; + + if bit_v == 1 { + input.swap(i, j); + } + } + + if forwards { + r += 1; + if r == rounds { + break; + } + } else { + if r == 0 { + break; + } + r -= 1; + } + } + + Some(input) +} + +fn bytes_to_int64(bytes: &[u8]) -> u64 { + let mut cursor = Cursor::new(bytes); + cursor.get_u64_le() +} + +#[cfg(test)] +mod tests { + use super::*; + use hex; + use std::{fs::File, io::prelude::*, path::PathBuf}; + use yaml_rust::yaml; + + #[test] + fn returns_none_for_zero_length_list() { + assert_eq!(None, shuffle_list(vec![], 90, &[42, 42], true)); + } + + #[test] + fn test_vectors() { + let mut file = { + let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + file_path_buf.push("src/specs/test_vector_permutated_index.yml"); + + File::open(file_path_buf).unwrap() + }; + + let mut yaml_str = String::new(); + + file.read_to_string(&mut yaml_str).unwrap(); + + let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap(); + let doc = &docs[0]; + let test_cases = doc["test_cases"].as_vec().unwrap(); + + for (i, test_case) in test_cases.iter().enumerate() { + let index = test_case["index"].as_i64().unwrap() as usize; + let list_size = test_case["list_size"].as_i64().unwrap() as usize; + let permutated_index = test_case["permutated_index"].as_i64().unwrap() as usize; + let shuffle_round_count = test_case["shuffle_round_count"].as_i64().unwrap(); + let seed_string = test_case["seed"].clone().into_string().unwrap(); + let seed = hex::decode(seed_string.replace("0x", "")).unwrap(); + + let shuffle_round_count = if shuffle_round_count < (u8::max_value() as i64) { + shuffle_round_count as u8 + } else { + panic!("shuffle_round_count must be a u8") + }; + + let list: Vec = (0..list_size).collect(); + + let shuffled = + shuffle_list(list.clone(), shuffle_round_count, &seed[..], true).unwrap(); + + assert_eq!( + list[index], shuffled[permutated_index], + "Failure on case #{} index: {}, list_size: {}, round_count: {}, seed: {}", + i, index, list_size, shuffle_round_count, seed_string + ); + } + } +} From 3753782c40187d06df310730f773acfef916d5fa Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Fri, 1 Mar 2019 13:10:50 +1100 Subject: [PATCH 7/8] Just some small gramatical improvements on READMEs --- README.md | 8 ++++---- eth2/README.md | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index c26dae219..8552d64ee 100644 --- a/README.md +++ b/README.md @@ -33,12 +33,12 @@ If you'd like some background on Sigma Prime, please see the [Lighthouse Update - [`beacon_node/`](beacon_node/): the "Beacon Node" binary and crates exclusively associated with it. -- [`docs/`](docs/): documentation related to the repository. This include contributor - guides, etc. (Code documentation is produced with `cargo doc`). +- [`docs/`](docs/): documentation related to the repository. This includes contributor + guides, etc. (It does not include code documentation, which can be produced with `cargo doc`). - [`eth2/`](eth2/): Crates containing common logic across the Lighthouse project. For - example; Ethereum 2.0 types ([`BeaconBlock`](eth2/types/src/beacon_block.rs), [`BeaconState`](eth2/types/src/beacon_state.rs), etc) and + example: Ethereum 2.0 types ([`BeaconBlock`](eth2/types/src/beacon_block.rs), [`BeaconState`](eth2/types/src/beacon_state.rs), etc) and SimpleSerialize (SSZ). -- [`protos/`](protos/): protobuf/gRPC definitions common across the Lighthouse project. +- [`protos/`](protos/): protobuf/gRPC definitions that are common across the Lighthouse project. - [`validator_client/`](validator_client/): the "Validator Client" binary and crates exclusively associated with it. diff --git a/eth2/README.md b/eth2/README.md index e7b69635a..cf041e987 100644 --- a/eth2/README.md +++ b/eth2/README.md @@ -8,19 +8,19 @@ Rust crates containing logic common across the Lighthouse project. - [`block_proposer/`](block_proposer/): Core logic for proposing beacon blocks. - [`fork_choice/`](fork_choice/): A collection of fork-choice algorithms for the Beacon Chain. -- [`state_processing/`](state_processing/): Provides per-slot, per-block and +- [`state_processing/`](state_processing/): Provides per-slot, per-block, and per-epoch state processing. - [`types/`](types/): Defines base Ethereum 2.0 types (e.g., `BeaconBlock`, `BeaconState`, etc). - [`utils/`](utils/): - - [`bls`](utils/bls/): A wrapper around an external BLS encryption library. - - [`boolean-bitfield`](utils/boolean-bitfield/): Provides an expandable Vec + - [`bls`](utils/bls/): A wrapper for an external BLS encryption library. + - [`boolean-bitfield`](utils/boolean-bitfield/): Provides an expandable vector of bools, specifically for use in Eth2. - [`fisher-yates-shuffle`](utils/fisher-yates-shuffle/): shuffles a list pseudo-randomly. - - [`hashing`](utils/hashing/): A wrapper around external hashing libraries. + - [`hashing`](utils/hashing/): A wrapper for external hashing libraries. - [`honey-badger-split`](utils/honey-badger-split/): Splits a list in `n` - parts without giving AF about the length of the list, `n` or anything + parts without giving AF about the length of the list, `n`, or anything else. - [`int-to-bytes`](utils/int-to-bytes/): Simple library which converts ints into byte-strings of various lengths. @@ -30,8 +30,8 @@ Rust crates containing logic common across the Lighthouse project. - [`ssz`](utils/ssz/): an implementation of the SimpleSerialize serialization/deserialization protocol used by Eth 2.0. - [`ssz_derive`](utils/ssz_derive/): provides procedural macros for - deriving SSZ `Encodable`, `Decodable` and `TreeHash` methods. + deriving SSZ `Encodable`, `Decodable`, and `TreeHash` methods. - [`swap_or_not_shuffle`](utils/swap_or_not_shuffle/): a list-shuffling - method which is slow, but allows for shuffling a subset of indices. + method which is slow, but allows for a subset of indices to be shuffled. - [`test_random_derive`](utils/test_random_derive/): provides procedural macros for deriving the `TestRandom` trait defined in `types`. From 13c957bef7623fbac689330b0aa221b404443e14 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Fri, 1 Mar 2019 14:38:07 +1100 Subject: [PATCH 8/8] Correct bitwise fork-choice rule. --- eth2/fork_choice/src/bitwise_lmd_ghost.rs | 9 ++++-- .../tests/bitwise_lmd_ghost_test_vectors.yaml | 28 +++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/eth2/fork_choice/src/bitwise_lmd_ghost.rs b/eth2/fork_choice/src/bitwise_lmd_ghost.rs index e1d246e92..1e66de079 100644 --- a/eth2/fork_choice/src/bitwise_lmd_ghost.rs +++ b/eth2/fork_choice/src/bitwise_lmd_ghost.rs @@ -192,7 +192,7 @@ where } // Check if there is a clear block winner at this height. If so return it. for (hash, votes) in current_votes.iter() { - if *votes >= total_vote_count / 2 { + if *votes > total_vote_count / 2 { // we have a clear winner, return it return Some(*hash); } @@ -371,7 +371,10 @@ impl ForkChoice for BitwiseLMDGhost { // if there are no children, we are done, return the current_head let children = match self.children.get(¤t_head) { Some(children) => children.clone(), - None => return Ok(current_head), + None => { + debug!("Head found: {}", current_head); + return Ok(current_head); + } }; // logarithmic lookup blocks to see if there are obvious winners, if so, @@ -391,7 +394,7 @@ impl ForkChoice for BitwiseLMDGhost { step /= 2; } if step > 0 { - trace!("Found clear winner in log lookup"); + trace!("Found clear winner: {}", current_head); } // if our skip lookup failed and we only have one child, progress to that child else if children.len() == 1 { diff --git a/eth2/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml b/eth2/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml index 1578673cd..3233137ab 100644 --- a/eth2/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml +++ b/eth2/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml @@ -35,3 +35,31 @@ test_cases: - b3: 3 heads: - id: 'b2' +- blocks: + - id: 'b0' + parent: 'b0' + - id: 'b1' + parent: 'b0' + - id: 'b2' + parent: 'b0' + - id: 'b3' + parent: 'b1' + - id: 'b4' + parent: 'b1' + - id: 'b5' + parent: 'b1' + - id: 'b6' + parent: 'b2' + - id: 'b7' + parent: 'b6' + weights: + - b0: 0 + - b1: 3 + - b2: 2 + - b3: 1 + - b4: 1 + - b5: 1 + - b6: 2 + - b7: 2 + heads: + - id: 'b4'