diff --git a/.gitignore b/.gitignore index 9050bdab9..346ef9afa 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ target/ Cargo.lock *.pk *.sk +*.raw_keypairs diff --git a/Cargo.toml b/Cargo.toml index 58395f82a..89158542e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "eth2/utils/honey-badger-split", "eth2/utils/merkle_proof", "eth2/utils/int_to_bytes", + "eth2/utils/serde_hex", "eth2/utils/slot_clock", "eth2/utils/ssz", "eth2/utils/ssz_derive", diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 6be030fd5..e7aaf938d 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -16,3 +16,4 @@ ctrlc = { version = "3.1.1", features = ["termination"] } tokio = "0.1.15" futures = "0.1.25" exit-future = "0.1.3" +state_processing = { path = "../eth2/state_processing" } diff --git a/beacon_node/beacon_chain/src/attestation_aggregator.rs b/beacon_node/beacon_chain/src/attestation_aggregator.rs index 75cfd7ee5..9b4e5a687 100644 --- a/beacon_node/beacon_chain/src/attestation_aggregator.rs +++ b/beacon_node/beacon_chain/src/attestation_aggregator.rs @@ -1,4 +1,3 @@ -use log::trace; use ssz::TreeHash; use state_processing::per_block_processing::validate_attestation_without_signature; use std::collections::{HashMap, HashSet}; @@ -86,34 +85,22 @@ impl AttestationAggregator { free_attestation: &FreeAttestation, spec: &ChainSpec, ) -> Result { - let attestation_duties = match state.attestation_slot_and_shard_for_validator( - free_attestation.validator_index as usize, - spec, - ) { - Err(BeaconStateError::EpochCacheUninitialized(e)) => { - panic!("Attempted to access unbuilt cache {:?}.", e) - } - Err(BeaconStateError::EpochOutOfBounds) => invalid_outcome!(Message::TooOld), - Err(BeaconStateError::ShardOutOfBounds) => invalid_outcome!(Message::BadShard), - Err(e) => return Err(e), - Ok(None) => invalid_outcome!(Message::BadValidatorIndex), - Ok(Some(attestation_duties)) => attestation_duties, - }; + let duties = + match state.get_attestation_duties(free_attestation.validator_index as usize, spec) { + Err(BeaconStateError::EpochCacheUninitialized(e)) => { + panic!("Attempted to access unbuilt cache {:?}.", e) + } + Err(BeaconStateError::EpochOutOfBounds) => invalid_outcome!(Message::TooOld), + Err(BeaconStateError::ShardOutOfBounds) => invalid_outcome!(Message::BadShard), + Err(e) => return Err(e), + Ok(None) => invalid_outcome!(Message::BadValidatorIndex), + Ok(Some(attestation_duties)) => attestation_duties, + }; - let (slot, shard, committee_index) = attestation_duties; - - trace!( - "slot: {}, shard: {}, committee_index: {}, val_index: {}", - slot, - shard, - committee_index, - free_attestation.validator_index - ); - - if free_attestation.data.slot != slot { + if free_attestation.data.slot != duties.slot { invalid_outcome!(Message::BadSlot); } - if free_attestation.data.shard != shard { + if free_attestation.data.shard != duties.shard { invalid_outcome!(Message::BadShard); } @@ -143,7 +130,7 @@ impl AttestationAggregator { if let Some(updated_attestation) = aggregate_attestation( existing_attestation, &free_attestation.signature, - committee_index as usize, + duties.committee_index as usize, ) { self.store.insert(signable_message, updated_attestation); valid_outcome!(Message::Aggregated); @@ -154,7 +141,7 @@ impl AttestationAggregator { let mut aggregate_signature = AggregateSignature::new(); aggregate_signature.add(&free_attestation.signature); let mut aggregation_bitfield = Bitfield::new(); - aggregation_bitfield.set(committee_index as usize, true); + aggregation_bitfield.set(duties.committee_index as usize, true); let new_attestation = Attestation { data: free_attestation.data.clone(), aggregation_bitfield, @@ -177,9 +164,13 @@ impl AttestationAggregator { ) -> Vec { let mut known_attestation_data: HashSet = HashSet::new(); - state.latest_attestations.iter().for_each(|attestation| { - known_attestation_data.insert(attestation.data.clone()); - }); + state + .previous_epoch_attestations + .iter() + .chain(state.current_epoch_attestations.iter()) + .for_each(|attestation| { + known_attestation_data.insert(attestation.data.clone()); + }); self.store .values() diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 3d2efa8ae..bf87adf10 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -15,10 +15,7 @@ use state_processing::{ per_slot_processing, BlockProcessingError, SlotProcessingError, }; use std::sync::Arc; -use types::{ - readers::{BeaconBlockReader, BeaconStateReader}, - *, -}; +use types::*; #[derive(Debug, PartialEq)] pub enum ValidBlock { @@ -73,45 +70,30 @@ where F: ForkChoice, { /// Instantiate a new Beacon Chain, from genesis. - #[allow(clippy::too_many_arguments)] // Will be re-factored in the coming weeks. - pub fn genesis( + pub fn from_genesis( state_store: Arc>, block_store: Arc>, slot_clock: U, - genesis_time: u64, - latest_eth1_data: Eth1Data, - initial_validator_deposits: Vec, + mut genesis_state: BeaconState, + genesis_block: BeaconBlock, spec: ChainSpec, fork_choice: F, ) -> Result { - if initial_validator_deposits.is_empty() { - return Err(Error::InsufficientValidators); - } - - let mut genesis_state = BeaconState::genesis( - genesis_time, - initial_validator_deposits, - latest_eth1_data, - &spec, - )?; let state_root = genesis_state.canonical_root(); state_store.put(&state_root, &ssz_encode(&genesis_state)[..])?; - let genesis_block = BeaconBlock::genesis(state_root, &spec); - let block_root = genesis_block.canonical_root(); + let block_root = genesis_block.into_header().canonical_root(); block_store.put(&block_root, &ssz_encode(&genesis_block)[..])?; let finalized_head = RwLock::new(CheckPoint::new( genesis_block.clone(), block_root, - // TODO: this is a memory waste; remove full clone. genesis_state.clone(), state_root, )); let canonical_head = RwLock::new(CheckPoint::new( genesis_block.clone(), block_root, - // TODO: this is a memory waste; remove full clone. genesis_state.clone(), state_root, )); @@ -119,7 +101,8 @@ where genesis_state.build_epoch_cache(RelativeEpoch::Previous, &spec)?; genesis_state.build_epoch_cache(RelativeEpoch::Current, &spec)?; - genesis_state.build_epoch_cache(RelativeEpoch::Next, &spec)?; + genesis_state.build_epoch_cache(RelativeEpoch::NextWithoutRegistryChange, &spec)?; + genesis_state.build_epoch_cache(RelativeEpoch::NextWithRegistryChange, &spec)?; Ok(Self { block_store, @@ -205,10 +188,13 @@ where /// processing applied to it. pub fn advance_state(&self, slot: Slot) -> Result<(), SlotProcessingError> { let state_slot = self.state.read().slot; - let head_block_root = self.head().beacon_block_root; + + let latest_block_header = self.head().beacon_block.into_header(); + for _ in state_slot.as_u64()..slot.as_u64() { - per_slot_processing(&mut *self.state.write(), head_block_root, &self.spec)?; + per_slot_processing(&mut *self.state.write(), &latest_block_header, &self.spec)?; } + Ok(()) } @@ -261,19 +247,15 @@ where /// present and prior epoch is available. pub fn block_proposer(&self, slot: Slot) -> Result { trace!("BeaconChain::block_proposer: slot: {}", slot); - let index = self - .state - .read() - .get_beacon_proposer_index(slot, &self.spec)?; + let index = self.state.read().get_beacon_proposer_index( + slot, + RelativeEpoch::Current, + &self.spec, + )?; Ok(index) } - /// Returns the justified slot for the present state. - pub fn justified_epoch(&self) -> Epoch { - self.state.read().justified_epoch - } - /// Returns the attestation slot and shard for a given validator index. /// /// Information is read from the current state, so only information from the present and prior @@ -286,12 +268,12 @@ where "BeaconChain::validator_attestion_slot_and_shard: validator_index: {}", validator_index ); - if let Some((slot, shard, _committee)) = self + if let Some(attestation_duty) = self .state .read() - .attestation_slot_and_shard_for_validator(validator_index, &self.spec)? + .get_attestation_duties(validator_index, &self.spec)? { - Ok(Some((slot, shard))) + Ok(Some((attestation_duty.slot, attestation_duty.shard))) } else { Ok(None) } @@ -300,37 +282,33 @@ where /// Produce an `AttestationData` that is valid for the present `slot` and given `shard`. pub fn produce_attestation_data(&self, shard: u64) -> Result { trace!("BeaconChain::produce_attestation_data: shard: {}", shard); - let justified_epoch = self.justified_epoch(); - let justified_block_root = *self - .state - .read() - .get_block_root( - justified_epoch.start_slot(self.spec.slots_per_epoch), - &self.spec, - ) - .ok_or_else(|| Error::BadRecentBlockRoots)?; + let source_epoch = self.state.read().current_justified_epoch; + let source_root = *self.state.read().get_block_root( + source_epoch.start_slot(self.spec.slots_per_epoch), + &self.spec, + )?; - let epoch_boundary_root = *self - .state - .read() - .get_block_root( - self.state.read().current_epoch_start_slot(&self.spec), - &self.spec, - ) - .ok_or_else(|| Error::BadRecentBlockRoots)?; + let target_root = *self.state.read().get_block_root( + self.state + .read() + .slot + .epoch(self.spec.slots_per_epoch) + .start_slot(self.spec.slots_per_epoch), + &self.spec, + )?; Ok(AttestationData { slot: self.state.read().slot, shard, beacon_block_root: self.head().beacon_block_root, - epoch_boundary_root, + target_root, crosslink_data_root: Hash256::zero(), - latest_crosslink: Crosslink { + previous_crosslink: Crosslink { epoch: self.state.read().slot.epoch(self.spec.slots_per_epoch), crosslink_data_root: Hash256::zero(), }, - justified_epoch, - justified_block_root, + source_epoch, + source_root, }) } @@ -577,66 +555,13 @@ where } } - /// Dumps the entire canonical chain, from the head to genesis to a vector for analysis. - /// - /// This could be a very expensive operation and should only be done in testing/analysis - /// activities. - pub fn chain_dump(&self) -> Result, Error> { - let mut dump = vec![]; - - let mut last_slot = CheckPoint { - beacon_block: self.head().beacon_block.clone(), - beacon_block_root: self.head().beacon_block_root, - beacon_state: self.head().beacon_state.clone(), - beacon_state_root: self.head().beacon_state_root, - }; - - dump.push(last_slot.clone()); - - loop { - let beacon_block_root = last_slot.beacon_block.parent_root; - - if beacon_block_root == self.spec.zero_hash { - break; // Genesis has been reached. - } - - let beacon_block = self - .block_store - .get_deserialized(&beacon_block_root)? - .ok_or_else(|| { - Error::DBInconsistent(format!("Missing block {}", beacon_block_root)) - })?; - let beacon_state_root = beacon_block.state_root; - let beacon_state = self - .state_store - .get_deserialized(&beacon_state_root)? - .ok_or_else(|| { - Error::DBInconsistent(format!("Missing state {}", beacon_state_root)) - })?; - - let slot = CheckPoint { - beacon_block, - beacon_block_root, - beacon_state, - beacon_state_root, - }; - - dump.push(slot.clone()); - last_slot = slot; - } - - dump.reverse(); - - Ok(dump) - } - /// Accept some block and attempt to add it to block DAG. /// /// Will accept blocks from prior slots, however it will reject any block from a future slot. pub fn process_block(&self, block: BeaconBlock) -> Result { - debug!("Processing block with slot {}...", block.slot()); + debug!("Processing block with slot {}...", block.slot); - let block_root = block.canonical_root(); + let block_root = block.into_header().canonical_root(); let present_slot = self.present_slot(); @@ -648,9 +573,9 @@ where // Load the blocks parent block from the database, returning invalid if that block is not // found. - let parent_block_root = block.parent_root; - let parent_block = match self.block_store.get_reader(&parent_block_root)? { - Some(parent_root) => parent_root, + let parent_block_root = block.previous_block_root; + let parent_block = match self.block_store.get_deserialized(&parent_block_root)? { + Some(previous_block_root) => previous_block_root, None => { return Ok(BlockProcessingOutcome::InvalidBlock( InvalidBlock::ParentUnknown, @@ -660,23 +585,21 @@ where // Load the parent blocks state from the database, returning an error if it is not found. // It is an error because if know the parent block we should also know the parent state. - let parent_state_root = parent_block.state_root(); + let parent_state_root = parent_block.state_root; let parent_state = self .state_store - .get_reader(&parent_state_root)? - .ok_or_else(|| Error::DBInconsistent(format!("Missing state {}", parent_state_root)))? - .into_beacon_state() - .ok_or_else(|| { - Error::DBInconsistent(format!("State SSZ invalid {}", parent_state_root)) - })?; + .get_deserialized(&parent_state_root)? + .ok_or_else(|| Error::DBInconsistent(format!("Missing state {}", parent_state_root)))?; // TODO: check the block proposer signature BEFORE doing a state transition. This will // significantly lower exposure surface to DoS attacks. // Transition the parent state to the present slot. let mut state = parent_state; + println!("parent process state: {:?}", state.latest_block_header); + let previous_block_header = parent_block.into_header(); for _ in state.slot.as_u64()..present_slot.as_u64() { - if let Err(e) = per_slot_processing(&mut state, parent_block_root, &self.spec) { + if let Err(e) = per_slot_processing(&mut state, &previous_block_header, &self.spec) { return Ok(BlockProcessingOutcome::InvalidBlock( InvalidBlock::SlotProcessingError(e), )); @@ -691,6 +614,8 @@ where )); } + println!("process state: {:?}", state.latest_block_header); + let state_root = state.canonical_root(); if block.state_root != state_root { @@ -752,22 +677,22 @@ where attestations.len() ); - let parent_root = *state - .get_block_root(state.slot.saturating_sub(1_u64), &self.spec) - .ok_or_else(|| BlockProductionError::UnableToGetBlockRootFromState)?; + let previous_block_root = *state + .get_block_root(state.slot - 1, &self.spec) + .map_err(|_| BlockProductionError::UnableToGetBlockRootFromState)?; let mut block = BeaconBlock { slot: state.slot, - parent_root, + previous_block_root, state_root: Hash256::zero(), // Updated after the state is calculated. - randao_reveal, - eth1_data: Eth1Data { - // TODO: replace with real data - deposit_root: Hash256::zero(), - block_hash: Hash256::zero(), - }, signature: self.spec.empty_signature.clone(), // To be completed by a validator. body: BeaconBlockBody { + randao_reveal, + eth1_data: Eth1Data { + // TODO: replace with real data + deposit_root: Hash256::zero(), + block_hash: Hash256::zero(), + }, proposer_slashings: self.get_proposer_slashings_for_block(), attester_slashings: self.get_attester_slashings_for_block(), attestations, @@ -781,6 +706,8 @@ where per_block_processing_without_verifying_block_signature(&mut state, &block, &self.spec)?; + println!("produce state: {:?}", state.latest_block_header); + let state_root = state.canonical_root(); block.state_root = state_root; @@ -815,6 +742,59 @@ where Ok(()) } + + /// Dumps the entire canonical chain, from the head to genesis to a vector for analysis. + /// + /// This could be a very expensive operation and should only be done in testing/analysis + /// activities. + pub fn chain_dump(&self) -> Result, Error> { + let mut dump = vec![]; + + let mut last_slot = CheckPoint { + beacon_block: self.head().beacon_block.clone(), + beacon_block_root: self.head().beacon_block_root, + beacon_state: self.head().beacon_state.clone(), + beacon_state_root: self.head().beacon_state_root, + }; + + dump.push(last_slot.clone()); + + loop { + let beacon_block_root = last_slot.beacon_block.previous_block_root; + + if beacon_block_root == self.spec.zero_hash { + break; // Genesis has been reached. + } + + let beacon_block = self + .block_store + .get_deserialized(&beacon_block_root)? + .ok_or_else(|| { + Error::DBInconsistent(format!("Missing block {}", beacon_block_root)) + })?; + let beacon_state_root = beacon_block.state_root; + let beacon_state = self + .state_store + .get_deserialized(&beacon_state_root)? + .ok_or_else(|| { + Error::DBInconsistent(format!("Missing state {}", beacon_state_root)) + })?; + + let slot = CheckPoint { + beacon_block, + beacon_block_root, + beacon_state, + beacon_state_root, + }; + + dump.push(slot.clone()); + last_slot = slot; + } + + dump.reverse(); + + Ok(dump) + } } impl From for Error { diff --git a/beacon_node/beacon_chain/src/initialise.rs b/beacon_node/beacon_chain/src/initialise.rs index 21b145d42..7d3c87965 100644 --- a/beacon_node/beacon_chain/src/initialise.rs +++ b/beacon_node/beacon_chain/src/initialise.rs @@ -3,19 +3,20 @@ // testnet. These are examples. Also. there is code duplication which can/should be cleaned up. use crate::BeaconChain; -use bls; use db::stores::{BeaconBlockStore, BeaconStateStore}; use db::{DiskDB, MemoryDB}; use fork_choice::BitwiseLMDGhost; use slot_clock::SystemTimeSlotClock; +use ssz::TreeHash; use std::path::PathBuf; use std::sync::Arc; -use types::{ChainSpec, Deposit, DepositData, DepositInput, Eth1Data, Hash256, Keypair}; +use types::test_utils::TestingBeaconStateBuilder; +use types::{BeaconBlock, ChainSpec, Hash256}; //TODO: Correct this for prod //TODO: Account for historical db pub fn initialise_beacon_chain( - chain_spec: &ChainSpec, + spec: &ChainSpec, db_name: Option<&PathBuf>, ) -> Arc>> { // set up the db @@ -23,124 +24,71 @@ pub fn initialise_beacon_chain( db_name.expect("Database directory must be included"), None, )); + let block_store = Arc::new(BeaconBlockStore::new(db.clone())); let state_store = Arc::new(BeaconStateStore::new(db.clone())); + let state_builder = TestingBeaconStateBuilder::from_deterministic_keypairs(8, &spec); + let (genesis_state, _keypairs) = state_builder.build(); + + let mut genesis_block = BeaconBlock::empty(&spec); + genesis_block.state_root = Hash256::from_slice(&genesis_state.hash_tree_root()); + // Slot clock - let genesis_time = 1_549_935_547; // 12th Feb 2018 (arbitrary value in the past). - let slot_clock = SystemTimeSlotClock::new(genesis_time, chain_spec.seconds_per_slot) + let slot_clock = SystemTimeSlotClock::new(genesis_state.genesis_time, spec.seconds_per_slot) .expect("Unable to load SystemTimeSlotClock"); // Choose the fork choice let fork_choice = BitwiseLMDGhost::new(block_store.clone(), state_store.clone()); - /* - * Generate some random data to start a chain with. - * - * This is will need to be replace for production usage. - */ - let latest_eth1_data = Eth1Data { - deposit_root: Hash256::zero(), - block_hash: Hash256::zero(), - }; - let keypairs: Vec = (0..10) - .collect::>() - .iter() - .map(|_| Keypair::random()) - .collect(); - let initial_validator_deposits = keypairs - .iter() - .map(|keypair| Deposit { - branch: vec![], // branch verification is not chain_specified. - index: 0, // index verification is not chain_specified. - deposit_data: DepositData { - amount: 32_000_000_000, // 32 ETH (in Gwei) - timestamp: genesis_time - 1, - deposit_input: DepositInput { - pubkey: keypair.pk.clone(), - withdrawal_credentials: Hash256::zero(), // Withdrawal not possible. - proof_of_possession: bls::create_proof_of_possession(&keypair), - }, - }, - }) - .collect(); - // Genesis chain - // TODO:Remove the expect here. Propagate errors and handle somewhat gracefully. + //TODO: Handle error correctly Arc::new( - BeaconChain::genesis( + BeaconChain::from_genesis( state_store.clone(), block_store.clone(), slot_clock, - genesis_time, - latest_eth1_data, - initial_validator_deposits, - chain_spec.clone(), + genesis_state, + genesis_block, + spec.clone(), fork_choice, ) - .expect("Cannot initialise a beacon chain. Exiting"), + .expect("Terminate if beacon chain generation fails"), ) } /// Initialisation of a test beacon chain, uses an in memory db with fixed genesis time. pub fn initialise_test_beacon_chain( - chain_spec: &ChainSpec, + spec: &ChainSpec, _db_name: Option<&PathBuf>, ) -> Arc>> { let db = Arc::new(MemoryDB::open()); let block_store = Arc::new(BeaconBlockStore::new(db.clone())); let state_store = Arc::new(BeaconStateStore::new(db.clone())); + let state_builder = TestingBeaconStateBuilder::from_deterministic_keypairs(8, spec); + let (genesis_state, _keypairs) = state_builder.build(); + + let mut genesis_block = BeaconBlock::empty(spec); + genesis_block.state_root = Hash256::from_slice(&genesis_state.hash_tree_root()); + // Slot clock - let genesis_time = 1_549_935_547; // 12th Feb 2018 (arbitrary value in the past). - let slot_clock = SystemTimeSlotClock::new(genesis_time, chain_spec.seconds_per_slot) + let slot_clock = SystemTimeSlotClock::new(genesis_state.genesis_time, spec.seconds_per_slot) .expect("Unable to load SystemTimeSlotClock"); // Choose the fork choice let fork_choice = BitwiseLMDGhost::new(block_store.clone(), state_store.clone()); - /* - * Generate some random data to start a chain with. - * - * This is will need to be replace for production usage. - */ - let latest_eth1_data = Eth1Data { - deposit_root: Hash256::zero(), - block_hash: Hash256::zero(), - }; - let keypairs: Vec = (0..8) - .collect::>() - .iter() - .map(|_| Keypair::random()) - .collect(); - let initial_validator_deposits = keypairs - .iter() - .map(|keypair| Deposit { - branch: vec![], // branch verification is not chain_specified. - index: 0, // index verification is not chain_specified. - deposit_data: DepositData { - amount: 32_000_000_000, // 32 ETH (in Gwei) - timestamp: genesis_time - 1, - deposit_input: DepositInput { - pubkey: keypair.pk.clone(), - withdrawal_credentials: Hash256::zero(), // Withdrawal not possible. - proof_of_possession: bls::create_proof_of_possession(&keypair), - }, - }, - }) - .collect(); - // Genesis chain - // TODO: Handle error correctly + //TODO: Handle error correctly Arc::new( - BeaconChain::genesis( + BeaconChain::from_genesis( state_store.clone(), block_store.clone(), slot_clock, - genesis_time, - latest_eth1_data, - initial_validator_deposits, - chain_spec.clone(), + genesis_state, + genesis_block, + spec.clone(), fork_choice, ) - .expect("Cannot generate beacon chain"), + .expect("Terminate if beacon chain generation fails"), ) } diff --git a/beacon_node/beacon_chain/test_harness/Cargo.toml b/beacon_node/beacon_chain/test_harness/Cargo.toml index bd7a58270..50d154732 100644 --- a/beacon_node/beacon_chain/test_harness/Cargo.toml +++ b/beacon_node/beacon_chain/test_harness/Cargo.toml @@ -12,12 +12,7 @@ path = "src/bin.rs" name = "test_harness" path = "src/lib.rs" -[[bench]] -name = "state_transition" -harness = false - [dev-dependencies] -criterion = "0.2" state_processing = { path = "../../../eth2/state_processing" } [dependencies] @@ -33,12 +28,14 @@ failure = "0.1" failure_derive = "0.1" fork_choice = { path = "../../../eth2/fork_choice" } hashing = { path = "../../../eth2/utils/hashing" } +int_to_bytes = { path = "../../../eth2/utils/int_to_bytes" } log = "0.4" env_logger = "0.6.0" rayon = "1.0" serde = "1.0" serde_derive = "1.0" serde_json = "1.0" +serde_yaml = "0.8" slot_clock = { path = "../../../eth2/utils/slot_clock" } ssz = { path = "../../../eth2/utils/ssz" } types = { path = "../../../eth2/types" } diff --git a/beacon_node/beacon_chain/test_harness/benches/state_transition.rs b/beacon_node/beacon_chain/test_harness/benches/state_transition.rs deleted file mode 100644 index 7d1c44653..000000000 --- a/beacon_node/beacon_chain/test_harness/benches/state_transition.rs +++ /dev/null @@ -1,69 +0,0 @@ -use criterion::Criterion; -use criterion::{black_box, criterion_group, criterion_main, Benchmark}; -// use env_logger::{Builder, Env}; -use state_processing::SlotProcessable; -use test_harness::BeaconChainHarness; -use types::{ChainSpec, Hash256}; - -fn mid_epoch_state_transition(c: &mut Criterion) { - // Builder::from_env(Env::default().default_filter_or("debug")).init(); - - let validator_count = 1000; - let mut rig = BeaconChainHarness::new(ChainSpec::foundation(), validator_count); - - let epoch_depth = (rig.spec.slots_per_epoch * 2) + (rig.spec.slots_per_epoch / 2); - - for _ in 0..epoch_depth { - rig.advance_chain_with_block(); - } - - let state = rig.beacon_chain.state.read().clone(); - - assert!((state.slot + 1) % rig.spec.slots_per_epoch != 0); - - c.bench_function("mid-epoch state transition 10k validators", move |b| { - let state = state.clone(); - b.iter(|| { - let mut state = state.clone(); - black_box(state.per_slot_processing(Hash256::zero(), &rig.spec)) - }) - }); -} - -fn epoch_boundary_state_transition(c: &mut Criterion) { - // Builder::from_env(Env::default().default_filter_or("debug")).init(); - - let validator_count = 10000; - let mut rig = BeaconChainHarness::new(ChainSpec::foundation(), validator_count); - - let epoch_depth = rig.spec.slots_per_epoch * 2; - - for _ in 0..(epoch_depth - 1) { - rig.advance_chain_with_block(); - } - - let state = rig.beacon_chain.state.read().clone(); - - assert_eq!((state.slot + 1) % rig.spec.slots_per_epoch, 0); - - c.bench( - "routines", - Benchmark::new("routine_1", move |b| { - let state = state.clone(); - b.iter(|| { - let mut state = state.clone(); - black_box(black_box( - state.per_slot_processing(Hash256::zero(), &rig.spec), - )) - }) - }) - .sample_size(5), // sample size is low because function is sloooow. - ); -} - -criterion_group!( - benches, - mid_epoch_state_transition, - epoch_boundary_state_transition -); -criterion_main!(benches); diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index f220619ce..bc5c93b94 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -1,7 +1,6 @@ use super::ValidatorHarness; use beacon_chain::{BeaconChain, BlockProcessingOutcome}; pub use beacon_chain::{BeaconChainError, CheckPoint}; -use bls::{create_proof_of_possession, get_withdrawal_credentials}; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, MemoryDB, @@ -10,10 +9,11 @@ use fork_choice::BitwiseLMDGhost; use log::debug; use rayon::prelude::*; use slot_clock::TestingSlotClock; +use ssz::TreeHash; use std::collections::HashSet; use std::iter::FromIterator; use std::sync::Arc; -use types::*; +use types::{test_utils::TestingBeaconStateBuilder, *}; /// The beacon chain harness simulates a single beacon node with `validator_count` validators connected /// to it. Each validator is provided a borrow to the beacon chain, where it may read @@ -39,58 +39,24 @@ impl BeaconChainHarness { let db = Arc::new(MemoryDB::open()); let block_store = Arc::new(BeaconBlockStore::new(db.clone())); let state_store = Arc::new(BeaconStateStore::new(db.clone())); - let genesis_time = 1_549_935_547; // 12th Feb 2018 (arbitrary value in the past). let slot_clock = TestingSlotClock::new(spec.genesis_slot.as_u64()); let fork_choice = BitwiseLMDGhost::new(block_store.clone(), state_store.clone()); - let latest_eth1_data = Eth1Data { - deposit_root: Hash256::zero(), - block_hash: Hash256::zero(), - }; - debug!("Generating validator keypairs..."); + let state_builder = + TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, &spec); + let (genesis_state, keypairs) = state_builder.build(); - let keypairs: Vec = (0..validator_count) - .collect::>() - .par_iter() - .map(|_| Keypair::random()) - .collect(); - - debug!("Creating validator deposits..."); - - let initial_validator_deposits = keypairs - .par_iter() - .map(|keypair| Deposit { - branch: vec![], // branch verification is not specified. - index: 0, // index verification is not specified. - deposit_data: DepositData { - amount: 32_000_000_000, // 32 ETH (in Gwei) - timestamp: genesis_time - 1, - deposit_input: DepositInput { - pubkey: keypair.pk.clone(), - // Validator can withdraw using their main keypair. - withdrawal_credentials: Hash256::from_slice( - &get_withdrawal_credentials( - &keypair.pk, - spec.bls_withdrawal_prefix_byte, - )[..], - ), - proof_of_possession: create_proof_of_possession(&keypair), - }, - }, - }) - .collect(); - - debug!("Creating the BeaconChain..."); + let mut genesis_block = BeaconBlock::empty(&spec); + genesis_block.state_root = Hash256::from_slice(&genesis_state.hash_tree_root()); // Create the Beacon Chain let beacon_chain = Arc::new( - BeaconChain::genesis( + BeaconChain::from_genesis( state_store.clone(), block_store.clone(), slot_clock, - genesis_time, - latest_eth1_data, - initial_validator_deposits, + genesis_state, + genesis_block, spec.clone(), fork_choice, ) @@ -161,8 +127,8 @@ impl BeaconChainHarness { .get_crosslink_committees_at_slot(present_slot, &self.spec) .unwrap() .iter() - .fold(vec![], |mut acc, (committee, _slot)| { - acc.append(&mut committee.clone()); + .fold(vec![], |mut acc, c| { + acc.append(&mut c.committee.clone()); acc }); let attesting_validators: HashSet = @@ -267,6 +233,27 @@ impl BeaconChainHarness { Some(Signature::new(message, domain, &validator.keypair.sk)) } + /// Returns the current `Fork` of the `beacon_chain`. + pub fn fork(&self) -> Fork { + self.beacon_chain.state.read().fork.clone() + } + + /// Returns the current `epoch` of the `beacon_chain`. + pub fn epoch(&self) -> Epoch { + self.beacon_chain + .state + .read() + .slot + .epoch(self.spec.slots_per_epoch) + } + + /// Returns the keypair for some validator index. + pub fn validator_keypair(&self, validator_index: usize) -> Option<&Keypair> { + self.validators + .get(validator_index) + .and_then(|v| Some(&v.keypair)) + } + /// Submit a deposit to the `BeaconChain` and, if given a keypair, create a new /// `ValidatorHarness` instance for this validator. /// diff --git a/beacon_node/beacon_chain/test_harness/src/bin.rs b/beacon_node/beacon_chain/test_harness/src/bin.rs index 283cb0dfa..3afc921de 100644 --- a/beacon_node/beacon_chain/test_harness/src/bin.rs +++ b/beacon_node/beacon_chain/test_harness/src/bin.rs @@ -1,69 +1,102 @@ -use clap::{App, Arg}; +use clap::{App, Arg, SubCommand}; use env_logger::{Builder, Env}; -use std::{fs::File, io::prelude::*}; -use test_case::TestCase; -use yaml_rust::YamlLoader; +use gen_keys::gen_keys; +use run_test::run_test; +use std::fs; +use types::test_utils::keypairs_path; +use types::ChainSpec; mod beacon_chain_harness; +mod gen_keys; +mod run_test; mod test_case; mod validator_harness; use validator_harness::ValidatorHarness; fn main() { + let validator_file_path = keypairs_path(); + + let _ = fs::create_dir(validator_file_path.parent().unwrap()); + let matches = App::new("Lighthouse Test Harness Runner") .version("0.0.1") .author("Sigma Prime ") .about("Runs `test_harness` using a YAML test_case.") - .arg( - Arg::with_name("yaml") - .long("yaml") - .value_name("FILE") - .help("YAML file test_case.") - .required(true), - ) .arg( Arg::with_name("log") .long("log-level") + .short("l") .value_name("LOG_LEVEL") .help("Logging level.") .possible_values(&["error", "warn", "info", "debug", "trace"]) .default_value("debug") .required(true), ) + .arg( + Arg::with_name("spec") + .long("spec") + .short("s") + .value_name("SPECIFICATION") + .help("ChainSpec instantiation.") + .possible_values(&["foundation", "few_validators"]) + .default_value("foundation"), + ) + .subcommand( + SubCommand::with_name("run_test") + .about("Executes a YAML test specification") + .arg( + Arg::with_name("yaml") + .long("yaml") + .value_name("FILE") + .help("YAML file test_case.") + .required(true), + ) + .arg( + Arg::with_name("validators_dir") + .long("validators-dir") + .short("v") + .value_name("VALIDATORS_DIR") + .help("A directory with validator deposits and keypair YAML."), + ), + ) + .subcommand( + SubCommand::with_name("gen_keys") + .about("Builds a file of BLS keypairs for faster tests.") + .arg( + Arg::with_name("validator_count") + .long("validator_count") + .short("n") + .value_name("VALIDATOR_COUNT") + .help("Number of validators to generate.") + .required(true), + ) + .arg( + Arg::with_name("output_file") + .long("output_file") + .short("d") + .value_name("GENESIS_TIME") + .help("Output directory for generated YAML.") + .default_value(validator_file_path.to_str().unwrap()), + ), + ) .get_matches(); if let Some(log_level) = matches.value_of("log") { Builder::from_env(Env::default().default_filter_or(log_level)).init(); } - if let Some(yaml_file) = matches.value_of("yaml") { - let docs = { - let mut file = File::open(yaml_file).unwrap(); + let _spec = match matches.value_of("spec") { + Some("foundation") => ChainSpec::foundation(), + Some("few_validators") => ChainSpec::few_validators(), + _ => unreachable!(), // Has a default value, should always exist. + }; - let mut yaml_str = String::new(); - file.read_to_string(&mut yaml_str).unwrap(); + if let Some(matches) = matches.subcommand_matches("run_test") { + run_test(matches); + } - YamlLoader::load_from_str(&yaml_str).unwrap() - }; - - for doc in &docs { - // For each `test_cases` YAML in the document, build a `TestCase`, execute it and - // assert that the execution result matches the test_case description. - // - // In effect, for each `test_case` a new `BeaconChainHarness` is created from genesis - // and a new `BeaconChain` is built as per the test_case. - // - // After the `BeaconChain` has been built out as per the test_case, a dump of all blocks - // and states in the chain is obtained and checked against the `results` specified in - // the `test_case`. - // - // If any of the expectations in the results are not met, the process - // panics with a message. - for test_case in doc["test_cases"].as_vec().unwrap() { - let test_case = TestCase::from_yaml(test_case); - test_case.assert_result_valid(test_case.execute()) - } - } + if let Some(matches) = matches.subcommand_matches("gen_keys") { + gen_keys(matches); } } diff --git a/beacon_node/beacon_chain/test_harness/src/gen_keys.rs b/beacon_node/beacon_chain/test_harness/src/gen_keys.rs new file mode 100644 index 000000000..abd512423 --- /dev/null +++ b/beacon_node/beacon_chain/test_harness/src/gen_keys.rs @@ -0,0 +1,21 @@ +use clap::{value_t, ArgMatches}; +use log::debug; +use std::path::Path; +use types::test_utils::{generate_deterministic_keypairs, KeypairsFile}; + +/// Creates a file containing BLS keypairs. +pub fn gen_keys(matches: &ArgMatches) { + let validator_count = value_t!(matches.value_of("validator_count"), usize) + .expect("Validator count is required argument"); + let output_file = matches + .value_of("output_file") + .expect("Output file has a default value."); + + let keypairs = generate_deterministic_keypairs(validator_count); + + debug!("Writing keypairs to file..."); + + let keypairs_path = Path::new(output_file); + + keypairs.to_raw_file(&keypairs_path, &keypairs).unwrap(); +} diff --git a/beacon_node/beacon_chain/test_harness/src/run_test.rs b/beacon_node/beacon_chain/test_harness/src/run_test.rs new file mode 100644 index 000000000..4caa299d6 --- /dev/null +++ b/beacon_node/beacon_chain/test_harness/src/run_test.rs @@ -0,0 +1,37 @@ +use crate::test_case::TestCase; +use clap::ArgMatches; +use std::{fs::File, io::prelude::*}; +use yaml_rust::YamlLoader; + +/// Runs a YAML-specified test case. +pub fn run_test(matches: &ArgMatches) { + if let Some(yaml_file) = matches.value_of("yaml") { + let docs = { + let mut file = File::open(yaml_file).unwrap(); + + let mut yaml_str = String::new(); + file.read_to_string(&mut yaml_str).unwrap(); + + YamlLoader::load_from_str(&yaml_str).unwrap() + }; + + for doc in &docs { + // For each `test_cases` YAML in the document, build a `TestCase`, execute it and + // assert that the execution result matches the test_case description. + // + // In effect, for each `test_case` a new `BeaconChainHarness` is created from genesis + // and a new `BeaconChain` is built as per the test_case. + // + // After the `BeaconChain` has been built out as per the test_case, a dump of all blocks + // and states in the chain is obtained and checked against the `results` specified in + // the `test_case`. + // + // If any of the expectations in the results are not met, the process + // panics with a message. + for test_case in doc["test_cases"].as_vec().unwrap() { + let test_case = TestCase::from_yaml(test_case); + test_case.assert_result_valid(test_case.execute()) + } + } + } +} diff --git a/beacon_node/beacon_chain/test_harness/src/test_case.rs b/beacon_node/beacon_chain/test_harness/src/test_case.rs index b2709edfc..1361127a1 100644 --- a/beacon_node/beacon_chain/test_harness/src/test_case.rs +++ b/beacon_node/beacon_chain/test_harness/src/test_case.rs @@ -3,14 +3,11 @@ use crate::beacon_chain_harness::BeaconChainHarness; use beacon_chain::CheckPoint; -use bls::{create_proof_of_possession, get_withdrawal_credentials}; use log::{info, warn}; use ssz::SignedRoot; use types::*; -use types::{ - attester_slashing::AttesterSlashingBuilder, proposer_slashing::ProposerSlashingBuilder, -}; +use types::test_utils::*; use yaml_rust::Yaml; mod config; @@ -224,27 +221,20 @@ impl TestCase { } /// Builds a `Deposit` this is valid for the given `BeaconChainHarness` at its next slot. -fn build_transfer(harness: &BeaconChainHarness, from: u64, to: u64, amount: u64) -> Transfer { +fn build_transfer( + harness: &BeaconChainHarness, + sender: u64, + recipient: u64, + amount: u64, +) -> Transfer { let slot = harness.beacon_chain.state.read().slot + 1; - let mut transfer = Transfer { - from, - to, - amount, - fee: 0, - slot, - pubkey: harness.validators[from as usize].keypair.pk.clone(), - signature: Signature::empty_signature(), - }; + let mut builder = TestingTransferBuilder::new(sender, recipient, amount, slot); - let message = transfer.signed_root(); - let epoch = slot.epoch(harness.spec.slots_per_epoch); + let keypair = harness.validator_keypair(sender as usize).unwrap(); + builder.sign(keypair.clone(), &harness.fork(), &harness.spec); - transfer.signature = harness - .validator_sign(from as usize, &message[..], epoch, Domain::Transfer) - .expect("Unable to sign Transfer"); - - transfer + builder.build() } /// Builds a `Deposit` this is valid for the given `BeaconChainHarness`. @@ -257,29 +247,12 @@ fn build_deposit( index_offset: u64, ) -> (Deposit, Keypair) { let keypair = Keypair::random(); - let proof_of_possession = create_proof_of_possession(&keypair); - let index = harness.beacon_chain.state.read().deposit_index + index_offset; - let withdrawal_credentials = Hash256::from_slice( - &get_withdrawal_credentials(&keypair.pk, harness.spec.bls_withdrawal_prefix_byte)[..], - ); - let deposit = Deposit { - // Note: `branch` and `index` will need to be updated once the spec defines their - // validity. - branch: vec![], - index, - deposit_data: DepositData { - amount, - timestamp: 1, - deposit_input: DepositInput { - pubkey: keypair.pk.clone(), - withdrawal_credentials, - proof_of_possession, - }, - }, - }; + let mut builder = TestingDepositBuilder::new(keypair.pk.clone(), amount); + builder.set_index(harness.beacon_chain.state.read().deposit_index + index_offset); + builder.sign(&keypair, harness.epoch(), &harness.fork(), &harness.spec); - (deposit, keypair) + (builder.build(), keypair) } /// Builds a `VoluntaryExit` this is valid for the given `BeaconChainHarness`. @@ -318,7 +291,7 @@ fn build_double_vote_attester_slashing( .expect("Unable to sign AttesterSlashing") }; - AttesterSlashingBuilder::double_vote(validator_indices, signer) + TestingAttesterSlashingBuilder::double_vote(validator_indices, signer) } /// Builds an `ProposerSlashing` for some `validator_index`. @@ -331,5 +304,5 @@ fn build_proposer_slashing(harness: &BeaconChainHarness, validator_index: u64) - .expect("Unable to sign AttesterSlashing") }; - ProposerSlashingBuilder::double_vote(validator_index, signer, &harness.spec) + TestingProposerSlashingBuilder::double_vote(validator_index, signer, &harness.spec) } diff --git a/beacon_node/db/src/stores/beacon_block_store.rs b/beacon_node/db/src/stores/beacon_block_store.rs index 8c9cbcb56..e2e16e60b 100644 --- a/beacon_node/db/src/stores/beacon_block_store.rs +++ b/beacon_node/db/src/stores/beacon_block_store.rs @@ -2,7 +2,7 @@ use super::BLOCKS_DB_COLUMN as DB_COLUMN; use super::{ClientDB, DBError}; use ssz::Decodable; use std::sync::Arc; -use types::{readers::BeaconBlockReader, BeaconBlock, Hash256, Slot}; +use types::{BeaconBlock, Hash256, Slot}; #[derive(Clone, Debug, PartialEq)] pub enum BeaconBlockAtSlotError { @@ -38,23 +38,6 @@ impl BeaconBlockStore { } } - /// Returns an object implementing `BeaconBlockReader`, or `None` (if hash not known). - /// - /// Note: Presently, this function fully deserializes a `BeaconBlock` and returns that. In the - /// future, it would be ideal to return an object capable of reading directly from serialized - /// SSZ bytes. - pub fn get_reader(&self, hash: &Hash256) -> Result, DBError> { - match self.get(&hash)? { - None => Ok(None), - Some(ssz) => { - let (block, _) = BeaconBlock::ssz_decode(&ssz, 0).map_err(|_| DBError { - message: "Bad BeaconBlock SSZ.".to_string(), - })?; - Ok(Some(block)) - } - } - } - /// Retrieve the block at a slot given a "head_hash" and a slot. /// /// A "head_hash" must be a block hash with a slot number greater than or equal to the desired @@ -72,17 +55,17 @@ impl BeaconBlockStore { &self, head_hash: &Hash256, slot: Slot, - ) -> Result, BeaconBlockAtSlotError> { + ) -> Result, BeaconBlockAtSlotError> { let mut current_hash = *head_hash; loop { - if let Some(block_reader) = self.get_reader(¤t_hash)? { - if block_reader.slot() == slot { - break Ok(Some((current_hash, block_reader))); - } else if block_reader.slot() < slot { + if let Some(block) = self.get_deserialized(¤t_hash)? { + if block.slot == slot { + break Ok(Some((current_hash, block))); + } else if block.slot < slot { break Ok(None); } else { - current_hash = block_reader.parent_root(); + current_hash = block.previous_block_root; } } else { break Err(BeaconBlockAtSlotError::UnknownBeaconBlock(current_hash)); @@ -198,6 +181,7 @@ mod tests { } #[test] + #[ignore] fn test_block_at_slot() { let db = Arc::new(MemoryDB::open()); let bs = Arc::new(BeaconBlockStore::new(db.clone())); @@ -227,7 +211,7 @@ mod tests { for i in 0..block_count { let mut block = BeaconBlock::random_for_test(&mut rng); - block.parent_root = parent_hashes[i]; + block.previous_block_root = parent_hashes[i]; block.slot = slots[i]; let ssz = ssz_encode(&block); @@ -239,12 +223,12 @@ mod tests { // Test that certain slots can be reached from certain hashes. let test_cases = vec![(4, 4), (4, 3), (4, 2), (4, 1), (4, 0)]; for (hashes_index, slot_index) in test_cases { - let (matched_block_hash, reader) = bs + let (matched_block_hash, block) = bs .block_at_slot(&hashes[hashes_index], slots[slot_index]) .unwrap() .unwrap(); assert_eq!(matched_block_hash, hashes[slot_index]); - assert_eq!(reader.slot(), slots[slot_index]); + assert_eq!(block.slot, slots[slot_index]); } let ssz = bs.block_at_slot(&hashes[4], Slot::new(2)).unwrap(); diff --git a/beacon_node/db/src/stores/beacon_state_store.rs b/beacon_node/db/src/stores/beacon_state_store.rs index ed22696cb..fd6ff569a 100644 --- a/beacon_node/db/src/stores/beacon_state_store.rs +++ b/beacon_node/db/src/stores/beacon_state_store.rs @@ -2,7 +2,7 @@ use super::STATES_DB_COLUMN as DB_COLUMN; use super::{ClientDB, DBError}; use ssz::Decodable; use std::sync::Arc; -use types::{readers::BeaconStateReader, BeaconState, Hash256}; +use types::{BeaconState, Hash256}; pub struct BeaconStateStore where @@ -30,23 +30,6 @@ impl BeaconStateStore { } } } - - /// Retuns an object implementing `BeaconStateReader`, or `None` (if hash not known). - /// - /// Note: Presently, this function fully deserializes a `BeaconState` and returns that. In the - /// future, it would be ideal to return an object capable of reading directly from serialized - /// SSZ bytes. - pub fn get_reader(&self, hash: &Hash256) -> Result, DBError> { - match self.get(&hash)? { - None => Ok(None), - Some(ssz) => { - let (state, _) = BeaconState::ssz_decode(&ssz, 0).map_err(|_| DBError { - message: "Bad State SSZ.".to_string(), - })?; - Ok(Some(state)) - } - } - } } #[cfg(test)] @@ -72,8 +55,7 @@ mod tests { store.put(&state_root, &ssz_encode(&state)).unwrap(); - let reader = store.get_reader(&state_root).unwrap().unwrap(); - let decoded = reader.into_beacon_state().unwrap(); + let decoded = store.get_deserialized(&state_root).unwrap().unwrap(); assert_eq!(state, decoded); } diff --git a/eth2/block_proposer/src/lib.rs b/eth2/block_proposer/src/lib.rs index 5cddbaedc..e62c4b71d 100644 --- a/eth2/block_proposer/src/lib.rs +++ b/eth2/block_proposer/src/lib.rs @@ -4,7 +4,7 @@ mod traits; use slot_clock::SlotClock; use ssz::{SignedRoot, TreeHash}; use std::sync::Arc; -use types::{BeaconBlock, ChainSpec, Domain, Hash256, Proposal, Slot}; +use types::{BeaconBlock, ChainSpec, Domain, Slot}; pub use self::traits::{ BeaconNode, BeaconNodeError, DutiesReader, DutiesReaderError, PublishOutcome, Signer, @@ -158,7 +158,7 @@ impl BlockProducer BlockProducer Option { self.store_produce(&block); - let proposal = Proposal { - slot: block.slot, - shard: self.spec.beacon_chain_shard_number, - block_root: Hash256::from_slice(&block.signed_root()[..]), - signature: block.signature.clone(), - }; - match self .signer - .sign_block_proposal(&proposal.signed_root()[..], domain) + .sign_block_proposal(&block.signed_root()[..], domain) { None => None, Some(signature) => { diff --git a/eth2/block_proposer/src/test_utils/epoch_map.rs b/eth2/block_proposer/src/test_utils/epoch_map.rs index 6658c7526..c06c376c6 100644 --- a/eth2/block_proposer/src/test_utils/epoch_map.rs +++ b/eth2/block_proposer/src/test_utils/epoch_map.rs @@ -28,8 +28,8 @@ impl DutiesReader for EpochMap { fn fork(&self) -> Result { Ok(Fork { - previous_version: 0, - current_version: 0, + previous_version: [0; 4], + current_version: [0; 4], epoch: Epoch::new(0), }) } diff --git a/eth2/fork_choice/src/bitwise_lmd_ghost.rs b/eth2/fork_choice/src/bitwise_lmd_ghost.rs index fd1c3dea4..9410fd203 100644 --- a/eth2/fork_choice/src/bitwise_lmd_ghost.rs +++ b/eth2/fork_choice/src/bitwise_lmd_ghost.rs @@ -11,8 +11,8 @@ use log::{debug, trace}; use std::collections::HashMap; use std::sync::Arc; use types::{ - readers::BeaconBlockReader, validator_registry::get_active_validator_indices, BeaconBlock, - ChainSpec, Hash256, Slot, SlotHeight, + validator_registry::get_active_validator_indices, BeaconBlock, ChainSpec, Hash256, Slot, + SlotHeight, }; //TODO: Pruning - Children @@ -255,17 +255,17 @@ impl ForkChoice for BitwiseLMDGhost { // get the height of the parent let parent_height = self .block_store - .get_deserialized(&block.parent_root)? - .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(block.parent_root))? - .slot() + .get_deserialized(&block.previous_block_root)? + .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(block.previous_block_root))? + .slot .height(spec.genesis_slot); - let parent_hash = &block.parent_root; + let parent_hash = &block.previous_block_root; // add the new block to the children of parent (*self .children - .entry(block.parent_root) + .entry(block.previous_block_root) .or_insert_with(|| vec![])) .push(block_hash.clone()); @@ -309,7 +309,7 @@ impl ForkChoice for BitwiseLMDGhost { .block_store .get_deserialized(&target_block_root)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*target_block_root))? - .slot() + .slot .height(spec.genesis_slot); // get the height of the past target block @@ -317,7 +317,7 @@ impl ForkChoice for BitwiseLMDGhost { .block_store .get_deserialized(&attestation_target)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*attestation_target))? - .slot() + .slot .height(spec.genesis_slot); // update the attestation only if the new target is higher if past_block_height < block_height { @@ -343,8 +343,8 @@ impl ForkChoice for BitwiseLMDGhost { .get_deserialized(&justified_block_start)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*justified_block_start))?; - let block_slot = block.slot(); - let state_root = block.state_root(); + let block_slot = block.slot; + let state_root = block.state_root; let mut block_height = block_slot.height(spec.genesis_slot); let mut current_head = *justified_block_start; @@ -409,11 +409,23 @@ impl ForkChoice for BitwiseLMDGhost { *child_votes.entry(child).or_insert_with(|| 0) += vote; } } - // given the votes on the children, find the best child - current_head = self - .choose_best_child(&child_votes) - .ok_or(ForkChoiceError::CannotFindBestChild)?; - trace!("Best child found: {}", current_head); + // check if we have votes of children, if not select the smallest hash child + if child_votes.is_empty() { + current_head = *children + .iter() + .min_by(|child1, child2| child1.cmp(child2)) + .expect("Must be children here"); + trace!( + "Children have no votes - smallest hash chosen: {}", + current_head + ); + } else { + // given the votes on the children, find the best child + current_head = self + .choose_best_child(&child_votes) + .ok_or(ForkChoiceError::CannotFindBestChild)?; + trace!("Best child found: {}", current_head); + } } // didn't find head yet, proceed to next iteration @@ -422,7 +434,7 @@ impl ForkChoice for BitwiseLMDGhost { .block_store .get_deserialized(¤t_head)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(current_head))? - .slot() + .slot .height(spec.genesis_slot); // prune the latest votes for votes that are not part of current chosen chain // more specifically, only keep votes that have head as an ancestor diff --git a/eth2/fork_choice/src/lib.rs b/eth2/fork_choice/src/lib.rs index 318ac99a8..016cd5dea 100644 --- a/eth2/fork_choice/src/lib.rs +++ b/eth2/fork_choice/src/lib.rs @@ -22,6 +22,7 @@ extern crate types; pub mod bitwise_lmd_ghost; pub mod longest_chain; +pub mod optimized_lmd_ghost; pub mod slow_lmd_ghost; use db::stores::BeaconBlockAtSlotError; @@ -30,6 +31,7 @@ use types::{BeaconBlock, ChainSpec, Hash256}; pub use bitwise_lmd_ghost::BitwiseLMDGhost; pub use longest_chain::LongestChain; +pub use optimized_lmd_ghost::OptimizedLMDGhost; pub use slow_lmd_ghost::SlowLMDGhost; /// Defines the interface for Fork Choices. Each Fork choice will define their own data structures @@ -102,4 +104,6 @@ pub enum ForkChoiceAlgorithm { SlowLMDGhost, /// An optimised version of bitwise LMD-GHOST by Vitalik. BitwiseLMDGhost, + /// An optimised implementation of LMD ghost. + OptimizedLMDGhost, } diff --git a/eth2/fork_choice/src/longest_chain.rs b/eth2/fork_choice/src/longest_chain.rs index 333553c02..423edc567 100644 --- a/eth2/fork_choice/src/longest_chain.rs +++ b/eth2/fork_choice/src/longest_chain.rs @@ -34,7 +34,7 @@ impl ForkChoice for LongestChain { ) -> Result<(), ForkChoiceError> { // add the block hash to head_block_hashes removing the parent if it exists self.head_block_hashes - .retain(|hash| *hash != block.parent_root); + .retain(|hash| *hash != block.previous_block_root); self.head_block_hashes.push(*block_hash); Ok(()) } diff --git a/eth2/fork_choice/src/optimized_lmd_ghost.rs b/eth2/fork_choice/src/optimized_lmd_ghost.rs new file mode 100644 index 000000000..e1b8914a6 --- /dev/null +++ b/eth2/fork_choice/src/optimized_lmd_ghost.rs @@ -0,0 +1,465 @@ +//! The optimised bitwise LMD-GHOST fork choice rule. +extern crate bit_vec; + +use crate::{ForkChoice, ForkChoiceError}; +use db::{ + stores::{BeaconBlockStore, BeaconStateStore}, + ClientDB, +}; +use log::{debug, trace}; +use std::cmp::Ordering; +use std::collections::HashMap; +use std::sync::Arc; +use types::{ + validator_registry::get_active_validator_indices, BeaconBlock, ChainSpec, Hash256, Slot, + SlotHeight, +}; + +//TODO: Pruning - Children +//TODO: Handle Syncing + +// NOTE: This uses u32 to represent difference between block heights. Thus this is only +// applicable for block height differences in the range of a u32. +// This can potentially be parallelized in some parts. + +/// Compute the base-2 logarithm of an integer, floored (rounded down) +#[inline] +fn log2_int(x: u64) -> u32 { + if x == 0 { + return 0; + } + 63 - x.leading_zeros() +} + +fn power_of_2_below(x: u64) -> u64 { + 2u64.pow(log2_int(x)) +} + +/// Stores the necessary data structures to run the optimised lmd ghost algorithm. +pub struct OptimizedLMDGhost { + /// A cache of known ancestors at given heights for a specific block. + //TODO: Consider FnvHashMap + cache: HashMap, Hash256>, + /// Log lookup table for blocks to their ancestors. + //TODO: Verify we only want/need a size 16 log lookup + ancestors: Vec>, + /// Stores the children for any given parent. + children: HashMap>, + /// The latest attestation targets as a map of validator index to block hash. + //TODO: Could this be a fixed size vec + latest_attestation_targets: HashMap, + /// Block storage access. + block_store: Arc>, + /// State storage access. + state_store: Arc>, + max_known_height: SlotHeight, +} + +impl OptimizedLMDGhost +where + T: ClientDB + Sized, +{ + pub fn new( + block_store: Arc>, + state_store: Arc>, + ) -> Self { + OptimizedLMDGhost { + cache: HashMap::new(), + ancestors: vec![HashMap::new(); 16], + latest_attestation_targets: HashMap::new(), + children: HashMap::new(), + max_known_height: SlotHeight::new(0), + block_store, + state_store, + } + } + + /// Finds the latest votes weighted by validator balance. Returns a hashmap of block_hash to + /// weighted votes. + pub fn get_latest_votes( + &self, + state_root: &Hash256, + block_slot: Slot, + spec: &ChainSpec, + ) -> Result, ForkChoiceError> { + // get latest votes + // Note: Votes are weighted by min(balance, MAX_DEPOSIT_AMOUNT) // + // FORK_CHOICE_BALANCE_INCREMENT + // build a hashmap of block_hash to weighted votes + let mut latest_votes: HashMap = HashMap::new(); + // gets the current weighted votes + let current_state = self + .state_store + .get_deserialized(&state_root)? + .ok_or_else(|| ForkChoiceError::MissingBeaconState(*state_root))?; + + let active_validator_indices = get_active_validator_indices( + ¤t_state.validator_registry[..], + block_slot.epoch(spec.slots_per_epoch), + ); + + for index in active_validator_indices { + let balance = std::cmp::min( + current_state.validator_balances[index], + spec.max_deposit_amount, + ) / spec.fork_choice_balance_increment; + if balance > 0 { + if let Some(target) = self.latest_attestation_targets.get(&(index as u64)) { + *latest_votes.entry(*target).or_insert_with(|| 0) += balance; + } + } + } + trace!("Latest votes: {:?}", latest_votes); + Ok(latest_votes) + } + + /// Gets the ancestor at a given height `at_height` of a block specified by `block_hash`. + fn get_ancestor( + &mut self, + block_hash: Hash256, + target_height: SlotHeight, + spec: &ChainSpec, + ) -> Option { + // return None if we can't get the block from the db. + let block_height = { + let block_slot = self + .block_store + .get_deserialized(&block_hash) + .ok()? + .expect("Should have returned already if None") + .slot; + + block_slot.height(spec.genesis_slot) + }; + + // verify we haven't exceeded the block height + if target_height >= block_height { + if target_height > block_height { + return None; + } else { + return Some(block_hash); + } + } + // check if the result is stored in our cache + let cache_key = CacheKey::new(&block_hash, target_height.as_u64()); + if let Some(ancestor) = self.cache.get(&cache_key) { + return Some(*ancestor); + } + + // not in the cache recursively search for ancestors using a log-lookup + if let Some(ancestor) = { + let ancestor_lookup = self.ancestors + [log2_int((block_height - target_height - 1u64).as_u64()) as usize] + .get(&block_hash) + //TODO: Panic if we can't lookup and fork choice fails + .expect("All blocks should be added to the ancestor log lookup table"); + self.get_ancestor(*ancestor_lookup, target_height, &spec) + } { + // add the result to the cache + self.cache.insert(cache_key, ancestor); + return Some(ancestor); + } + + None + } + + // looks for an obvious block winner given the latest votes for a specific height + fn get_clear_winner( + &mut self, + latest_votes: &HashMap, + block_height: SlotHeight, + spec: &ChainSpec, + ) -> Option { + // map of vote counts for every hash at this height + let mut current_votes: HashMap = HashMap::new(); + let mut total_vote_count = 0; + + trace!("Clear winner at block height: {}", block_height); + // loop through the latest votes and count all votes + // these have already been weighted by balance + for (hash, votes) in latest_votes.iter() { + if let Some(ancestor) = self.get_ancestor(*hash, block_height, spec) { + let current_vote_value = current_votes.get(&ancestor).unwrap_or_else(|| &0); + current_votes.insert(ancestor, current_vote_value + *votes); + total_vote_count += votes; + } + } + // Check if there is a clear block winner at this height. If so return it. + for (hash, votes) in current_votes.iter() { + if *votes > total_vote_count / 2 { + // we have a clear winner, return it + return Some(*hash); + } + } + // didn't find a clear winner + None + } + + // Finds the best child (one with highest votes) + fn choose_best_child(&self, votes: &HashMap) -> Option { + if votes.is_empty() { + return None; + } + + // Iterate through hashmap to get child with maximum votes + let best_child = votes.iter().max_by(|(child1, v1), (child2, v2)| { + let mut result = v1.cmp(v2); + // If votes are equal, choose smaller hash to break ties deterministically + if result == Ordering::Equal { + // Reverse so that max_by chooses smaller hash + result = child1.cmp(child2).reverse(); + } + result + }); + + Some(*best_child.unwrap().0) + } +} + +impl ForkChoice for OptimizedLMDGhost { + fn add_block( + &mut self, + block: &BeaconBlock, + block_hash: &Hash256, + spec: &ChainSpec, + ) -> Result<(), ForkChoiceError> { + // get the height of the parent + let parent_height = self + .block_store + .get_deserialized(&block.previous_block_root)? + .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(block.previous_block_root))? + .slot + .height(spec.genesis_slot); + + let parent_hash = &block.previous_block_root; + + // add the new block to the children of parent + (*self + .children + .entry(block.previous_block_root) + .or_insert_with(|| vec![])) + .push(block_hash.clone()); + + // build the ancestor data structure + for index in 0..16 { + if parent_height % (1 << index) == 0 { + self.ancestors[index].insert(*block_hash, *parent_hash); + } else { + // TODO: This is unsafe. Will panic if parent_hash doesn't exist. Using it for debugging + let parent_ancestor = self.ancestors[index][parent_hash]; + self.ancestors[index].insert(*block_hash, parent_ancestor); + } + } + // update the max height + self.max_known_height = std::cmp::max(self.max_known_height, parent_height + 1); + Ok(()) + } + + fn add_attestation( + &mut self, + validator_index: u64, + target_block_root: &Hash256, + spec: &ChainSpec, + ) -> Result<(), ForkChoiceError> { + // simply add the attestation to the latest_attestation_target if the block_height is + // larger + trace!( + "Adding attestation of validator: {:?} for block: {}", + validator_index, + target_block_root + ); + let attestation_target = self + .latest_attestation_targets + .entry(validator_index) + .or_insert_with(|| *target_block_root); + // if we already have a value + if attestation_target != target_block_root { + trace!("Old attestation found: {:?}", attestation_target); + // get the height of the target block + let block_height = self + .block_store + .get_deserialized(&target_block_root)? + .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*target_block_root))? + .slot + .height(spec.genesis_slot); + + // get the height of the past target block + let past_block_height = self + .block_store + .get_deserialized(&attestation_target)? + .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*attestation_target))? + .slot + .height(spec.genesis_slot); + // update the attestation only if the new target is higher + if past_block_height < block_height { + trace!("Updating old attestation"); + *attestation_target = *target_block_root; + } + } + Ok(()) + } + + /// Perform lmd_ghost on the current chain to find the head. + fn find_head( + &mut self, + justified_block_start: &Hash256, + spec: &ChainSpec, + ) -> Result { + debug!( + "Starting optimised fork choice at block: {}", + justified_block_start + ); + let block = self + .block_store + .get_deserialized(&justified_block_start)? + .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*justified_block_start))?; + + let block_slot = block.slot; + let state_root = block.state_root; + let mut block_height = block_slot.height(spec.genesis_slot); + + let mut current_head = *justified_block_start; + + let mut latest_votes = self.get_latest_votes(&state_root, block_slot, spec)?; + + // remove any votes that don't relate to our current head. + latest_votes + .retain(|hash, _| self.get_ancestor(*hash, block_height, spec) == Some(current_head)); + + // begin searching for the head + loop { + debug!( + "Iteration for block: {} with vote length: {}", + current_head, + latest_votes.len() + ); + // if there are no children, we are done, return the current_head + let children = match self.children.get(¤t_head) { + Some(children) => children.clone(), + None => { + debug!("Head found: {}", current_head); + return Ok(current_head); + } + }; + + // logarithmic lookup blocks to see if there are obvious winners, if so, + // progress to the next iteration. + let mut step = + power_of_2_below(self.max_known_height.saturating_sub(block_height).as_u64()) / 2; + while step > 0 { + trace!("Current Step: {}", step); + if let Some(clear_winner) = self.get_clear_winner( + &latest_votes, + block_height - (block_height % step) + step, + spec, + ) { + current_head = clear_winner; + break; + } + step /= 2; + } + if step > 0 { + trace!("Found clear winner: {}", current_head); + } + // if our skip lookup failed and we only have one child, progress to that child + else if children.len() == 1 { + current_head = children[0]; + trace!( + "Lookup failed, only one child, proceeding to child: {}", + current_head + ); + } + // we need to find the best child path to progress down. + else { + trace!("Searching for best child"); + let mut child_votes = HashMap::new(); + for (voted_hash, vote) in latest_votes.iter() { + // if the latest votes correspond to a child + if let Some(child) = self.get_ancestor(*voted_hash, block_height + 1, spec) { + // add up the votes for each child + *child_votes.entry(child).or_insert_with(|| 0) += vote; + } + } + // check if we have votes of children, if not select the smallest hash child + if child_votes.is_empty() { + current_head = *children + .iter() + .min_by(|child1, child2| child1.cmp(child2)) + .expect("Must be children here"); + trace!( + "Children have no votes - smallest hash chosen: {}", + current_head + ); + } else { + // given the votes on the children, find the best child + current_head = self + .choose_best_child(&child_votes) + .ok_or(ForkChoiceError::CannotFindBestChild)?; + trace!("Best child found: {}", current_head); + } + } + + // didn't find head yet, proceed to next iteration + // update block height + block_height = self + .block_store + .get_deserialized(¤t_head)? + .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(current_head))? + .slot + .height(spec.genesis_slot); + // prune the latest votes for votes that are not part of current chosen chain + // more specifically, only keep votes that have head as an ancestor + for hash in latest_votes.keys() { + trace!( + "Ancestor for vote: {} at height: {} is: {:?}", + hash, + block_height, + self.get_ancestor(*hash, block_height, spec) + ); + } + latest_votes.retain(|hash, _| { + self.get_ancestor(*hash, block_height, spec) == Some(current_head) + }); + } + } +} + +/// Type for storing blocks in a memory cache. Key is comprised of block-hash plus the height. +#[derive(PartialEq, Eq, Hash)] +pub struct CacheKey { + block_hash: Hash256, + block_height: T, +} + +impl CacheKey { + pub fn new(block_hash: &Hash256, block_height: T) -> Self { + CacheKey { + block_hash: *block_hash, + block_height, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + pub fn test_power_of_2_below() { + assert_eq!(power_of_2_below(4), 4); + assert_eq!(power_of_2_below(5), 4); + assert_eq!(power_of_2_below(7), 4); + assert_eq!(power_of_2_below(24), 16); + assert_eq!(power_of_2_below(32), 32); + assert_eq!(power_of_2_below(33), 32); + assert_eq!(power_of_2_below(63), 32); + } + + #[test] + pub fn test_power_of_2_below_large() { + let pow: u64 = 1 << 24; + for x in (pow - 20)..(pow + 20) { + assert!(power_of_2_below(x) <= x, "{}", x); + } + } +} diff --git a/eth2/fork_choice/src/protolambda_lmd_ghost.rs b/eth2/fork_choice/src/protolambda_lmd_ghost.rs deleted file mode 100644 index 8b1378917..000000000 --- a/eth2/fork_choice/src/protolambda_lmd_ghost.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/eth2/fork_choice/src/slow_lmd_ghost.rs b/eth2/fork_choice/src/slow_lmd_ghost.rs index ab4cd2ada..af58aa7b8 100644 --- a/eth2/fork_choice/src/slow_lmd_ghost.rs +++ b/eth2/fork_choice/src/slow_lmd_ghost.rs @@ -9,8 +9,7 @@ use log::{debug, trace}; use std::collections::HashMap; use std::sync::Arc; use types::{ - readers::BeaconBlockReader, validator_registry::get_active_validator_indices, BeaconBlock, - ChainSpec, Hash256, Slot, + validator_registry::get_active_validator_indices, BeaconBlock, ChainSpec, Hash256, Slot, }; //TODO: Pruning and syncing @@ -95,7 +94,7 @@ where .block_store .get_deserialized(&block_root)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*block_root))? - .slot(); + .slot; for (vote_hash, votes) in latest_votes.iter() { let (root_at_slot, _) = self @@ -122,7 +121,7 @@ impl ForkChoice for SlowLMDGhost { // add the new block to the children of parent (*self .children - .entry(block.parent_root) + .entry(block.previous_block_root) .or_insert_with(|| vec![])) .push(block_hash.clone()); @@ -155,7 +154,7 @@ impl ForkChoice for SlowLMDGhost { .block_store .get_deserialized(&target_block_root)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*target_block_root))? - .slot() + .slot .height(spec.genesis_slot); // get the height of the past target block @@ -163,7 +162,7 @@ impl ForkChoice for SlowLMDGhost { .block_store .get_deserialized(&attestation_target)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*attestation_target))? - .slot() + .slot .height(spec.genesis_slot); // update the attestation only if the new target is higher if past_block_height < block_height { @@ -186,9 +185,9 @@ impl ForkChoice for SlowLMDGhost { .get_deserialized(&justified_block_start)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*justified_block_start))?; - let start_state_root = start.state_root(); + let start_state_root = start.state_root; - let latest_votes = self.get_latest_votes(&start_state_root, start.slot(), spec)?; + let latest_votes = self.get_latest_votes(&start_state_root, start.slot, spec)?; let mut head_hash = *justified_block_start; @@ -210,6 +209,7 @@ impl ForkChoice for SlowLMDGhost { trace!("Children found: {:?}", children); let mut head_vote_count = 0; + head_hash = children[0]; for child_hash in children { let vote_count = self.get_vote_count(&latest_votes, &child_hash)?; trace!("Vote count for child: {} is: {}", child_hash, vote_count); @@ -218,6 +218,12 @@ impl ForkChoice for SlowLMDGhost { head_hash = *child_hash; head_vote_count = vote_count; } + // resolve ties - choose smaller hash + else if vote_count == head_vote_count { + if *child_hash < head_hash { + head_hash = *child_hash; + } + } } } Ok(head_hash) diff --git a/eth2/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml b/eth2/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml index 3233137ab..61b0b05c4 100644 --- a/eth2/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml +++ b/eth2/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml @@ -63,3 +63,82 @@ test_cases: - b7: 2 heads: - id: 'b4' +- blocks: + - id: 'b0' + parent: 'b0' + - id: 'b1' + parent: 'b0' + - id: 'b2' + parent: 'b0' + - id: 'b3' + parent: 'b0' + - id: 'b4' + parent: 'b1' + - id: 'b5' + parent: 'b1' + - id: 'b6' + parent: 'b2' + - id: 'b7' + parent: 'b2' + - id: 'b8' + parent: 'b3' + - id: 'b9' + parent: 'b3' + weights: + - b1: 2 + - b2: 1 + - b3: 1 + - b4: 7 + - b5: 5 + - b6: 2 + - b7: 4 + - b8: 4 + - b9: 2 + heads: + - id: 'b4' +- blocks: + - id: 'b0' + parent: 'b0' + - id: 'b1' + parent: 'b0' + - id: 'b2' + parent: 'b0' + - id: 'b3' + parent: 'b0' + - id: 'b4' + parent: 'b1' + - id: 'b5' + parent: 'b1' + - id: 'b6' + parent: 'b2' + - id: 'b7' + parent: 'b2' + - id: 'b8' + parent: 'b3' + - id: 'b9' + parent: 'b3' + weights: + - b1: 1 + - b2: 1 + - b3: 1 + - b4: 7 + - b5: 5 + - b6: 2 + - b7: 4 + - b8: 4 + - b9: 2 + heads: + - id: 'b7' +- blocks: + - id: 'b0' + parent: 'b0' + - id: 'b1' + parent: 'b0' + - id: 'b2' + parent: 'b0' + weights: + - b1: 0 + - b2: 0 + heads: + - id: 'b1' + diff --git a/eth2/fork_choice/tests/lmd_ghost_test_vectors.yaml b/eth2/fork_choice/tests/lmd_ghost_test_vectors.yaml index 4676d8201..e7847de11 100644 --- a/eth2/fork_choice/tests/lmd_ghost_test_vectors.yaml +++ b/eth2/fork_choice/tests/lmd_ghost_test_vectors.yaml @@ -35,3 +35,31 @@ test_cases: - b3: 3 heads: - id: 'b1' +# equal weights children. Should choose lower hash b2 +- blocks: + - id: 'b0' + parent: 'b0' + - id: 'b1' + parent: 'b0' + - id: 'b2' + parent: 'b0' + - id: 'b3' + parent: 'b0' + weights: + - b1: 5 + - b2: 6 + - b3: 6 + heads: + - id: 'b2' +- blocks: + - id: 'b0' + parent: 'b0' + - id: 'b1' + parent: 'b0' + - id: 'b2' + parent: 'b0' + weights: + - b1: 0 + - b2: 0 + heads: + - id: 'b1' diff --git a/eth2/fork_choice/tests/tests.rs b/eth2/fork_choice/tests/tests.rs index a3cab6a7c..80fbbbe20 100644 --- a/eth2/fork_choice/tests/tests.rs +++ b/eth2/fork_choice/tests/tests.rs @@ -3,7 +3,7 @@ extern crate beacon_chain; extern crate bls; extern crate db; -//extern crate env_logger; // for debugging +// extern crate env_logger; // for debugging extern crate fork_choice; extern crate hex; extern crate log; @@ -12,22 +12,35 @@ extern crate types; extern crate yaml_rust; pub use beacon_chain::BeaconChain; -use bls::{PublicKey, Signature}; +use bls::Signature; use db::stores::{BeaconBlockStore, BeaconStateStore}; use db::MemoryDB; -//use env_logger::{Builder, Env}; -use fork_choice::{BitwiseLMDGhost, ForkChoice, ForkChoiceAlgorithm, LongestChain, SlowLMDGhost}; +// use env_logger::{Builder, Env}; +use fork_choice::{ + BitwiseLMDGhost, ForkChoice, ForkChoiceAlgorithm, LongestChain, OptimizedLMDGhost, SlowLMDGhost, +}; use ssz::ssz_encode; use std::collections::HashMap; use std::sync::Arc; use std::{fs::File, io::prelude::*, path::PathBuf}; -use types::{ - BeaconBlock, BeaconBlockBody, BeaconState, ChainSpec, Epoch, Eth1Data, Hash256, Slot, Validator, -}; +use types::test_utils::TestingBeaconStateBuilder; +use types::{BeaconBlock, BeaconBlockBody, ChainSpec, Eth1Data, Hash256, Keypair, Slot}; use yaml_rust::yaml; // Note: We Assume the block Id's are hex-encoded. +#[test] +fn test_optimized_lmd_ghost() { + // set up logging + // Builder::from_env(Env::default().default_filter_or("trace")).init(); + + test_yaml_vectors( + ForkChoiceAlgorithm::OptimizedLMDGhost, + "tests/lmd_ghost_test_vectors.yaml", + 100, + ); +} + #[test] fn test_bitwise_lmd_ghost() { // set up logging @@ -77,6 +90,8 @@ fn test_yaml_vectors( let randao_reveal = Signature::empty_signature(); let signature = Signature::empty_signature(); let body = BeaconBlockBody { + eth1_data, + randao_reveal, proposer_slashings: vec![], attester_slashings: vec![], attestations: vec![], @@ -104,14 +119,14 @@ fn test_yaml_vectors( // default params for genesis let block_hash = id_to_hash(&block_id); let mut slot = spec.genesis_slot; - let parent_root = id_to_hash(&parent_id); + let previous_block_root = id_to_hash(&parent_id); // set the slot and parent based off the YAML. Start with genesis; // if not the genesis, update slot if parent_id != block_id { // find parent slot slot = *(block_slot - .get(&parent_root) + .get(&previous_block_root) .expect("Parent should have a slot number")) + 1; } else { @@ -124,10 +139,8 @@ fn test_yaml_vectors( // build the BeaconBlock let beacon_block = BeaconBlock { slot, - parent_root, + previous_block_root, state_root: state_root.clone(), - randao_reveal: randao_reveal.clone(), - eth1_data: eth1_data.clone(), signature: signature.clone(), body: body.clone(), }; @@ -205,16 +218,18 @@ fn load_test_cases_from_yaml(file_path: &str) -> Vec { // initialise a single validator and state. All blocks will reference this state root. fn setup_inital_state( fork_choice_algo: &ForkChoiceAlgorithm, - no_validators: usize, + num_validators: usize, ) -> (Box, Arc>, Hash256) { - let zero_hash = Hash256::zero(); - let db = Arc::new(MemoryDB::open()); let block_store = Arc::new(BeaconBlockStore::new(db.clone())); let state_store = Arc::new(BeaconStateStore::new(db.clone())); // the fork choice instantiation let fork_choice: Box = match fork_choice_algo { + ForkChoiceAlgorithm::OptimizedLMDGhost => Box::new(OptimizedLMDGhost::new( + block_store.clone(), + state_store.clone(), + )), ForkChoiceAlgorithm::BitwiseLMDGhost => Box::new(BitwiseLMDGhost::new( block_store.clone(), state_store.clone(), @@ -225,40 +240,11 @@ fn setup_inital_state( ForkChoiceAlgorithm::LongestChain => Box::new(LongestChain::new(block_store.clone())), }; - // misc vars for setting up the state - let genesis_time = 1_550_381_159; - - let latest_eth1_data = Eth1Data { - deposit_root: zero_hash.clone(), - block_hash: zero_hash.clone(), - }; - - let initial_validator_deposits = vec![]; let spec = ChainSpec::foundation(); - // create the state - let mut state = BeaconState::genesis( - genesis_time, - initial_validator_deposits, - latest_eth1_data, - &spec, - ) - .unwrap(); - - let default_validator = Validator { - pubkey: PublicKey::default(), - withdrawal_credentials: zero_hash, - activation_epoch: Epoch::from(0u64), - exit_epoch: spec.far_future_epoch, - withdrawable_epoch: spec.far_future_epoch, - initiated_exit: false, - slashed: false, - }; - // activate the validators - for _ in 0..no_validators { - state.validator_registry.push(default_validator.clone()); - state.validator_balances.push(32_000_000_000); - } + let state_builder = + TestingBeaconStateBuilder::from_single_keypair(num_validators, &Keypair::random(), &spec); + let (state, _keypairs) = state_builder.build(); let state_root = state.canonical_root(); state_store diff --git a/eth2/state_processing/Cargo.toml b/eth2/state_processing/Cargo.toml index c51ce8372..4e37fce0c 100644 --- a/eth2/state_processing/Cargo.toml +++ b/eth2/state_processing/Cargo.toml @@ -11,9 +11,13 @@ harness = false [dev-dependencies] criterion = "0.2" env_logger = "0.6.0" +serde = "1.0" +serde_derive = "1.0" +serde_yaml = "0.8" [dependencies] bls = { path = "../utils/bls" } +fnv = "1.0" hashing = { path = "../utils/hashing" } int_to_bytes = { path = "../utils/int_to_bytes" } integer-sqrt = "0.1" diff --git a/eth2/state_processing/benches/bench_block_processing.rs b/eth2/state_processing/benches/bench_block_processing.rs new file mode 100644 index 000000000..2ee08c96a --- /dev/null +++ b/eth2/state_processing/benches/bench_block_processing.rs @@ -0,0 +1,270 @@ +use criterion::Criterion; +use criterion::{black_box, Benchmark}; +use ssz::TreeHash; +use state_processing::{ + per_block_processing, + per_block_processing::{ + process_attestations, process_attester_slashings, process_deposits, process_eth1_data, + process_exits, process_proposer_slashings, process_randao, process_transfers, + verify_block_signature, + }, +}; +use types::*; + +/// Run the detailed benchmarking suite on the given `BeaconState`. +/// +/// `desc` will be added to the title of each bench. +pub fn bench_block_processing( + c: &mut Criterion, + initial_block: &BeaconBlock, + initial_state: &BeaconState, + initial_spec: &ChainSpec, + desc: &str, +) { + let state = initial_state.clone(); + let block = initial_block.clone(); + let spec = initial_spec.clone(); + c.bench( + &format!("{}/block_processing", desc), + Benchmark::new("verify_block_signature", move |b| { + b.iter_batched( + || state.clone(), + |mut state| { + verify_block_signature(&mut state, &block, &spec).unwrap(); + state + }, + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(10), + ); + + let state = initial_state.clone(); + let block = initial_block.clone(); + let spec = initial_spec.clone(); + c.bench( + &format!("{}/block_processing", desc), + Benchmark::new("process_randao", move |b| { + b.iter_batched( + || state.clone(), + |mut state| { + process_randao(&mut state, &block, &spec).unwrap(); + state + }, + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(10), + ); + + let state = initial_state.clone(); + let block = initial_block.clone(); + c.bench( + &format!("{}/block_processing", desc), + Benchmark::new("process_eth1_data", move |b| { + b.iter_batched( + || state.clone(), + |mut state| { + process_eth1_data(&mut state, &block.eth1_data).unwrap(); + state + }, + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(10), + ); + + let state = initial_state.clone(); + let block = initial_block.clone(); + let spec = initial_spec.clone(); + c.bench( + &format!("{}/block_processing", desc), + Benchmark::new("process_proposer_slashings", move |b| { + b.iter_batched( + || state.clone(), + |mut state| { + process_proposer_slashings(&mut state, &block.body.proposer_slashings, &spec) + .unwrap(); + state + }, + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(10), + ); + + let state = initial_state.clone(); + let block = initial_block.clone(); + let spec = initial_spec.clone(); + c.bench( + &format!("{}/block_processing", desc), + Benchmark::new("process_attester_slashings", move |b| { + b.iter_batched( + || state.clone(), + |mut state| { + process_attester_slashings(&mut state, &block.body.attester_slashings, &spec) + .unwrap(); + state + }, + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(10), + ); + + let state = initial_state.clone(); + let block = initial_block.clone(); + let spec = initial_spec.clone(); + c.bench( + &format!("{}/block_processing", desc), + Benchmark::new("process_attestations", move |b| { + b.iter_batched( + || state.clone(), + |mut state| { + process_attestations(&mut state, &block.body.attestations, &spec).unwrap(); + state + }, + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(10), + ); + + let state = initial_state.clone(); + let block = initial_block.clone(); + let spec = initial_spec.clone(); + c.bench( + &format!("{}/block_processing", desc), + Benchmark::new("process_deposits", move |b| { + b.iter_batched( + || state.clone(), + |mut state| { + process_deposits(&mut state, &block.body.deposits, &spec).unwrap(); + state + }, + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(10), + ); + + let state = initial_state.clone(); + let block = initial_block.clone(); + let spec = initial_spec.clone(); + c.bench( + &format!("{}/block_processing", desc), + Benchmark::new("process_exits", move |b| { + b.iter_batched( + || state.clone(), + |mut state| { + process_exits(&mut state, &block.body.voluntary_exits, &spec).unwrap(); + state + }, + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(10), + ); + + let state = initial_state.clone(); + let block = initial_block.clone(); + let spec = initial_spec.clone(); + c.bench( + &format!("{}/block_processing", desc), + Benchmark::new("process_transfers", move |b| { + b.iter_batched( + || state.clone(), + |mut state| { + process_transfers(&mut state, &block.body.transfers, &spec).unwrap(); + state + }, + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(10), + ); + + let state = initial_state.clone(); + let block = initial_block.clone(); + let spec = initial_spec.clone(); + c.bench( + &format!("{}/block_processing", desc), + Benchmark::new("per_block_processing", move |b| { + b.iter_batched( + || state.clone(), + |mut state| { + per_block_processing(&mut state, &block, &spec).unwrap(); + state + }, + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(10), + ); + + let mut state = initial_state.clone(); + state.drop_cache(RelativeEpoch::Previous); + let spec = initial_spec.clone(); + c.bench( + &format!("{}/block_processing", desc), + Benchmark::new("build_previous_state_epoch_cache", move |b| { + b.iter_batched( + || state.clone(), + |mut state| { + state + .build_epoch_cache(RelativeEpoch::Previous, &spec) + .unwrap(); + state + }, + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(10), + ); + + let mut state = initial_state.clone(); + state.drop_cache(RelativeEpoch::Current); + let spec = initial_spec.clone(); + c.bench( + &format!("{}/block_processing", desc), + Benchmark::new("build_current_state_epoch_cache", move |b| { + b.iter_batched( + || state.clone(), + |mut state| { + state + .build_epoch_cache(RelativeEpoch::Current, &spec) + .unwrap(); + state + }, + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(10), + ); + + let mut state = initial_state.clone(); + state.drop_pubkey_cache(); + c.bench( + &format!("{}/block_processing", desc), + Benchmark::new("build_pubkey_cache", move |b| { + b.iter_batched( + || state.clone(), + |mut state| { + state.update_pubkey_cache().unwrap(); + state + }, + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(10), + ); + + let block = initial_block.clone(); + c.bench( + &format!("{}/block_processing", desc), + Benchmark::new("tree_hash_block", move |b| { + b.iter(|| black_box(block.hash_tree_root())) + }) + .sample_size(10), + ); +} diff --git a/eth2/state_processing/benches/bench_epoch_processing.rs b/eth2/state_processing/benches/bench_epoch_processing.rs new file mode 100644 index 000000000..cc7701296 --- /dev/null +++ b/eth2/state_processing/benches/bench_epoch_processing.rs @@ -0,0 +1,263 @@ +use criterion::Criterion; +use criterion::{black_box, Benchmark}; +use ssz::TreeHash; +use state_processing::{ + per_epoch_processing, + per_epoch_processing::{ + clean_attestations, initialize_validator_statuses, process_crosslinks, process_eth1_data, + process_justification, process_rewards_and_penalities, process_validator_registry, + update_active_tree_index_roots, update_latest_slashed_balances, + }, +}; +use types::test_utils::TestingBeaconStateBuilder; +use types::*; + +pub const BENCHING_SAMPLE_SIZE: usize = 10; +pub const SMALL_BENCHING_SAMPLE_SIZE: usize = 10; + +/// Run the benchmarking suite on a foundation spec with 16,384 validators. +pub fn bench_epoch_processing_n_validators(c: &mut Criterion, validator_count: usize) { + let spec = ChainSpec::foundation(); + + let mut builder = + TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, &spec); + + // Set the state to be just before an epoch transition. + let target_slot = (spec.genesis_epoch + 4).end_slot(spec.slots_per_epoch); + builder.teleport_to_slot(target_slot, &spec); + + // Builds all caches; benches will not contain shuffling/committee building times. + builder.build_caches(&spec).unwrap(); + + // Inserts one attestation with full participation for each committee able to include an + // attestation in this state. + builder.insert_attestations(&spec); + + let (state, _keypairs) = builder.build(); + + // Assert that the state has an attestations for each committee that is able to include an + // attestation in the state. + let committees_per_epoch = spec.get_epoch_committee_count(validator_count); + let committees_per_slot = committees_per_epoch / spec.slots_per_epoch; + let previous_epoch_attestations = committees_per_epoch; + let current_epoch_attestations = + committees_per_slot * (spec.slots_per_epoch - spec.min_attestation_inclusion_delay); + assert_eq!( + state.latest_attestations.len() as u64, + previous_epoch_attestations + current_epoch_attestations, + "The state should have an attestation for each committee." + ); + + // Assert that we will run the first arm of process_rewards_and_penalities + let epochs_since_finality = state.next_epoch(&spec) - state.finalized_epoch; + assert_eq!( + epochs_since_finality, 4, + "Epochs since finality should be 4" + ); + + bench_epoch_processing(c, &state, &spec, &format!("{}_validators", validator_count)); +} + +/// Run the detailed benchmarking suite on the given `BeaconState`. +/// +/// `desc` will be added to the title of each bench. +fn bench_epoch_processing(c: &mut Criterion, state: &BeaconState, spec: &ChainSpec, desc: &str) { + let state_clone = state.clone(); + let spec_clone = spec.clone(); + c.bench( + &format!("{}/epoch_processing", desc), + Benchmark::new("process_eth1_data", move |b| { + b.iter_batched( + || state_clone.clone(), + |mut state| { + process_eth1_data(&mut state, &spec_clone); + state + }, + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(BENCHING_SAMPLE_SIZE), + ); + + let state_clone = state.clone(); + let spec_clone = spec.clone(); + c.bench( + &format!("{}/epoch_processing", desc), + Benchmark::new("initialize_validator_statuses", move |b| { + b.iter_batched( + || state_clone.clone(), + |mut state| { + initialize_validator_statuses(&mut state, &spec_clone).unwrap(); + state + }, + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(BENCHING_SAMPLE_SIZE), + ); + + let state_clone = state.clone(); + let spec_clone = spec.clone(); + let attesters = initialize_validator_statuses(&state, &spec).unwrap(); + c.bench( + &format!("{}/epoch_processing", desc), + Benchmark::new("process_justification", move |b| { + b.iter_batched( + || state_clone.clone(), + |mut state| { + process_justification(&mut state, &attesters.total_balances, &spec_clone); + state + }, + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(10), + ); + + let state_clone = state.clone(); + let spec_clone = spec.clone(); + c.bench( + &format!("{}/epoch_processing", desc), + Benchmark::new("process_crosslinks", move |b| { + b.iter_batched( + || state_clone.clone(), + |mut state| black_box(process_crosslinks(&mut state, &spec_clone).unwrap()), + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(BENCHING_SAMPLE_SIZE), + ); + + let mut state_clone = state.clone(); + let spec_clone = spec.clone(); + let attesters = initialize_validator_statuses(&state, &spec).unwrap(); + let winning_root_for_shards = process_crosslinks(&mut state_clone, &spec).unwrap(); + c.bench( + &format!("{}/epoch_processing", desc), + Benchmark::new("process_rewards_and_penalties", move |b| { + b.iter_batched( + || (state_clone.clone(), attesters.clone()), + |(mut state, mut attesters)| { + process_rewards_and_penalities( + &mut state, + &mut attesters, + &winning_root_for_shards, + &spec_clone, + ) + .unwrap(); + state + }, + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(SMALL_BENCHING_SAMPLE_SIZE), + ); + + let state_clone = state.clone(); + let spec_clone = spec.clone(); + c.bench( + &format!("{}/epoch_processing", desc), + Benchmark::new("process_ejections", move |b| { + b.iter_batched( + || state_clone.clone(), + |mut state| { + state.process_ejections(&spec_clone); + state + }, + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(BENCHING_SAMPLE_SIZE), + ); + + let state_clone = state.clone(); + let spec_clone = spec.clone(); + c.bench( + &format!("{}/epoch_processing", desc), + Benchmark::new("process_validator_registry", move |b| { + b.iter_batched( + || state_clone.clone(), + |mut state| { + process_validator_registry(&mut state, &spec_clone).unwrap(); + state + }, + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(BENCHING_SAMPLE_SIZE), + ); + + let state_clone = state.clone(); + let spec_clone = spec.clone(); + c.bench( + &format!("{}/epoch_processing", desc), + Benchmark::new("update_active_tree_index_roots", move |b| { + b.iter_batched( + || state_clone.clone(), + |mut state| { + update_active_tree_index_roots(&mut state, &spec_clone).unwrap(); + state + }, + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(BENCHING_SAMPLE_SIZE), + ); + + let state_clone = state.clone(); + let spec_clone = spec.clone(); + c.bench( + &format!("{}/epoch_processing", desc), + Benchmark::new("update_latest_slashed_balances", move |b| { + b.iter_batched( + || state_clone.clone(), + |mut state| { + update_latest_slashed_balances(&mut state, &spec_clone); + state + }, + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(BENCHING_SAMPLE_SIZE), + ); + + let state_clone = state.clone(); + let spec_clone = spec.clone(); + c.bench( + &format!("{}/epoch_processing", desc), + Benchmark::new("clean_attestations", move |b| { + b.iter_batched( + || state_clone.clone(), + |mut state| { + clean_attestations(&mut state, &spec_clone); + state + }, + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(BENCHING_SAMPLE_SIZE), + ); + + let state_clone = state.clone(); + let spec_clone = spec.clone(); + c.bench( + &format!("{}/epoch_processing", desc), + Benchmark::new("per_epoch_processing", move |b| { + b.iter_batched( + || state_clone.clone(), + |mut state| black_box(per_epoch_processing(&mut state, &spec_clone).unwrap()), + criterion::BatchSize::SmallInput, + ) + }) + .sample_size(SMALL_BENCHING_SAMPLE_SIZE), + ); + + let state_clone = state.clone(); + c.bench( + &format!("{}/epoch_processing", desc), + Benchmark::new("tree_hash_state", move |b| { + b.iter(|| black_box(state_clone.hash_tree_root())) + }) + .sample_size(SMALL_BENCHING_SAMPLE_SIZE), + ); +} diff --git a/eth2/state_processing/benches/benches.rs b/eth2/state_processing/benches/benches.rs index 682259eef..0cf797147 100644 --- a/eth2/state_processing/benches/benches.rs +++ b/eth2/state_processing/benches/benches.rs @@ -1,65 +1,103 @@ +use block_benching_builder::BlockBenchingBuilder; use criterion::Criterion; -use criterion::{black_box, criterion_group, criterion_main, Benchmark}; -// use env_logger::{Builder, Env}; -use state_processing::SlotProcessable; -use types::beacon_state::BeaconStateBuilder; +use criterion::{criterion_group, criterion_main}; +use env_logger::{Builder, Env}; +use log::info; use types::*; -fn epoch_processing(c: &mut Criterion) { - // Builder::from_env(Env::default().default_filter_or("debug")).init(); +mod bench_block_processing; +mod bench_epoch_processing; +mod block_benching_builder; - let mut builder = BeaconStateBuilder::new(16_384); +pub const VALIDATOR_COUNT: usize = 16_384; - builder.build_fast().unwrap(); - builder.teleport_to_end_of_epoch(builder.spec.genesis_epoch + 4); +// `LOG_LEVEL == "info"` gives handy messages. +pub const LOG_LEVEL: &str = "info"; - let mut state = builder.cloned_state(); - - // Build all the caches so the following state does _not_ include the cache-building time. - state - .build_epoch_cache(RelativeEpoch::Previous, &builder.spec) - .unwrap(); - state - .build_epoch_cache(RelativeEpoch::Current, &builder.spec) - .unwrap(); - state - .build_epoch_cache(RelativeEpoch::Next, &builder.spec) - .unwrap(); - - let cached_state = state.clone(); - - // Drop all the caches so the following state includes the cache-building time. - state.drop_cache(RelativeEpoch::Previous); - state.drop_cache(RelativeEpoch::Current); - state.drop_cache(RelativeEpoch::Next); - - let cacheless_state = state; - - let spec_a = builder.spec.clone(); - let spec_b = builder.spec.clone(); - - c.bench( - "epoch processing", - Benchmark::new("with pre-built caches", move |b| { - b.iter_with_setup( - || cached_state.clone(), - |mut state| black_box(state.per_slot_processing(Hash256::zero(), &spec_a).unwrap()), - ) - }) - .sample_size(10), +/// Build a worst-case block and benchmark processing it. +pub fn block_processing_worst_case(c: &mut Criterion) { + if LOG_LEVEL != "" { + Builder::from_env(Env::default().default_filter_or(LOG_LEVEL)).init(); + } + info!( + "Building worst case block bench with {} validators", + VALIDATOR_COUNT ); - c.bench( - "epoch processing", - Benchmark::new("without pre-built caches", move |b| { - b.iter_with_setup( - || cacheless_state.clone(), - |mut state| black_box(state.per_slot_processing(Hash256::zero(), &spec_b).unwrap()), - ) - }) - .sample_size(10), + // Use the specifications from the Eth2.0 spec. + let spec = ChainSpec::foundation(); + + // Create a builder for configuring the block and state for benching. + let mut bench_builder = BlockBenchingBuilder::new(VALIDATOR_COUNT, &spec); + + // Set the number of included operations to be maximum (e.g., `MAX_ATTESTATIONS`, etc.) + bench_builder.maximize_block_operations(&spec); + + // Set the state and block to be in the last slot of the 4th epoch. + let last_slot_of_epoch = (spec.genesis_epoch + 4).end_slot(spec.slots_per_epoch); + bench_builder.set_slot(last_slot_of_epoch, &spec); + + // Build all the state caches so the build times aren't included in the benches. + bench_builder.build_caches(&spec); + + // Generate the block and state then run benches. + let (block, state) = bench_builder.build(&spec); + bench_block_processing::bench_block_processing( + c, + &block, + &state, + &spec, + &format!("{}_validators/worst_case", VALIDATOR_COUNT), ); } -criterion_group!(benches, epoch_processing,); +/// Build a reasonable-case block and benchmark processing it. +pub fn block_processing_reasonable_case(c: &mut Criterion) { + info!( + "Building reasonable case block bench with {} validators", + VALIDATOR_COUNT + ); + + // Use the specifications from the Eth2.0 spec. + let spec = ChainSpec::foundation(); + + // Create a builder for configuring the block and state for benching. + let mut bench_builder = BlockBenchingBuilder::new(VALIDATOR_COUNT, &spec); + + // Set the number of included operations to what we might expect normally. + bench_builder.num_proposer_slashings = 0; + bench_builder.num_attester_slashings = 0; + bench_builder.num_attestations = (spec.shard_count / spec.slots_per_epoch) as usize; + bench_builder.num_deposits = 2; + bench_builder.num_exits = 2; + bench_builder.num_transfers = 2; + + // Set the state and block to be in the last slot of the 4th epoch. + let last_slot_of_epoch = (spec.genesis_epoch + 4).end_slot(spec.slots_per_epoch); + bench_builder.set_slot(last_slot_of_epoch, &spec); + + // Build all the state caches so the build times aren't included in the benches. + bench_builder.build_caches(&spec); + + // Generate the block and state then run benches. + let (block, state) = bench_builder.build(&spec); + bench_block_processing::bench_block_processing( + c, + &block, + &state, + &spec, + &format!("{}_validators/reasonable_case", VALIDATOR_COUNT), + ); +} + +pub fn state_processing(c: &mut Criterion) { + bench_epoch_processing::bench_epoch_processing_n_validators(c, VALIDATOR_COUNT); +} + +criterion_group!( + benches, + block_processing_reasonable_case, + block_processing_worst_case, + state_processing +); criterion_main!(benches); diff --git a/eth2/state_processing/benches/block_benching_builder.rs b/eth2/state_processing/benches/block_benching_builder.rs new file mode 100644 index 000000000..b993851d7 --- /dev/null +++ b/eth2/state_processing/benches/block_benching_builder.rs @@ -0,0 +1,175 @@ +use log::info; +use types::test_utils::{TestingBeaconBlockBuilder, TestingBeaconStateBuilder}; +use types::*; + +pub struct BlockBenchingBuilder { + pub state_builder: TestingBeaconStateBuilder, + pub block_builder: TestingBeaconBlockBuilder, + + pub num_validators: usize, + pub num_proposer_slashings: usize, + pub num_attester_slashings: usize, + pub num_indices_per_slashable_vote: usize, + pub num_attestations: usize, + pub num_deposits: usize, + pub num_exits: usize, + pub num_transfers: usize, +} + +impl BlockBenchingBuilder { + pub fn new(num_validators: usize, spec: &ChainSpec) -> Self { + let state_builder = + TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(num_validators, &spec); + let block_builder = TestingBeaconBlockBuilder::new(spec); + + Self { + state_builder, + block_builder, + num_validators: 0, + num_proposer_slashings: 0, + num_attester_slashings: 0, + num_indices_per_slashable_vote: spec.max_indices_per_slashable_vote as usize, + num_attestations: 0, + num_deposits: 0, + num_exits: 0, + num_transfers: 0, + } + } + + pub fn maximize_block_operations(&mut self, spec: &ChainSpec) { + self.num_proposer_slashings = spec.max_proposer_slashings as usize; + self.num_attester_slashings = spec.max_attester_slashings as usize; + self.num_indices_per_slashable_vote = spec.max_indices_per_slashable_vote as usize; + self.num_attestations = spec.max_attestations as usize; + self.num_deposits = spec.max_deposits as usize; + self.num_exits = spec.max_voluntary_exits as usize; + self.num_transfers = spec.max_transfers as usize; + } + + pub fn set_slot(&mut self, slot: Slot, spec: &ChainSpec) { + self.state_builder.teleport_to_slot(slot, &spec); + } + + pub fn build_caches(&mut self, spec: &ChainSpec) { + // Builds all caches; benches will not contain shuffling/committee building times. + self.state_builder.build_caches(&spec).unwrap(); + } + + pub fn build(mut self, spec: &ChainSpec) -> (BeaconBlock, BeaconState) { + let (mut state, keypairs) = self.state_builder.build(); + let builder = &mut self.block_builder; + + builder.set_slot(state.slot); + + let proposer_index = state.get_beacon_proposer_index(state.slot, spec).unwrap(); + let keypair = &keypairs[proposer_index]; + + builder.set_randao_reveal(&keypair.sk, &state.fork, spec); + + // Used as a stream of validator indices for use in slashings, exits, etc. + let mut validators_iter = (0..keypairs.len() as u64).into_iter(); + + // Insert `ProposerSlashing` objects. + for _ in 0..self.num_proposer_slashings { + let validator_index = validators_iter.next().expect("Insufficient validators."); + + builder.insert_proposer_slashing( + validator_index, + &keypairs[validator_index as usize].sk, + &state.fork, + spec, + ); + } + info!( + "Inserted {} proposer slashings.", + builder.block.body.proposer_slashings.len() + ); + + // Insert `AttesterSlashing` objects + for _ in 0..self.num_attester_slashings { + let mut attesters: Vec = vec![]; + let mut secret_keys: Vec<&SecretKey> = vec![]; + + for _ in 0..self.num_indices_per_slashable_vote { + let validator_index = validators_iter.next().expect("Insufficient validators."); + + attesters.push(validator_index); + secret_keys.push(&keypairs[validator_index as usize].sk); + } + + builder.insert_attester_slashing(&attesters, &secret_keys, &state.fork, spec); + } + info!( + "Inserted {} attester slashings.", + builder.block.body.attester_slashings.len() + ); + + // Insert `Attestation` objects. + let all_secret_keys: Vec<&SecretKey> = keypairs.iter().map(|keypair| &keypair.sk).collect(); + builder + .insert_attestations( + &state, + &all_secret_keys, + self.num_attestations as usize, + spec, + ) + .unwrap(); + info!( + "Inserted {} attestations.", + builder.block.body.attestations.len() + ); + + // Insert `Deposit` objects. + for i in 0..self.num_deposits { + builder.insert_deposit( + 32_000_000_000, + state.deposit_index + (i as u64), + &state, + spec, + ); + } + info!("Inserted {} deposits.", builder.block.body.deposits.len()); + + // Insert the maximum possible number of `Exit` objects. + for _ in 0..self.num_exits { + let validator_index = validators_iter.next().expect("Insufficient validators."); + + builder.insert_exit( + &state, + validator_index, + &keypairs[validator_index as usize].sk, + spec, + ); + } + info!( + "Inserted {} exits.", + builder.block.body.voluntary_exits.len() + ); + + // Insert the maximum possible number of `Transfer` objects. + for _ in 0..self.num_transfers { + let validator_index = validators_iter.next().expect("Insufficient validators."); + + // Manually set the validator to be withdrawn. + state.validator_registry[validator_index as usize].withdrawable_epoch = + state.previous_epoch(spec); + + builder.insert_transfer( + &state, + validator_index, + validator_index, + 1, + keypairs[validator_index as usize].clone(), + spec, + ); + } + info!("Inserted {} transfers.", builder.block.body.transfers.len()); + + let mut block = self.block_builder.build(&keypair.sk, &state.fork, spec); + + // Set the eth1 data to be different from the state. + block.eth1_data.block_hash = Hash256::from_slice(&vec![42; 32]); + + (block, state) + } +} diff --git a/eth2/state_processing/specs/example.yml b/eth2/state_processing/specs/example.yml new file mode 100644 index 000000000..95b749bc4 --- /dev/null +++ b/eth2/state_processing/specs/example.yml @@ -0,0 +1,347 @@ +title: Sanity tests +summary: Basic sanity checks from phase 0 spec pythonization. All tests are run with + `verify_signatures` as set to False. +test_suite: beacon_state +fork: tchaikovsky +version: v0.5.0 +test_cases: +- name: test_empty_block_transition + config: {SHARD_COUNT: 8, TARGET_COMMITTEE_SIZE: 4, MAX_BALANCE_CHURN_QUOTIENT: 32, + MAX_INDICES_PER_SLASHABLE_VOTE: 4096, MAX_EXIT_DEQUEUES_PER_EPOCH: 4, SHUFFLE_ROUND_COUNT: 90, + DEPOSIT_CONTRACT_TREE_DEPTH: 32, MIN_DEPOSIT_AMOUNT: 1000000000, MAX_DEPOSIT_AMOUNT: 32000000000, + FORK_CHOICE_BALANCE_INCREMENT: 1000000000, EJECTION_BALANCE: 16000000000, GENESIS_FORK_VERSION: 0, + GENESIS_SLOT: 4294967296, GENESIS_EPOCH: 536870912, GENESIS_START_SHARD: 0, BLS_WITHDRAWAL_PREFIX_BYTE: 0, + SECONDS_PER_SLOT: 6, MIN_ATTESTATION_INCLUSION_DELAY: 2, SLOTS_PER_EPOCH: 8, MIN_SEED_LOOKAHEAD: 1, + ACTIVATION_EXIT_DELAY: 4, EPOCHS_PER_ETH1_VOTING_PERIOD: 16, SLOTS_PER_HISTORICAL_ROOT: 64, + MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256, PERSISTENT_COMMITTEE_PERIOD: 2048, LATEST_RANDAO_MIXES_LENGTH: 64, + LATEST_ACTIVE_INDEX_ROOTS_LENGTH: 64, LATEST_SLASHED_EXIT_LENGTH: 64, BASE_REWARD_QUOTIENT: 32, + WHISTLEBLOWER_REWARD_QUOTIENT: 512, ATTESTATION_INCLUSION_REWARD_QUOTIENT: 8, + INACTIVITY_PENALTY_QUOTIENT: 16777216, MIN_PENALTY_QUOTIENT: 32, MAX_PROPOSER_SLASHINGS: 16, + MAX_ATTESTER_SLASHINGS: 1, MAX_ATTESTATIONS: 128, MAX_DEPOSITS: 16, MAX_VOLUNTARY_EXITS: 16, + MAX_TRANSFERS: 16, DOMAIN_BEACON_BLOCK: 0, DOMAIN_RANDAO: 1, DOMAIN_ATTESTATION: 2, + DOMAIN_DEPOSIT: 3, DOMAIN_VOLUNTARY_EXIT: 4, DOMAIN_TRANSFER: 5} + verify_signatures: false + initial_state: + slot: 4294967296 + genesis_time: 0 + fork: {previous_version: 0, current_version: 0, epoch: 536870912} + validator_registry: + - {pubkey: '0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x020000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x030000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x050000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x060000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x070000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x090000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x0a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x0b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x0c0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x0d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x0e0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x0f0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x120000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x130000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x140000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x150000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x160000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x170000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x180000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x190000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x1a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x1b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x1c0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x1d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x1e0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + - {pubkey: '0x1f0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + withdrawal_credentials: '0x2222222222222222222222222222222222222222222222222222222222222222', + activation_epoch: 536870912, exit_epoch: 18446744073709551615, withdrawable_epoch: 18446744073709551615, + initiated_exit: false, slashed: false} + validator_balances: [32000000000, 32000000000, 32000000000, 32000000000, 32000000000, + 32000000000, 32000000000, 32000000000, 32000000000, 32000000000, 32000000000, + 32000000000, 32000000000, 32000000000, 32000000000, 32000000000, 32000000000, + 32000000000, 32000000000, 32000000000, 32000000000, 32000000000, 32000000000, + 32000000000, 32000000000, 32000000000, 32000000000, 32000000000, 32000000000, + 32000000000, 32000000000, 32000000000] + validator_registry_update_epoch: 536870912 + latest_randao_mixes: ['0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000'] + previous_shuffling_start_shard: 0 + current_shuffling_start_shard: 0 + previous_shuffling_epoch: 536870912 + current_shuffling_epoch: 536870912 + previous_shuffling_seed: '0x0000000000000000000000000000000000000000000000000000000000000000' + current_shuffling_seed: '0x94ab448e948e6d501a2b48c1e9a0946f871100969f6fa70a990acf2348c9b185' + previous_epoch_attestations: [] + current_epoch_attestations: [] + previous_justified_epoch: 536870912 + current_justified_epoch: 536870912 + previous_justified_root: '0x0000000000000000000000000000000000000000000000000000000000000000' + current_justified_root: '0x0000000000000000000000000000000000000000000000000000000000000000' + justification_bitfield: 0 + finalized_epoch: 536870912 + finalized_root: '0x0000000000000000000000000000000000000000000000000000000000000000' + latest_crosslinks: + - {epoch: 536870912, crosslink_data_root: '0x0000000000000000000000000000000000000000000000000000000000000000'} + - {epoch: 536870912, crosslink_data_root: '0x0000000000000000000000000000000000000000000000000000000000000000'} + - {epoch: 536870912, crosslink_data_root: '0x0000000000000000000000000000000000000000000000000000000000000000'} + - {epoch: 536870912, crosslink_data_root: '0x0000000000000000000000000000000000000000000000000000000000000000'} + - {epoch: 536870912, crosslink_data_root: '0x0000000000000000000000000000000000000000000000000000000000000000'} + - {epoch: 536870912, crosslink_data_root: '0x0000000000000000000000000000000000000000000000000000000000000000'} + - {epoch: 536870912, crosslink_data_root: '0x0000000000000000000000000000000000000000000000000000000000000000'} + - {epoch: 536870912, crosslink_data_root: '0x0000000000000000000000000000000000000000000000000000000000000000'} + latest_block_roots: ['0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000'] + latest_state_roots: ['0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000'] + latest_active_index_roots: ['0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42', + '0xf6b8ca96e524598ba62d563347f5aea6ce2d81d644e2788687e5a92844df1b42'] + latest_slashed_balances: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + latest_block_header: {slot: 4294967296, previous_block_root: '0x0000000000000000000000000000000000000000000000000000000000000000', + state_root: '0x0000000000000000000000000000000000000000000000000000000000000000', + block_body_root: '0x5359b62990beb1d78e1cec479f5a4d80af84709886a8e16c535dff0556dc0e2d', + signature: '0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'} + historical_roots: [] + latest_eth1_data: {deposit_root: '0xb05de6a9059df0c9a2ab5f76708d256941dfe9eb89e6fda549b30713087d2a5e', + block_hash: '0x0000000000000000000000000000000000000000000000000000000000000000'} + eth1_data_votes: [] + deposit_index: 32 + blocks: + - slot: 4294967297 + previous_block_root: '0x92ed652508d2b4c109a857107101716b18e257e7ce0d199d4b16232956e9e27e' + state_root: '0x0000000000000000000000000000000000000000000000000000000000000000' + body: + randao_reveal: '0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000' + eth1_data: {deposit_root: '0x0000000000000000000000000000000000000000000000000000000000000000', + block_hash: '0x0000000000000000000000000000000000000000000000000000000000000000'} + proposer_slashings: [] + attester_slashings: [] + attestations: [] + deposits: [] + voluntary_exits: [] + transfers: [] + signature: '0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000' + expected_state: {slot: 4294967297} diff --git a/eth2/state_processing/src/get_genesis_state.rs b/eth2/state_processing/src/get_genesis_state.rs new file mode 100644 index 000000000..3c6612349 --- /dev/null +++ b/eth2/state_processing/src/get_genesis_state.rs @@ -0,0 +1,59 @@ +use super::per_block_processing::{errors::BlockProcessingError, process_deposits}; +use ssz::TreeHash; +use types::*; + +pub enum GenesisError { + BlockProcessingError(BlockProcessingError), + BeaconStateError(BeaconStateError), +} + +/// Returns the genesis `BeaconState` +/// +/// Spec v0.5.0 +pub fn get_genesis_state( + genesis_validator_deposits: &[Deposit], + genesis_time: u64, + genesis_eth1_data: Eth1Data, + spec: &ChainSpec, +) -> Result<(), BlockProcessingError> { + // Get the genesis `BeaconState` + let mut state = BeaconState::genesis(genesis_time, genesis_eth1_data, spec); + + // Process genesis deposits. + process_deposits(&mut state, genesis_validator_deposits, spec)?; + + // Process genesis activations. + for i in 0..state.validator_registry.len() { + if state.get_effective_balance(i, spec)? >= spec.max_deposit_amount { + state.validator_registry[i].activation_epoch = spec.genesis_epoch; + } + } + + // Ensure the current epoch cache is built. + state.build_epoch_cache(RelativeEpoch::Current, spec)?; + + // Set all the active index roots to be the genesis active index root. + let active_validator_indices = state + .get_active_validator_indices(spec.genesis_epoch, spec)? + .to_vec(); + let genesis_active_index_root = Hash256::from_slice(&active_validator_indices.hash_tree_root()); + state.latest_active_index_roots = + vec![genesis_active_index_root; spec.latest_active_index_roots_length as usize]; + + // Generate the current shuffling seed. + state.current_shuffling_seed = state.generate_seed(spec.genesis_epoch, spec)?; + + Ok(()) +} + +impl From for GenesisError { + fn from(e: BlockProcessingError) -> GenesisError { + GenesisError::BlockProcessingError(e) + } +} + +impl From for GenesisError { + fn from(e: BeaconStateError) -> GenesisError { + GenesisError::BeaconStateError(e) + } +} diff --git a/eth2/state_processing/src/lib.rs b/eth2/state_processing/src/lib.rs index 2b30844cb..78dc7270d 100644 --- a/eth2/state_processing/src/lib.rs +++ b/eth2/state_processing/src/lib.rs @@ -1,10 +1,12 @@ #[macro_use] mod macros; +pub mod get_genesis_state; pub mod per_block_processing; pub mod per_epoch_processing; pub mod per_slot_processing; +pub use get_genesis_state::get_genesis_state; pub use per_block_processing::{ errors::{BlockInvalid, BlockProcessingError}, per_block_processing, per_block_processing_without_verifying_block_signature, diff --git a/eth2/state_processing/src/per_block_processing.rs b/eth2/state_processing/src/per_block_processing.rs index 1ab1eed71..78cf927f5 100644 --- a/eth2/state_processing/src/per_block_processing.rs +++ b/eth2/state_processing/src/per_block_processing.rs @@ -1,14 +1,16 @@ use self::verify_proposer_slashing::verify_proposer_slashing; use errors::{BlockInvalid as Invalid, BlockProcessingError as Error, IntoWithIndex}; -use hashing::hash; -use log::debug; -use ssz::{ssz_encode, SignedRoot, TreeHash}; +use rayon::prelude::*; +use ssz::{SignedRoot, TreeHash}; use types::*; -pub use self::verify_attester_slashing::verify_attester_slashing; +pub use self::verify_attester_slashing::{ + gather_attester_slashing_indices, verify_attester_slashing, +}; pub use validate_attestation::{validate_attestation, validate_attestation_without_signature}; -pub use verify_deposit::verify_deposit; +pub use verify_deposit::{get_existing_validator_index, verify_deposit, verify_deposit_index}; pub use verify_exit::verify_exit; +pub use verify_slashable_attestation::verify_slashable_attestation; pub use verify_transfer::{execute_transfer, verify_transfer}; pub mod errors; @@ -30,7 +32,7 @@ const VERIFY_DEPOSIT_MERKLE_PROOFS: bool = false; /// Returns `Ok(())` if the block is valid and the state was successfully updated. Otherwise /// returns an error describing why the block was invalid or how the function failed to execute. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn per_block_processing( state: &mut BeaconState, block: &BeaconBlock, @@ -45,7 +47,7 @@ pub fn per_block_processing( /// Returns `Ok(())` if the block is valid and the state was successfully updated. Otherwise /// returns an error describing why the block was invalid or how the function failed to execute. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn per_block_processing_without_verifying_block_signature( state: &mut BeaconState, block: &BeaconBlock, @@ -60,63 +62,75 @@ pub fn per_block_processing_without_verifying_block_signature( /// Returns `Ok(())` if the block is valid and the state was successfully updated. Otherwise /// returns an error describing why the block was invalid or how the function failed to execute. /// -/// Spec v0.4.0 +/// Spec v0.5.0 fn per_block_processing_signature_optional( mut state: &mut BeaconState, block: &BeaconBlock, should_verify_block_signature: bool, spec: &ChainSpec, ) -> Result<(), Error> { - // Verify that `block.slot == state.slot`. - verify!(block.slot == state.slot, Invalid::StateSlotMismatch); + process_block_header(state, block, spec)?; - // Ensure the current epoch cache is built. + // Ensure the current and previous epoch cache is built. + state.build_epoch_cache(RelativeEpoch::Previous, spec)?; state.build_epoch_cache(RelativeEpoch::Current, spec)?; if should_verify_block_signature { verify_block_signature(&state, &block, &spec)?; } process_randao(&mut state, &block, &spec)?; - process_eth1_data(&mut state, &block.eth1_data)?; - process_proposer_slashings(&mut state, &block.body.proposer_slashings[..], spec)?; - process_attester_slashings(&mut state, &block.body.attester_slashings[..], spec)?; - process_attestations(&mut state, &block.body.attestations[..], spec)?; - process_deposits(&mut state, &block.body.deposits[..], spec)?; - process_exits(&mut state, &block.body.voluntary_exits[..], spec)?; - process_transfers(&mut state, &block.body.transfers[..], spec)?; + process_eth1_data(&mut state, &block.body.eth1_data)?; + process_proposer_slashings(&mut state, &block.body.proposer_slashings, spec)?; + process_attester_slashings(&mut state, &block.body.attester_slashings, spec)?; + process_attestations(&mut state, &block.body.attestations, spec)?; + process_deposits(&mut state, &block.body.deposits, spec)?; + process_exits(&mut state, &block.body.voluntary_exits, spec)?; + process_transfers(&mut state, &block.body.transfers, spec)?; - debug!("per_block_processing complete."); + Ok(()) +} + +/// Processes the block header. +/// +/// Spec v0.5.0 +pub fn process_block_header( + state: &mut BeaconState, + block: &BeaconBlock, + spec: &ChainSpec, +) -> Result<(), Error> { + verify!(block.slot == state.slot, Invalid::StateSlotMismatch); + + verify!( + block.previous_block_root.as_bytes() == &state.latest_block_header.hash_tree_root()[..], + Invalid::ParentBlockRootMismatch + ); + + state.latest_block_header = block.into_temporary_header(spec); Ok(()) } /// Verifies the signature of a block. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn verify_block_signature( state: &BeaconState, block: &BeaconBlock, spec: &ChainSpec, ) -> Result<(), Error> { - let block_proposer = - &state.validator_registry[state.get_beacon_proposer_index(block.slot, spec)?]; + let block_proposer = &state.validator_registry + [state.get_beacon_proposer_index(block.slot, RelativeEpoch::Current, spec)?]; - let proposal = Proposal { - slot: block.slot, - shard: spec.beacon_chain_shard_number, - block_root: Hash256::from_slice(&block.signed_root()[..]), - signature: block.signature.clone(), - }; let domain = spec.get_domain( block.slot.epoch(spec.slots_per_epoch), - Domain::Proposal, + Domain::BeaconBlock, &state.fork, ); verify!( - proposal + block .signature - .verify(&proposal.signed_root()[..], domain, &block_proposer.pubkey), + .verify(&block.signed_root()[..], domain, &block_proposer.pubkey), Invalid::BadSignature ); @@ -126,21 +140,18 @@ pub fn verify_block_signature( /// Verifies the `randao_reveal` against the block's proposer pubkey and updates /// `state.latest_randao_mixes`. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn process_randao( state: &mut BeaconState, block: &BeaconBlock, spec: &ChainSpec, ) -> Result<(), Error> { - // Let `proposer = state.validator_registry[get_beacon_proposer_index(state, state.slot)]`. - let block_proposer = - &state.validator_registry[state.get_beacon_proposer_index(block.slot, spec)?]; + let block_proposer = &state.validator_registry + [state.get_beacon_proposer_index(block.slot, RelativeEpoch::Current, spec)?]; - // Verify that `bls_verify(pubkey=proposer.pubkey, - // message_hash=hash_tree_root(get_current_epoch(state)), signature=block.randao_reveal, - // domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_RANDAO))`. + // Verify the RANDAO is a valid signature of the proposer. verify!( - block.randao_reveal.verify( + block.body.randao_reveal.verify( &state.current_epoch(spec).hash_tree_root()[..], spec.get_domain( block.slot.epoch(spec.slots_per_epoch), @@ -152,21 +163,23 @@ pub fn process_randao( Invalid::BadRandaoSignature ); - // Update the state's RANDAO mix with the one revealed in the block. - update_randao(state, &block.randao_reveal, spec)?; + // Update the current epoch RANDAO mix. + state.update_randao_mix(state.current_epoch(spec), &block.body.randao_reveal, spec)?; Ok(()) } /// Update the `state.eth1_data_votes` based upon the `eth1_data` provided. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn process_eth1_data(state: &mut BeaconState, eth1_data: &Eth1Data) -> Result<(), Error> { - // Either increment the eth1_data vote count, or add a new eth1_data. + // Attempt to find a `Eth1DataVote` with matching `Eth1Data`. let matching_eth1_vote_index = state .eth1_data_votes .iter() .position(|vote| vote.eth1_data == *eth1_data); + + // If a vote exists, increment it's `vote_count`. Otherwise, create a new `Eth1DataVote`. if let Some(index) = matching_eth1_vote_index { state.eth1_data_votes[index].vote_count += 1; } else { @@ -179,46 +192,12 @@ pub fn process_eth1_data(state: &mut BeaconState, eth1_data: &Eth1Data) -> Resul Ok(()) } -/// Updates the present randao mix. -/// -/// Set `state.latest_randao_mixes[get_current_epoch(state) % LATEST_RANDAO_MIXES_LENGTH] = -/// xor(get_randao_mix(state, get_current_epoch(state)), hash(block.randao_reveal))`. -/// -/// Spec v0.4.0 -pub fn update_randao( - state: &mut BeaconState, - reveal: &Signature, - spec: &ChainSpec, -) -> Result<(), BeaconStateError> { - let hashed_reveal = { - let encoded_signature = ssz_encode(reveal); - Hash256::from_slice(&hash(&encoded_signature[..])[..]) - }; - - let current_epoch = state.slot.epoch(spec.slots_per_epoch); - - let current_mix = state - .get_randao_mix(current_epoch, spec) - .ok_or_else(|| BeaconStateError::InsufficientRandaoMixes)?; - - let new_mix = *current_mix ^ hashed_reveal; - - let index = current_epoch.as_usize() % spec.latest_randao_mixes_length; - - if index < state.latest_randao_mixes.len() { - state.latest_randao_mixes[index] = new_mix; - Ok(()) - } else { - Err(BeaconStateError::InsufficientRandaoMixes) - } -} - /// Validates each `ProposerSlashing` and updates the state, short-circuiting on an invalid object. /// /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns /// an `Err` describing the invalid object or cause of failure. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn process_proposer_slashings( state: &mut BeaconState, proposer_slashings: &[ProposerSlashing], @@ -228,9 +207,18 @@ pub fn process_proposer_slashings( proposer_slashings.len() as u64 <= spec.max_proposer_slashings, Invalid::MaxProposerSlashingsExceeded ); - for (i, proposer_slashing) in proposer_slashings.iter().enumerate() { - verify_proposer_slashing(proposer_slashing, &state, spec) - .map_err(|e| e.into_with_index(i))?; + + // Verify proposer slashings in parallel. + proposer_slashings + .par_iter() + .enumerate() + .try_for_each(|(i, proposer_slashing)| { + verify_proposer_slashing(proposer_slashing, &state, spec) + .map_err(|e| e.into_with_index(i)) + })?; + + // Update the state. + for proposer_slashing in proposer_slashings { state.slash_validator(proposer_slashing.proposer_index as usize, spec)?; } @@ -242,7 +230,7 @@ pub fn process_proposer_slashings( /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns /// an `Err` describing the invalid object or cause of failure. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn process_attester_slashings( state: &mut BeaconState, attester_slashings: &[AttesterSlashing], @@ -252,9 +240,41 @@ pub fn process_attester_slashings( attester_slashings.len() as u64 <= spec.max_attester_slashings, Invalid::MaxAttesterSlashingsExceed ); + + // Verify the `SlashableAttestation`s in parallel (these are the resource-consuming objects, not + // the `AttesterSlashing`s themselves). + let mut slashable_attestations: Vec<&SlashableAttestation> = + Vec::with_capacity(attester_slashings.len() * 2); + for attester_slashing in attester_slashings { + slashable_attestations.push(&attester_slashing.slashable_attestation_1); + slashable_attestations.push(&attester_slashing.slashable_attestation_2); + } + + // Verify slashable attestations in parallel. + slashable_attestations + .par_iter() + .enumerate() + .try_for_each(|(i, slashable_attestation)| { + verify_slashable_attestation(&state, slashable_attestation, spec) + .map_err(|e| e.into_with_index(i)) + })?; + let all_slashable_attestations_have_been_checked = true; + + // Gather the slashable indices and preform the final verification and update the state in series. for (i, attester_slashing) in attester_slashings.iter().enumerate() { - let slashable_indices = verify_attester_slashing(&state, &attester_slashing, spec) + let should_verify_slashable_attestations = !all_slashable_attestations_have_been_checked; + + verify_attester_slashing( + &state, + &attester_slashing, + should_verify_slashable_attestations, + spec, + ) + .map_err(|e| e.into_with_index(i))?; + + let slashable_indices = gather_attester_slashing_indices(&state, &attester_slashing, spec) .map_err(|e| e.into_with_index(i))?; + for i in slashable_indices { state.slash_validator(i as usize, spec)?; } @@ -268,7 +288,7 @@ pub fn process_attester_slashings( /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns /// an `Err` describing the invalid object or cause of failure. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn process_attestations( state: &mut BeaconState, attestations: &[Attestation], @@ -278,21 +298,34 @@ pub fn process_attestations( attestations.len() as u64 <= spec.max_attestations, Invalid::MaxAttestationsExceeded ); - for (i, attestation) in attestations.iter().enumerate() { - // Build the previous epoch cache only if required by an attestation. - if attestation.data.slot.epoch(spec.slots_per_epoch) == state.previous_epoch(spec) { - state.build_epoch_cache(RelativeEpoch::Previous, spec)?; - } - validate_attestation(state, attestation, spec).map_err(|e| e.into_with_index(i))?; + // Ensure the previous epoch cache exists. + state.build_epoch_cache(RelativeEpoch::Previous, spec)?; + // Verify attestations in parallel. + attestations + .par_iter() + .enumerate() + .try_for_each(|(i, attestation)| { + validate_attestation(state, attestation, spec).map_err(|e| e.into_with_index(i)) + })?; + + // Update the state in series. + for attestation in attestations { let pending_attestation = PendingAttestation { data: attestation.data.clone(), aggregation_bitfield: attestation.aggregation_bitfield.clone(), custody_bitfield: attestation.custody_bitfield.clone(), inclusion_slot: state.slot, }; - state.latest_attestations.push(pending_attestation); + + let attestation_epoch = attestation.data.slot.epoch(spec.slots_per_epoch); + + if attestation_epoch == state.current_epoch(spec) { + state.current_epoch_attestations.push(pending_attestation) + } else if attestation_epoch == state.previous_epoch(spec) { + state.previous_epoch_attestations.push(pending_attestation) + } } Ok(()) @@ -303,7 +336,7 @@ pub fn process_attestations( /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns /// an `Err` describing the invalid object or cause of failure. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn process_deposits( state: &mut BeaconState, deposits: &[Deposit], @@ -313,24 +346,54 @@ pub fn process_deposits( deposits.len() as u64 <= spec.max_deposits, Invalid::MaxDepositsExceeded ); - for (i, deposit) in deposits.iter().enumerate() { - verify_deposit(state, deposit, VERIFY_DEPOSIT_MERKLE_PROOFS, spec) - .map_err(|e| e.into_with_index(i))?; - state - .process_deposit( - deposit.deposit_data.deposit_input.pubkey.clone(), - deposit.deposit_data.amount, - deposit - .deposit_data - .deposit_input - .proof_of_possession - .clone(), - deposit.deposit_data.deposit_input.withdrawal_credentials, - None, - spec, - ) - .map_err(|_| Error::Invalid(Invalid::DepositProcessingFailed(i)))?; + // Verify deposits in parallel. + deposits + .par_iter() + .enumerate() + .try_for_each(|(i, deposit)| { + verify_deposit(state, deposit, VERIFY_DEPOSIT_MERKLE_PROOFS, spec) + .map_err(|e| e.into_with_index(i)) + })?; + + // Check `state.deposit_index` and update the state in series. + for (i, deposit) in deposits.iter().enumerate() { + verify_deposit_index(state, deposit).map_err(|e| e.into_with_index(i))?; + + // Ensure the state's pubkey cache is fully up-to-date, it will be used to check to see if the + // depositing validator already exists in the registry. + state.update_pubkey_cache()?; + + // Get an `Option` where `u64` is the validator index if this deposit public key + // already exists in the beacon_state. + // + // This function also verifies the withdrawal credentials. + let validator_index = + get_existing_validator_index(state, deposit).map_err(|e| e.into_with_index(i))?; + + let deposit_data = &deposit.deposit_data; + let deposit_input = &deposit.deposit_data.deposit_input; + + if let Some(index) = validator_index { + // Update the existing validator balance. + safe_add_assign!( + state.validator_balances[index as usize], + deposit_data.amount + ); + } else { + // Create a new validator. + let validator = Validator { + pubkey: deposit_input.pubkey.clone(), + withdrawal_credentials: deposit_input.withdrawal_credentials.clone(), + activation_epoch: spec.far_future_epoch, + exit_epoch: spec.far_future_epoch, + withdrawable_epoch: spec.far_future_epoch, + initiated_exit: false, + slashed: false, + }; + state.validator_registry.push(validator); + state.validator_balances.push(deposit_data.amount); + } state.deposit_index += 1; } @@ -343,7 +406,7 @@ pub fn process_deposits( /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns /// an `Err` describing the invalid object or cause of failure. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn process_exits( state: &mut BeaconState, voluntary_exits: &[VoluntaryExit], @@ -353,9 +416,17 @@ pub fn process_exits( voluntary_exits.len() as u64 <= spec.max_voluntary_exits, Invalid::MaxExitsExceeded ); - for (i, exit) in voluntary_exits.iter().enumerate() { - verify_exit(&state, exit, spec).map_err(|e| e.into_with_index(i))?; + // Verify exits in parallel. + voluntary_exits + .par_iter() + .enumerate() + .try_for_each(|(i, exit)| { + verify_exit(&state, exit, spec).map_err(|e| e.into_with_index(i)) + })?; + + // Update the state in series. + for exit in voluntary_exits { state.initiate_validator_exit(exit.validator_index as usize); } @@ -367,7 +438,7 @@ pub fn process_exits( /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns /// an `Err` describing the invalid object or cause of failure. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn process_transfers( state: &mut BeaconState, transfers: &[Transfer], @@ -377,8 +448,15 @@ pub fn process_transfers( transfers.len() as u64 <= spec.max_transfers, Invalid::MaxTransfersExceed ); + + transfers + .par_iter() + .enumerate() + .try_for_each(|(i, transfer)| { + verify_transfer(&state, transfer, spec).map_err(|e| e.into_with_index(i)) + })?; + for (i, transfer) in transfers.iter().enumerate() { - verify_transfer(&state, transfer, spec).map_err(|e| e.into_with_index(i))?; execute_transfer(state, transfer, spec).map_err(|e| e.into_with_index(i))?; } diff --git a/eth2/state_processing/src/per_block_processing/errors.rs b/eth2/state_processing/src/per_block_processing/errors.rs index b97d8bacc..c0fe252de 100644 --- a/eth2/state_processing/src/per_block_processing/errors.rs +++ b/eth2/state_processing/src/per_block_processing/errors.rs @@ -67,6 +67,7 @@ impl_from_beacon_state_error!(BlockProcessingError); #[derive(Debug, PartialEq)] pub enum BlockInvalid { StateSlotMismatch, + ParentBlockRootMismatch, BadSignature, BadRandaoSignature, MaxAttestationsExceeded, @@ -76,6 +77,10 @@ pub enum BlockInvalid { MaxExitsExceeded, MaxTransfersExceed, AttestationInvalid(usize, AttestationInvalid), + /// A `SlashableAttestation` inside an `AttesterSlashing` was invalid. + /// + /// To determine the offending `AttesterSlashing` index, divide the error message `usize` by two. + SlashableAttestationInvalid(usize, SlashableAttestationInvalid), AttesterSlashingInvalid(usize, AttesterSlashingInvalid), ProposerSlashingInvalid(usize, ProposerSlashingInvalid), DepositInvalid(usize, DepositInvalid), @@ -108,45 +113,55 @@ pub enum AttestationValidationError { #[derive(Debug, PartialEq)] pub enum AttestationInvalid { /// Attestation references a pre-genesis slot. - /// - /// (genesis_slot, attestation_slot) - PreGenesis(Slot, Slot), + PreGenesis { genesis: Slot, attestation: Slot }, /// Attestation included before the inclusion delay. - /// - /// (state_slot, inclusion_delay, attestation_slot) - IncludedTooEarly(Slot, u64, Slot), + IncludedTooEarly { + state: Slot, + delay: u64, + attestation: Slot, + }, /// Attestation slot is too far in the past to be included in a block. - /// - /// (state_slot, attestation_slot) - IncludedTooLate(Slot, Slot), + IncludedTooLate { state: Slot, attestation: Slot }, /// Attestation justified epoch does not match the states current or previous justified epoch. /// - /// (attestation_justified_epoch, state_epoch, used_previous_epoch) - WrongJustifiedEpoch(Epoch, Epoch, bool), + /// `is_current` is `true` if the attestation was compared to the + /// `state.current_justified_epoch`, `false` if compared to `state.previous_justified_epoch`. + WrongJustifiedEpoch { + state: Epoch, + attestation: Epoch, + is_current: bool, + }, /// Attestation justified epoch root does not match root known to the state. /// - /// (state_justified_root, attestation_justified_root) - WrongJustifiedRoot(Hash256, Hash256), + /// `is_current` is `true` if the attestation was compared to the + /// `state.current_justified_epoch`, `false` if compared to `state.previous_justified_epoch`. + WrongJustifiedRoot { + state: Hash256, + attestation: Hash256, + is_current: bool, + }, /// Attestation crosslink root does not match the state crosslink root for the attestations /// slot. - BadLatestCrosslinkRoot, + BadPreviousCrosslink, /// The custody bitfield has some bits set `true`. This is not allowed in phase 0. CustodyBitfieldHasSetBits, /// There are no set bits on the attestation -- an attestation must be signed by at least one /// validator. AggregationBitfieldIsEmpty, /// The custody bitfield length is not the smallest possible size to represent the committee. - /// - /// (committee_len, bitfield_len) - BadCustodyBitfieldLength(usize, usize), + BadCustodyBitfieldLength { + committee_len: usize, + bitfield_len: usize, + }, /// The aggregation bitfield length is not the smallest possible size to represent the committee. - /// - /// (committee_len, bitfield_len) - BadAggregationBitfieldLength(usize, usize), - /// There was no known committee for the given shard in the given slot. - /// - /// (attestation_data_shard, attestation_data_slot) - NoCommitteeForShard(u64, Slot), + BadAggregationBitfieldLength { + committee_len: usize, + bitfield_len: usize, + }, + /// There was no known committee in this `epoch` for the given shard and slot. + NoCommitteeForShard { shard: u64, slot: Slot }, + /// The validator index was unknown. + UnknownValidator(u64), /// The attestation signature verification failed. BadSignature, /// The shard block root was not set to zero. This is a phase 0 requirement. @@ -182,6 +197,8 @@ pub enum AttesterSlashingInvalid { SlashableAttestation2Invalid(SlashableAttestationInvalid), /// The validator index is unknown. One cannot slash one who does not exist. UnknownValidator(u64), + /// The specified validator has already been withdrawn. + ValidatorAlreadyWithdrawn(u64), /// There were no indices able to be slashed. NoSlashableIndices, } @@ -233,6 +250,11 @@ impl Into for SlashableAttestationValidationError { } } +impl_into_with_index_without_beacon_error!( + SlashableAttestationValidationError, + SlashableAttestationInvalid +); + /* * `ProposerSlashing` Validation */ @@ -253,16 +275,12 @@ pub enum ProposerSlashingInvalid { /// /// (proposal_1_slot, proposal_2_slot) ProposalSlotMismatch(Slot, Slot), - /// The two proposal have different shards. - /// - /// (proposal_1_shard, proposal_2_shard) - ProposalShardMismatch(u64, u64), - /// The two proposal have different block roots. - /// - /// (proposal_1_root, proposal_2_root) - ProposalBlockRootMismatch(Hash256, Hash256), + /// The proposals are identical and therefore not slashable. + ProposalsIdentical, /// The specified proposer has already been slashed. ProposerAlreadySlashed, + /// The specified proposer has already been withdrawn. + ProposerAlreadyWithdrawn(u64), /// The first proposal signature was invalid. BadProposal1Signature, /// The second proposal signature was invalid. @@ -283,21 +301,27 @@ impl_into_with_index_without_beacon_error!( pub enum DepositValidationError { /// Validation completed successfully and the object is invalid. Invalid(DepositInvalid), + /// Encountered a `BeaconStateError` whilst attempting to determine validity. + BeaconStateError(BeaconStateError), } /// Describes why an object is invalid. #[derive(Debug, PartialEq)] pub enum DepositInvalid { /// The deposit index does not match the state index. - /// - /// (state_index, deposit_index) - BadIndex(u64, u64), + BadIndex { state: u64, deposit: u64 }, + /// The proof-of-possession does not match the given pubkey. + BadProofOfPossession, + /// The withdrawal credentials for the depositing validator did not match the withdrawal + /// credentials of an existing validator with the same public key. + BadWithdrawalCredentials, /// The specified `branch` and `index` did not form a valid proof that the deposit is included /// in the eth1 deposit root. BadMerkleProof, } -impl_into_with_index_without_beacon_error!(DepositValidationError, DepositInvalid); +impl_from_beacon_state_error!(DepositValidationError); +impl_into_with_index_with_beacon_error!(DepositValidationError, DepositInvalid); /* * `Exit` Validation @@ -315,11 +339,14 @@ pub enum ExitValidationError { pub enum ExitInvalid { /// The specified validator is not in the state's validator registry. ValidatorUnknown(u64), - AlreadyExited, + /// The specified validator has a non-maximum exit epoch. + AlreadyExited(u64), + /// The specified validator has already initiated exit. + AlreadyInitiatedExited(u64), /// The exit is for a future epoch. - /// - /// (state_epoch, exit_epoch) - FutureEpoch(Epoch, Epoch), + FutureEpoch { state: Epoch, exit: Epoch }, + /// The validator has not been active for long enough. + TooYoungToLeave { lifespan: Epoch, expected: u64 }, /// The exit signature was not signed by the validator. BadSignature, } diff --git a/eth2/state_processing/src/per_block_processing/validate_attestation.rs b/eth2/state_processing/src/per_block_processing/validate_attestation.rs index 54bd2d332..272eeb18b 100644 --- a/eth2/state_processing/src/per_block_processing/validate_attestation.rs +++ b/eth2/state_processing/src/per_block_processing/validate_attestation.rs @@ -8,7 +8,7 @@ use types::*; /// /// Returns `Ok(())` if the `Attestation` is valid, otherwise indicates the reason for invalidity. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn validate_attestation( state: &BeaconState, attestation: &Attestation, @@ -22,7 +22,7 @@ pub fn validate_attestation( /// /// Returns `Ok(())` if the `Attestation` is valid, otherwise indicates the reason for invalidity. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn validate_attestation_without_signature( state: &BeaconState, attestation: &Attestation, @@ -35,74 +35,83 @@ pub fn validate_attestation_without_signature( /// given state, optionally validating the aggregate signature. /// /// -/// Spec v0.4.0 +/// Spec v0.5.0 fn validate_attestation_signature_optional( state: &BeaconState, attestation: &Attestation, spec: &ChainSpec, verify_signature: bool, ) -> Result<(), Error> { - // Verify that `attestation.data.slot >= GENESIS_SLOT`. + let state_epoch = state.slot.epoch(spec.slots_per_epoch); + let attestation_epoch = attestation.data.slot.epoch(spec.slots_per_epoch); + + // Can't submit pre-historic attestations. verify!( attestation.data.slot >= spec.genesis_slot, - Invalid::PreGenesis(spec.genesis_slot, attestation.data.slot) + Invalid::PreGenesis { + genesis: spec.genesis_slot, + attestation: attestation.data.slot + } ); - // Verify that `attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot`. + // Can't submit attestations too far in history. + verify!( + state.slot <= attestation.data.slot + spec.slots_per_epoch, + Invalid::IncludedTooLate { + state: spec.genesis_slot, + attestation: attestation.data.slot + } + ); + + // Can't submit attestation too quickly. verify!( attestation.data.slot + spec.min_attestation_inclusion_delay <= state.slot, - Invalid::IncludedTooEarly( - state.slot, - spec.min_attestation_inclusion_delay, - attestation.data.slot - ) + Invalid::IncludedTooEarly { + state: state.slot, + delay: spec.min_attestation_inclusion_delay, + attestation: attestation.data.slot + } ); - // Verify that `state.slot < attestation.data.slot + SLOTS_PER_EPOCH`. - verify!( - state.slot < attestation.data.slot + spec.slots_per_epoch, - Invalid::IncludedTooLate(state.slot, attestation.data.slot) - ); - - // Verify that `attestation.data.justified_epoch` is equal to `state.justified_epoch` if - // `slot_to_epoch(attestation.data.slot + 1) >= get_current_epoch(state) else - // state.previous_justified_epoch`. - if (attestation.data.slot + 1).epoch(spec.slots_per_epoch) >= state.current_epoch(spec) { + // Verify the justified epoch and root is correct. + if attestation_epoch >= state_epoch { verify!( - attestation.data.justified_epoch == state.justified_epoch, - Invalid::WrongJustifiedEpoch( - attestation.data.justified_epoch, - state.justified_epoch, - false - ) + attestation.data.source_epoch == state.current_justified_epoch, + Invalid::WrongJustifiedEpoch { + state: state.current_justified_epoch, + attestation: attestation.data.source_epoch, + is_current: true, + } + ); + verify!( + attestation.data.source_root == state.current_justified_root, + Invalid::WrongJustifiedRoot { + state: state.current_justified_root, + attestation: attestation.data.source_root, + is_current: true, + } ); } else { verify!( - attestation.data.justified_epoch == state.previous_justified_epoch, - Invalid::WrongJustifiedEpoch( - attestation.data.justified_epoch, - state.previous_justified_epoch, - true - ) + attestation.data.source_epoch == state.previous_justified_epoch, + Invalid::WrongJustifiedEpoch { + state: state.previous_justified_epoch, + attestation: attestation.data.source_epoch, + is_current: false, + } + ); + verify!( + attestation.data.source_root == state.previous_justified_root, + Invalid::WrongJustifiedRoot { + state: state.previous_justified_root, + attestation: attestation.data.source_root, + is_current: true, + } ); } - // Verify that `attestation.data.justified_block_root` is equal to `get_block_root(state, - // get_epoch_start_slot(attestation.data.justified_epoch))`. - let justified_block_root = *state - .get_block_root( - attestation - .data - .justified_epoch - .start_slot(spec.slots_per_epoch), - &spec, - ) - .ok_or(BeaconStateError::InsufficientBlockRoots)?; - verify!( - attestation.data.justified_block_root == justified_block_root, - Invalid::WrongJustifiedRoot(justified_block_root, attestation.data.justified_block_root) - ); - + // Check that the crosslink data is valid. + // // Verify that either: // // (i)`state.latest_crosslinks[attestation.data.shard] == attestation.data.latest_crosslink`, @@ -115,65 +124,72 @@ fn validate_attestation_signature_optional( epoch: attestation.data.slot.epoch(spec.slots_per_epoch), }; verify!( - (attestation.data.latest_crosslink + (attestation.data.previous_crosslink == state.latest_crosslinks[attestation.data.shard as usize]) | (state.latest_crosslinks[attestation.data.shard as usize] == potential_crosslink), - Invalid::BadLatestCrosslinkRoot + Invalid::BadPreviousCrosslink ); - // Get the committee for this attestation - let (committee, _shard) = state - .get_crosslink_committees_at_slot(attestation.data.slot, spec)? - .iter() - .find(|(_committee, shard)| *shard == attestation.data.shard) - .ok_or_else(|| { - Error::Invalid(Invalid::NoCommitteeForShard( - attestation.data.shard, - attestation.data.slot, - )) - })?; - - // Custody bitfield is all zeros (phase 0 requirement). - verify!( - attestation.custody_bitfield.num_set_bits() == 0, - Invalid::CustodyBitfieldHasSetBits - ); - // Custody bitfield length is correct. - verify!( - verify_bitfield_length(&attestation.custody_bitfield, committee.len()), - Invalid::BadCustodyBitfieldLength(committee.len(), attestation.custody_bitfield.len()) - ); - // Aggregation bitfield isn't empty. + // Attestation must be non-empty! verify!( attestation.aggregation_bitfield.num_set_bits() != 0, Invalid::AggregationBitfieldIsEmpty ); + // Custody bitfield must be empty (be be removed in phase 1) + verify!( + attestation.custody_bitfield.num_set_bits() == 0, + Invalid::CustodyBitfieldHasSetBits + ); + + // Get the committee for the specific shard that this attestation is for. + let crosslink_committee = state + .get_crosslink_committees_at_slot(attestation.data.slot, spec)? + .iter() + .find(|c| c.shard == attestation.data.shard) + .ok_or_else(|| { + Error::Invalid(Invalid::NoCommitteeForShard { + shard: attestation.data.shard, + slot: attestation.data.slot, + }) + })?; + let committee = &crosslink_committee.committee; + + // Custody bitfield length is correct. + // + // This is not directly in the spec, but it is inferred. + verify!( + verify_bitfield_length(&attestation.custody_bitfield, committee.len()), + Invalid::BadCustodyBitfieldLength { + committee_len: committee.len(), + bitfield_len: attestation.custody_bitfield.len() + } + ); // Aggregation bitfield length is correct. + // + // This is not directly in the spec, but it is inferred. verify!( verify_bitfield_length(&attestation.aggregation_bitfield, committee.len()), - Invalid::BadAggregationBitfieldLength( - committee.len(), - attestation.aggregation_bitfield.len() - ) + Invalid::BadAggregationBitfieldLength { + committee_len: committee.len(), + bitfield_len: attestation.custody_bitfield.len() + } ); if verify_signature { let attestation_epoch = attestation.data.slot.epoch(spec.slots_per_epoch); - verify!( - verify_attestation_signature( - state, - committee, - attestation_epoch, - &attestation.custody_bitfield, - &attestation.data, - &attestation.aggregate_signature, - spec - ), - Invalid::BadSignature - ); + verify_attestation_signature( + state, + committee, + attestation_epoch, + &attestation.aggregation_bitfield, + &attestation.custody_bitfield, + &attestation.data, + &attestation.aggregate_signature, + spec, + )?; } - // [TO BE REMOVED IN PHASE 1] Verify that `attestation.data.crosslink_data_root == ZERO_HASH`. + // Crosslink data root is zero (to be removed in phase 1). verify!( attestation.data.crosslink_data_root == spec.zero_hash, Invalid::ShardBlockRootNotZero @@ -190,35 +206,50 @@ fn validate_attestation_signature_optional( /// - `custody_bitfield` does not have a bit for each index of `committee`. /// - A `validator_index` in `committee` is not in `state.validator_registry`. /// -/// Spec v0.4.0 +/// Spec v0.5.0 fn verify_attestation_signature( state: &BeaconState, committee: &[usize], attestation_epoch: Epoch, + aggregation_bitfield: &Bitfield, custody_bitfield: &Bitfield, attestation_data: &AttestationData, aggregate_signature: &AggregateSignature, spec: &ChainSpec, -) -> bool { +) -> Result<(), Error> { let mut aggregate_pubs = vec![AggregatePublicKey::new(); 2]; let mut message_exists = vec![false; 2]; for (i, v) in committee.iter().enumerate() { - let custody_bit = match custody_bitfield.get(i) { - Ok(bit) => bit, - // Invalidate signature if custody_bitfield.len() < committee - Err(_) => return false, - }; + let validator_signed = aggregation_bitfield.get(i).map_err(|_| { + Error::Invalid(Invalid::BadAggregationBitfieldLength { + committee_len: committee.len(), + bitfield_len: aggregation_bitfield.len(), + }) + })?; - message_exists[custody_bit as usize] = true; + if validator_signed { + let custody_bit: bool = match custody_bitfield.get(i) { + Ok(bit) => bit, + // Invalidate signature if custody_bitfield.len() < committee + Err(_) => { + return Err(Error::Invalid(Invalid::BadCustodyBitfieldLength { + committee_len: committee.len(), + bitfield_len: aggregation_bitfield.len(), + })); + } + }; - match state.validator_registry.get(*v as usize) { - Some(validator) => { - aggregate_pubs[custody_bit as usize].add(&validator.pubkey); - } - // Invalidate signature if validator index is unknown. - None => return false, - }; + message_exists[custody_bit as usize] = true; + + match state.validator_registry.get(*v as usize) { + Some(validator) => { + aggregate_pubs[custody_bit as usize].add(&validator.pubkey); + } + // Return error if validator index is unknown. + None => return Err(Error::BeaconStateError(BeaconStateError::UnknownValidator)), + }; + } } // Message when custody bitfield is `false` @@ -251,5 +282,10 @@ fn verify_attestation_signature( let domain = spec.get_domain(attestation_epoch, Domain::Attestation, &state.fork); - aggregate_signature.verify_multiple(&messages[..], domain, &keys[..]) + verify!( + aggregate_signature.verify_multiple(&messages[..], domain, &keys[..]), + Invalid::BadSignature + ); + + Ok(()) } diff --git a/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs b/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs index 71ac97469..a198d2a3e 100644 --- a/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs +++ b/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs @@ -7,12 +7,13 @@ use types::*; /// /// Returns `Ok(())` if the `AttesterSlashing` is valid, otherwise indicates the reason for invalidity. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn verify_attester_slashing( state: &BeaconState, attester_slashing: &AttesterSlashing, + should_verify_slashable_attestations: bool, spec: &ChainSpec, -) -> Result, Error> { +) -> Result<(), Error> { let slashable_attestation_1 = &attester_slashing.slashable_attestation_1; let slashable_attestation_2 = &attester_slashing.slashable_attestation_2; @@ -26,24 +27,51 @@ pub fn verify_attester_slashing( Invalid::NotSlashable ); - verify_slashable_attestation(state, &slashable_attestation_1, spec) - .map_err(|e| Error::Invalid(Invalid::SlashableAttestation1Invalid(e.into())))?; - verify_slashable_attestation(state, &slashable_attestation_2, spec) - .map_err(|e| Error::Invalid(Invalid::SlashableAttestation2Invalid(e.into())))?; + if should_verify_slashable_attestations { + verify_slashable_attestation(state, &slashable_attestation_1, spec) + .map_err(|e| Error::Invalid(Invalid::SlashableAttestation1Invalid(e.into())))?; + verify_slashable_attestation(state, &slashable_attestation_2, spec) + .map_err(|e| Error::Invalid(Invalid::SlashableAttestation2Invalid(e.into())))?; + } - let mut slashable_indices = vec![]; + Ok(()) +} + +/// For a given attester slashing, return the indices able to be slashed. +/// +/// Returns Ok(indices) if `indices.len() > 0`. +/// +/// Spec v0.5.0 +pub fn gather_attester_slashing_indices( + state: &BeaconState, + attester_slashing: &AttesterSlashing, + spec: &ChainSpec, +) -> Result, Error> { + let slashable_attestation_1 = &attester_slashing.slashable_attestation_1; + let slashable_attestation_2 = &attester_slashing.slashable_attestation_2; + + let mut slashable_indices = Vec::with_capacity(spec.max_indices_per_slashable_vote); for i in &slashable_attestation_1.validator_indices { let validator = state .validator_registry .get(*i as usize) .ok_or_else(|| Error::Invalid(Invalid::UnknownValidator(*i)))?; - if slashable_attestation_1.validator_indices.contains(&i) & !validator.slashed { + if slashable_attestation_2.validator_indices.contains(&i) & !validator.slashed { + // TODO: verify that we should reject any slashable attestation which includes a + // withdrawn validator. PH has asked the question on gitter, awaiting response. + verify!( + validator.withdrawable_epoch > state.slot.epoch(spec.slots_per_epoch), + Invalid::ValidatorAlreadyWithdrawn(*i) + ); + slashable_indices.push(*i); } } verify!(!slashable_indices.is_empty(), Invalid::NoSlashableIndices); + slashable_indices.shrink_to_fit(); + Ok(slashable_indices) } diff --git a/eth2/state_processing/src/per_block_processing/verify_deposit.rs b/eth2/state_processing/src/per_block_processing/verify_deposit.rs index 69dae1533..80d8bc24f 100644 --- a/eth2/state_processing/src/per_block_processing/verify_deposit.rs +++ b/eth2/state_processing/src/per_block_processing/verify_deposit.rs @@ -10,9 +10,12 @@ use types::*; /// /// Returns `Ok(())` if the `Deposit` is valid, otherwise indicates the reason for invalidity. /// +/// This function _does not_ check `state.deposit_index` so this function may be run in parallel. +/// See the `verify_deposit_index` function for this. +/// /// Note: this function is incomplete. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn verify_deposit( state: &BeaconState, deposit: &Deposit, @@ -20,8 +23,15 @@ pub fn verify_deposit( spec: &ChainSpec, ) -> Result<(), Error> { verify!( - deposit.index == state.deposit_index, - Invalid::BadIndex(state.deposit_index, deposit.index) + deposit + .deposit_data + .deposit_input + .validate_proof_of_possession( + state.slot.epoch(spec.slots_per_epoch), + &state.fork, + spec + ), + Invalid::BadProofOfPossession ); if verify_merkle_branch { @@ -34,14 +44,58 @@ pub fn verify_deposit( Ok(()) } +/// Verify that the `Deposit` index is correct. +/// +/// Spec v0.5.0 +pub fn verify_deposit_index(state: &BeaconState, deposit: &Deposit) -> Result<(), Error> { + verify!( + deposit.index == state.deposit_index, + Invalid::BadIndex { + state: state.deposit_index, + deposit: deposit.index + } + ); + + Ok(()) +} + +/// Returns a `Some(validator index)` if a pubkey already exists in the `validator_registry`, +/// otherwise returns `None`. +/// +/// ## Errors +/// +/// Errors if the state's `pubkey_cache` is not current. +pub fn get_existing_validator_index( + state: &BeaconState, + deposit: &Deposit, +) -> Result, Error> { + let deposit_input = &deposit.deposit_data.deposit_input; + + let validator_index = state + .get_validator_index(&deposit_input.pubkey)? + .and_then(|i| Some(i)); + + match validator_index { + None => Ok(None), + Some(index) => { + verify!( + deposit_input.withdrawal_credentials + == state.validator_registry[index as usize].withdrawal_credentials, + Invalid::BadWithdrawalCredentials + ); + Ok(Some(index as u64)) + } + } +} + /// Verify that a deposit is included in the state's eth1 deposit root. /// -/// Spec v0.4.0 +/// Spec v0.5.0 fn verify_deposit_merkle_proof(state: &BeaconState, deposit: &Deposit, spec: &ChainSpec) -> bool { let leaf = hash(&get_serialized_deposit_data(deposit)); verify_merkle_proof( Hash256::from_slice(&leaf), - &deposit.branch, + &deposit.proof, spec.deposit_contract_tree_depth as usize, deposit.index as usize, state.latest_eth1_data.deposit_root, @@ -50,7 +104,7 @@ fn verify_deposit_merkle_proof(state: &BeaconState, deposit: &Deposit, spec: &Ch /// Helper struct for easily getting the serialized data generated by the deposit contract. /// -/// Spec v0.4.0 +/// Spec v0.5.0 #[derive(Encode)] struct SerializedDepositData { amount: u64, @@ -61,7 +115,7 @@ struct SerializedDepositData { /// Return the serialized data generated by the deposit contract that is used to generate the /// merkle proof. /// -/// Spec v0.4.0 +/// Spec v0.5.0 fn get_serialized_deposit_data(deposit: &Deposit) -> Vec { let serialized_deposit_data = SerializedDepositData { amount: deposit.deposit_data.amount, diff --git a/eth2/state_processing/src/per_block_processing/verify_exit.rs b/eth2/state_processing/src/per_block_processing/verify_exit.rs index 8cd54fb69..7893cea96 100644 --- a/eth2/state_processing/src/per_block_processing/verify_exit.rs +++ b/eth2/state_processing/src/per_block_processing/verify_exit.rs @@ -7,7 +7,7 @@ use types::*; /// /// Returns `Ok(())` if the `Exit` is valid, otherwise indicates the reason for invalidity. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn verify_exit( state: &BeaconState, exit: &VoluntaryExit, @@ -18,15 +18,35 @@ pub fn verify_exit( .get(exit.validator_index as usize) .ok_or_else(|| Error::Invalid(Invalid::ValidatorUnknown(exit.validator_index)))?; + // Verify that the validator has not yet exited. verify!( - validator.exit_epoch - > state.get_delayed_activation_exit_epoch(state.current_epoch(spec), spec), - Invalid::AlreadyExited + validator.exit_epoch == spec.far_future_epoch, + Invalid::AlreadyExited(exit.validator_index) ); + // Verify that the validator has not yet initiated. + verify!( + !validator.initiated_exit, + Invalid::AlreadyInitiatedExited(exit.validator_index) + ); + + // Exits must specify an epoch when they become valid; they are not valid before then. verify!( state.current_epoch(spec) >= exit.epoch, - Invalid::FutureEpoch(state.current_epoch(spec), exit.epoch) + Invalid::FutureEpoch { + state: state.current_epoch(spec), + exit: exit.epoch + } + ); + + // Must have been in the validator set long enough. + let lifespan = state.slot.epoch(spec.slots_per_epoch) - validator.activation_epoch; + verify!( + lifespan >= spec.persistent_committee_period, + Invalid::TooYoungToLeave { + lifespan, + expected: spec.persistent_committee_period, + } ); let message = exit.signed_root(); diff --git a/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs b/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs index c3c0079a9..dffb9d898 100644 --- a/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs +++ b/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs @@ -7,7 +7,7 @@ use types::*; /// /// Returns `Ok(())` if the `ProposerSlashing` is valid, otherwise indicates the reason for invalidity. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn verify_proposer_slashing( proposer_slashing: &ProposerSlashing, state: &BeaconState, @@ -21,34 +21,28 @@ pub fn verify_proposer_slashing( })?; verify!( - proposer_slashing.proposal_1.slot == proposer_slashing.proposal_2.slot, + proposer_slashing.header_1.slot == proposer_slashing.header_2.slot, Invalid::ProposalSlotMismatch( - proposer_slashing.proposal_1.slot, - proposer_slashing.proposal_2.slot + proposer_slashing.header_1.slot, + proposer_slashing.header_2.slot ) ); verify!( - proposer_slashing.proposal_1.shard == proposer_slashing.proposal_2.shard, - Invalid::ProposalShardMismatch( - proposer_slashing.proposal_1.shard, - proposer_slashing.proposal_2.shard - ) - ); - - verify!( - proposer_slashing.proposal_1.block_root != proposer_slashing.proposal_2.block_root, - Invalid::ProposalBlockRootMismatch( - proposer_slashing.proposal_1.block_root, - proposer_slashing.proposal_2.block_root - ) + proposer_slashing.header_1 != proposer_slashing.header_2, + Invalid::ProposalsIdentical ); verify!(!proposer.slashed, Invalid::ProposerAlreadySlashed); verify!( - verify_proposal_signature( - &proposer_slashing.proposal_1, + proposer.withdrawable_epoch > state.slot.epoch(spec.slots_per_epoch), + Invalid::ProposerAlreadyWithdrawn(proposer_slashing.proposer_index) + ); + + verify!( + verify_header_signature( + &proposer_slashing.header_1, &proposer.pubkey, &state.fork, spec @@ -56,8 +50,8 @@ pub fn verify_proposer_slashing( Invalid::BadProposal1Signature ); verify!( - verify_proposal_signature( - &proposer_slashing.proposal_2, + verify_header_signature( + &proposer_slashing.header_2, &proposer.pubkey, &state.fork, spec @@ -71,17 +65,19 @@ pub fn verify_proposer_slashing( /// Verifies the signature of a proposal. /// /// Returns `true` if the signature is valid. -fn verify_proposal_signature( - proposal: &Proposal, +/// +/// Spec v0.5.0 +fn verify_header_signature( + header: &BeaconBlockHeader, pubkey: &PublicKey, fork: &Fork, spec: &ChainSpec, ) -> bool { - let message = proposal.signed_root(); + let message = header.signed_root(); let domain = spec.get_domain( - proposal.slot.epoch(spec.slots_per_epoch), - Domain::Proposal, + header.slot.epoch(spec.slots_per_epoch), + Domain::BeaconBlock, fork, ); - proposal.signature.verify(&message[..], domain, pubkey) + header.signature.verify(&message[..], domain, pubkey) } diff --git a/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs b/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs index f0d371043..aa9a32196 100644 --- a/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs +++ b/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs @@ -10,7 +10,7 @@ use types::*; /// /// Returns `Ok(())` if the `SlashableAttestation` is valid, otherwise indicates the reason for invalidity. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn verify_slashable_attestation( state: &BeaconState, slashable_attestation: &SlashableAttestation, diff --git a/eth2/state_processing/src/per_block_processing/verify_transfer.rs b/eth2/state_processing/src/per_block_processing/verify_transfer.rs index 4746fc75c..f873cd850 100644 --- a/eth2/state_processing/src/per_block_processing/verify_transfer.rs +++ b/eth2/state_processing/src/per_block_processing/verify_transfer.rs @@ -10,16 +10,16 @@ use types::*; /// /// Note: this function is incomplete. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn verify_transfer( state: &BeaconState, transfer: &Transfer, spec: &ChainSpec, ) -> Result<(), Error> { - let from_balance = *state + let sender_balance = *state .validator_balances - .get(transfer.from as usize) - .ok_or_else(|| Error::Invalid(Invalid::FromValidatorUnknown(transfer.from)))?; + .get(transfer.sender as usize) + .ok_or_else(|| Error::Invalid(Invalid::FromValidatorUnknown(transfer.sender)))?; let total_amount = transfer .amount @@ -27,19 +27,22 @@ pub fn verify_transfer( .ok_or_else(|| Error::Invalid(Invalid::FeeOverflow(transfer.amount, transfer.fee)))?; verify!( - from_balance >= transfer.amount, - Invalid::FromBalanceInsufficient(transfer.amount, from_balance) + sender_balance >= transfer.amount, + Invalid::FromBalanceInsufficient(transfer.amount, sender_balance) ); verify!( - from_balance >= transfer.fee, - Invalid::FromBalanceInsufficient(transfer.fee, from_balance) + sender_balance >= transfer.fee, + Invalid::FromBalanceInsufficient(transfer.fee, sender_balance) ); verify!( - (from_balance == total_amount) - || (from_balance >= (total_amount + spec.min_deposit_amount)), - Invalid::InvalidResultingFromBalance(from_balance - total_amount, spec.min_deposit_amount) + (sender_balance == total_amount) + || (sender_balance >= (total_amount + spec.min_deposit_amount)), + Invalid::InvalidResultingFromBalance( + sender_balance - total_amount, + spec.min_deposit_amount + ) ); verify!( @@ -47,25 +50,25 @@ pub fn verify_transfer( Invalid::StateSlotMismatch(state.slot, transfer.slot) ); - let from_validator = state + let sender_validator = state .validator_registry - .get(transfer.from as usize) - .ok_or_else(|| Error::Invalid(Invalid::FromValidatorUnknown(transfer.from)))?; + .get(transfer.sender as usize) + .ok_or_else(|| Error::Invalid(Invalid::FromValidatorUnknown(transfer.sender)))?; let epoch = state.slot.epoch(spec.slots_per_epoch); verify!( - from_validator.is_withdrawable_at(epoch) - || from_validator.activation_epoch == spec.far_future_epoch, - Invalid::FromValidatorIneligableForTransfer(transfer.from) + sender_validator.is_withdrawable_at(epoch) + || sender_validator.activation_epoch == spec.far_future_epoch, + Invalid::FromValidatorIneligableForTransfer(transfer.sender) ); let transfer_withdrawal_credentials = Hash256::from_slice( &get_withdrawal_credentials(&transfer.pubkey, spec.bls_withdrawal_prefix_byte)[..], ); verify!( - from_validator.withdrawal_credentials == transfer_withdrawal_credentials, + sender_validator.withdrawal_credentials == transfer_withdrawal_credentials, Invalid::WithdrawalCredentialsMismatch( - from_validator.withdrawal_credentials, + sender_validator.withdrawal_credentials, transfer_withdrawal_credentials ) ); @@ -91,22 +94,23 @@ pub fn verify_transfer( /// /// Does not check that the transfer is valid, however checks for overflow in all actions. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn execute_transfer( state: &mut BeaconState, transfer: &Transfer, spec: &ChainSpec, ) -> Result<(), Error> { - let from_balance = *state + let sender_balance = *state .validator_balances - .get(transfer.from as usize) - .ok_or_else(|| Error::Invalid(Invalid::FromValidatorUnknown(transfer.from)))?; - let to_balance = *state + .get(transfer.sender as usize) + .ok_or_else(|| Error::Invalid(Invalid::FromValidatorUnknown(transfer.sender)))?; + let recipient_balance = *state .validator_balances - .get(transfer.to as usize) - .ok_or_else(|| Error::Invalid(Invalid::ToValidatorUnknown(transfer.to)))?; + .get(transfer.recipient as usize) + .ok_or_else(|| Error::Invalid(Invalid::ToValidatorUnknown(transfer.recipient)))?; - let proposer_index = state.get_beacon_proposer_index(state.slot, spec)?; + let proposer_index = + state.get_beacon_proposer_index(state.slot, RelativeEpoch::Current, spec)?; let proposer_balance = state.validator_balances[proposer_index]; let total_amount = transfer @@ -114,14 +118,22 @@ pub fn execute_transfer( .checked_add(transfer.fee) .ok_or_else(|| Error::Invalid(Invalid::FeeOverflow(transfer.amount, transfer.fee)))?; - state.validator_balances[transfer.from as usize] = - from_balance.checked_sub(total_amount).ok_or_else(|| { - Error::Invalid(Invalid::FromBalanceInsufficient(total_amount, from_balance)) + state.validator_balances[transfer.sender as usize] = + sender_balance.checked_sub(total_amount).ok_or_else(|| { + Error::Invalid(Invalid::FromBalanceInsufficient( + total_amount, + sender_balance, + )) })?; - state.validator_balances[transfer.to as usize] = to_balance + state.validator_balances[transfer.recipient as usize] = recipient_balance .checked_add(transfer.amount) - .ok_or_else(|| Error::Invalid(Invalid::ToBalanceOverflow(to_balance, transfer.amount)))?; + .ok_or_else(|| { + Error::Invalid(Invalid::ToBalanceOverflow( + recipient_balance, + transfer.amount, + )) + })?; state.validator_balances[proposer_index] = proposer_balance.checked_add(transfer.fee).ok_or_else(|| { diff --git a/eth2/state_processing/src/per_epoch_processing.rs b/eth2/state_processing/src/per_epoch_processing.rs index bd8aca3c4..d1bb4269a 100644 --- a/eth2/state_processing/src/per_epoch_processing.rs +++ b/eth2/state_processing/src/per_epoch_processing.rs @@ -1,120 +1,105 @@ -use attester_sets::AttesterSets; use errors::EpochProcessingError as Error; -use inclusion_distance::{inclusion_distance, inclusion_slot}; use integer_sqrt::IntegerSquareRoot; -use log::debug; +use process_validator_registry::process_validator_registry; use rayon::prelude::*; use ssz::TreeHash; -use std::collections::{HashMap, HashSet}; -use std::iter::FromIterator; +use std::collections::HashMap; use types::{validator_registry::get_active_validator_indices, *}; +use validator_statuses::{TotalBalances, ValidatorStatuses}; use winning_root::{winning_root, WinningRoot}; -pub mod attester_sets; pub mod errors; +pub mod get_attestation_participants; pub mod inclusion_distance; +pub mod process_validator_registry; pub mod tests; +pub mod validator_statuses; pub mod winning_root; +/// Maps a shard to a winning root. +/// +/// It is generated during crosslink processing and later used to reward/penalize validators. +pub type WinningRootHashSet = HashMap; + +/// Performs per-epoch processing on some BeaconState. +/// +/// Mutates the given `BeaconState`, returning early if an error is encountered. If an error is +/// returned, a state might be "half-processed" and therefore in an invalid state. +/// +/// Spec v0.4.0 pub fn per_epoch_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> { - let current_epoch = state.current_epoch(spec); - let previous_epoch = state.previous_epoch(spec); - let next_epoch = state.next_epoch(spec); - - debug!( - "Starting per-epoch processing on epoch {}...", - state.current_epoch(spec) - ); - - // Ensure all of the caches are built. + // Ensure the previous and next epoch caches are built. state.build_epoch_cache(RelativeEpoch::Previous, spec)?; state.build_epoch_cache(RelativeEpoch::Current, spec)?; - state.build_epoch_cache(RelativeEpoch::Next, spec)?; - let attesters = AttesterSets::new(&state, spec)?; - - let active_validator_indices = get_active_validator_indices( - &state.validator_registry, - state.slot.epoch(spec.slots_per_epoch), - ); - - let current_total_balance = state.get_total_balance(&active_validator_indices[..], spec); - let previous_total_balance = state.get_total_balance( - &get_active_validator_indices(&state.validator_registry, previous_epoch)[..], - spec, - ); + let mut statuses = initialize_validator_statuses(&state, spec)?; process_eth1_data(state, spec); - process_justification( - state, - current_total_balance, - previous_total_balance, - attesters.previous_epoch_boundary.balance, - attesters.current_epoch_boundary.balance, - spec, - ); + process_justification(state, &statuses.total_balances, spec); // Crosslinks let winning_root_for_shards = process_crosslinks(state, spec)?; // Rewards and Penalities - let active_validator_indices_hashset: HashSet = - HashSet::from_iter(active_validator_indices.iter().cloned()); - process_rewards_and_penalities( - state, - active_validator_indices_hashset, - &attesters, - previous_total_balance, - &winning_root_for_shards, - spec, - )?; + process_rewards_and_penalities(state, &mut statuses, &winning_root_for_shards, spec)?; // Ejections - state.process_ejections(spec); + state.process_ejections(spec)?; // Validator Registry process_validator_registry(state, spec)?; // Final updates - let active_tree_root = get_active_validator_indices( - &state.validator_registry, - next_epoch + Epoch::from(spec.activation_exit_delay), - ) - .hash_tree_root(); - state.latest_active_index_roots[(next_epoch.as_usize() - + spec.activation_exit_delay as usize) - % spec.latest_active_index_roots_length] = Hash256::from_slice(&active_tree_root[..]); - - state.latest_slashed_balances[next_epoch.as_usize() % spec.latest_slashed_exit_length] = - state.latest_slashed_balances[current_epoch.as_usize() % spec.latest_slashed_exit_length]; - state.latest_randao_mixes[next_epoch.as_usize() % spec.latest_randao_mixes_length] = state - .get_randao_mix(current_epoch, spec) - .and_then(|x| Some(*x)) - .ok_or_else(|| Error::NoRandaoSeed)?; - state.latest_attestations = state - .latest_attestations - .iter() - .filter(|a| a.data.slot.epoch(spec.slots_per_epoch) >= current_epoch) - .cloned() - .collect(); + update_active_tree_index_roots(state, spec)?; + update_latest_slashed_balances(state, spec); + clean_attestations(state); // Rotate the epoch caches to suit the epoch transition. state.advance_caches(); - debug!("Epoch transition complete."); - Ok(()) } -/// Spec v0.4.0 -fn process_eth1_data(state: &mut BeaconState, spec: &ChainSpec) { +/// Returns a list of active validator indices for the state's current epoch. +/// +/// Spec v0.5.0 +pub fn calculate_active_validator_indices(state: &BeaconState, spec: &ChainSpec) -> Vec { + get_active_validator_indices( + &state.validator_registry, + state.slot.epoch(spec.slots_per_epoch), + ) +} + +/// Calculates various sets of attesters, including: +/// +/// - current epoch attesters +/// - current epoch boundary attesters +/// - previous epoch attesters +/// - etc. +/// +/// Spec v0.5.0 +pub fn initialize_validator_statuses( + state: &BeaconState, + spec: &ChainSpec, +) -> Result { + let mut statuses = ValidatorStatuses::new(state, spec)?; + + statuses.process_attestations(&state, spec)?; + + Ok(statuses) +} + +/// Maybe resets the eth1 period. +/// +/// Spec v0.5.0 +pub fn process_eth1_data(state: &mut BeaconState, spec: &ChainSpec) { let next_epoch = state.next_epoch(spec); let voting_period = spec.epochs_per_eth1_voting_period; if next_epoch % voting_period == 0 { for eth1_data_vote in &state.eth1_data_votes { - if eth1_data_vote.vote_count * 2 > voting_period { + if eth1_data_vote.vote_count * 2 > voting_period * spec.slots_per_epoch { state.latest_eth1_data = eth1_data_vote.eth1_data.clone(); } } @@ -122,26 +107,31 @@ fn process_eth1_data(state: &mut BeaconState, spec: &ChainSpec) { } } +/// Update the following fields on the `BeaconState`: +/// +/// - `justification_bitfield`. +/// - `finalized_epoch` +/// - `justified_epoch` +/// - `previous_justified_epoch` +/// /// Spec v0.4.0 -fn process_justification( +pub fn process_justification( state: &mut BeaconState, - current_total_balance: u64, - previous_total_balance: u64, - previous_epoch_boundary_attesting_balance: u64, - current_epoch_boundary_attesting_balance: u64, + total_balances: &TotalBalances, spec: &ChainSpec, ) { let previous_epoch = state.previous_epoch(spec); let current_epoch = state.current_epoch(spec); - let mut new_justified_epoch = state.justified_epoch; + let mut new_justified_epoch = state.current_justified_epoch; state.justification_bitfield <<= 1; // If > 2/3 of the total balance attested to the previous epoch boundary // // - Set the 2nd bit of the bitfield. // - Set the previous epoch to be justified. - if (3 * previous_epoch_boundary_attesting_balance) >= (2 * previous_total_balance) { + if (3 * total_balances.previous_epoch_boundary_attesters) >= (2 * total_balances.previous_epoch) + { state.justification_bitfield |= 2; new_justified_epoch = previous_epoch; } @@ -149,7 +139,7 @@ fn process_justification( // // - Set the 1st bit of the bitfield. // - Set the current epoch to be justified. - if (3 * current_epoch_boundary_attesting_balance) >= (2 * current_total_balance) { + if (3 * total_balances.current_epoch_boundary_attesters) >= (2 * total_balances.current_epoch) { state.justification_bitfield |= 1; new_justified_epoch = current_epoch; } @@ -182,8 +172,10 @@ fn process_justification( // - The presently justified epoch was two epochs ago. // // Then, set the finalized epoch to two epochs ago. - if (state.justification_bitfield % 8 == 0b111) & (state.justified_epoch == previous_epoch - 1) { - state.finalized_epoch = state.justified_epoch; + if (state.justification_bitfield % 8 == 0b111) + & (state.current_justified_epoch == previous_epoch - 1) + { + state.finalized_epoch = state.current_justified_epoch; } // If: // @@ -191,32 +183,27 @@ fn process_justification( // - Set the previous epoch to be justified. // // Then, set the finalized epoch to be the previous epoch. - if (state.justification_bitfield % 4 == 0b11) & (state.justified_epoch == previous_epoch) { - state.finalized_epoch = state.justified_epoch; + if (state.justification_bitfield % 4 == 0b11) + & (state.current_justified_epoch == previous_epoch) + { + state.finalized_epoch = state.current_justified_epoch; } - state.previous_justified_epoch = state.justified_epoch; - state.justified_epoch = new_justified_epoch; + state.previous_justified_epoch = state.current_justified_epoch; + state.current_justified_epoch = new_justified_epoch; } -pub type WinningRootHashSet = HashMap; - -fn process_crosslinks( +/// Updates the following fields on the `BeaconState`: +/// +/// - `latest_crosslinks` +/// +/// Also returns a `WinningRootHashSet` for later use during epoch processing. +/// +/// Spec v0.5.0 +pub fn process_crosslinks( state: &mut BeaconState, spec: &ChainSpec, ) -> Result { - let current_epoch_attestations: Vec<&PendingAttestation> = state - .latest_attestations - .par_iter() - .filter(|a| a.data.slot.epoch(spec.slots_per_epoch) == state.current_epoch(spec)) - .collect(); - - let previous_epoch_attestations: Vec<&PendingAttestation> = state - .latest_attestations - .par_iter() - .filter(|a| a.data.slot.epoch(spec.slots_per_epoch) == state.previous_epoch(spec)) - .collect(); - let mut winning_root_for_shards: WinningRootHashSet = HashMap::new(); let previous_and_current_epoch_slots: Vec = state @@ -230,24 +217,18 @@ fn process_crosslinks( let crosslink_committees_at_slot = state.get_crosslink_committees_at_slot(slot, spec)?.clone(); - for (crosslink_committee, shard) in crosslink_committees_at_slot { - let shard = shard as u64; + for c in crosslink_committees_at_slot { + let shard = c.shard as u64; - let winning_root = winning_root( - state, - shard, - ¤t_epoch_attestations[..], - &previous_epoch_attestations[..], - spec, - )?; + let winning_root = winning_root(state, shard, spec)?; if let Some(winning_root) = winning_root { - let total_committee_balance = state.get_total_balance(&crosslink_committee, spec); + let total_committee_balance = state.get_total_balance(&c.committee, spec)?; // TODO: I think this has a bug. if (3 * winning_root.total_attesting_balance) >= (2 * total_committee_balance) { state.latest_crosslinks[shard as usize] = Crosslink { - epoch: state.current_epoch(spec), + epoch: slot.epoch(spec.slots_per_epoch), crosslink_data_root: winning_root.crosslink_data_root, } } @@ -259,183 +240,161 @@ fn process_crosslinks( Ok(winning_root_for_shards) } +/// Updates the following fields on the BeaconState: +/// +/// - `validator_balances` +/// /// Spec v0.4.0 -fn process_rewards_and_penalities( +pub fn process_rewards_and_penalities( state: &mut BeaconState, - active_validator_indices: HashSet, - attesters: &AttesterSets, - previous_total_balance: u64, + statuses: &mut ValidatorStatuses, winning_root_for_shards: &WinningRootHashSet, spec: &ChainSpec, ) -> Result<(), Error> { let next_epoch = state.next_epoch(spec); - let previous_epoch_attestations: Vec<&PendingAttestation> = state - .latest_attestations - .par_iter() - .filter(|a| a.data.slot.epoch(spec.slots_per_epoch) == state.previous_epoch(spec)) - .collect(); + statuses.process_winning_roots(state, winning_root_for_shards, spec)?; - let base_reward_quotient = previous_total_balance.integer_sqrt() / spec.base_reward_quotient; + let total_balances = &statuses.total_balances; + let base_reward_quotient = + total_balances.previous_epoch.integer_sqrt() / spec.base_reward_quotient; + + // Guard against a divide-by-zero during the validator balance update. if base_reward_quotient == 0 { return Err(Error::BaseRewardQuotientIsZero); } + // Guard against a divide-by-zero during the validator balance update. + if total_balances.previous_epoch == 0 { + return Err(Error::PreviousTotalBalanceIsZero); + } + // Guard against an out-of-bounds during the validator balance update. + if statuses.statuses.len() != state.validator_balances.len() { + return Err(Error::ValidatorStatusesInconsistent); + } // Justification and finalization let epochs_since_finality = next_epoch - state.finalized_epoch; - if epochs_since_finality <= 4 { - for index in 0..state.validator_balances.len() { - let base_reward = state.base_reward(index, base_reward_quotient, spec); - - // Expected FFG source - if attesters.previous_epoch.indices.contains(&index) { - safe_add_assign!( - state.validator_balances[index], - base_reward * attesters.previous_epoch.balance / previous_total_balance + state.validator_balances = state + .validator_balances + .par_iter() + .enumerate() + .map(|(index, &balance)| { + let mut balance = balance; + let status = &statuses.statuses[index]; + let base_reward = get_base_reward(state, index, total_balances.previous_epoch, spec) + .expect( + "Cannot fail to access a validator balance when iterating validator balances.", ); - } else if active_validator_indices.contains(&index) { - safe_sub_assign!(state.validator_balances[index], base_reward); - } - // Expected FFG target - if attesters.previous_epoch_boundary.indices.contains(&index) { - safe_add_assign!( - state.validator_balances[index], - base_reward * attesters.previous_epoch_boundary.balance - / previous_total_balance - ); - } else if active_validator_indices.contains(&index) { - safe_sub_assign!(state.validator_balances[index], base_reward); - } - - // Expected beacon chain head - if attesters.previous_epoch_head.indices.contains(&index) { - safe_add_assign!( - state.validator_balances[index], - base_reward * attesters.previous_epoch_head.balance / previous_total_balance - ); - } else if active_validator_indices.contains(&index) { - safe_sub_assign!(state.validator_balances[index], base_reward); - } - } - - // Inclusion distance - for &index in &attesters.previous_epoch.indices { - let base_reward = state.base_reward(index, base_reward_quotient, spec); - let inclusion_distance = - inclusion_distance(state, &previous_epoch_attestations, index, spec)?; - - safe_add_assign!( - state.validator_balances[index], - base_reward * spec.min_attestation_inclusion_delay / inclusion_distance - ) - } - } else { - for index in 0..state.validator_balances.len() { - let inactivity_penalty = - state.inactivity_penalty(index, epochs_since_finality, base_reward_quotient, spec); - - if active_validator_indices.contains(&index) { - if !attesters.previous_epoch.indices.contains(&index) { - safe_sub_assign!(state.validator_balances[index], inactivity_penalty); - } - if !attesters.previous_epoch_boundary.indices.contains(&index) { - safe_sub_assign!(state.validator_balances[index], inactivity_penalty); - } - if !attesters.previous_epoch_head.indices.contains(&index) { - safe_sub_assign!(state.validator_balances[index], inactivity_penalty); - } - - if state.validator_registry[index].slashed { - let base_reward = state.base_reward(index, base_reward_quotient, spec); - safe_sub_assign!( - state.validator_balances[index], - 2 * inactivity_penalty + base_reward + if epochs_since_finality <= 4 { + // Expected FFG source + if status.is_previous_epoch_attester { + safe_add_assign!( + balance, + base_reward * total_balances.previous_epoch_attesters + / total_balances.previous_epoch ); + } else if status.is_active_in_previous_epoch { + safe_sub_assign!(balance, base_reward); + } + + // Expected FFG target + if status.is_previous_epoch_boundary_attester { + safe_add_assign!( + balance, + base_reward * total_balances.previous_epoch_boundary_attesters + / total_balances.previous_epoch + ); + } else if status.is_active_in_previous_epoch { + safe_sub_assign!(balance, base_reward); + } + + // Expected beacon chain head + if status.is_previous_epoch_head_attester { + safe_add_assign!( + balance, + base_reward * total_balances.previous_epoch_head_attesters + / total_balances.previous_epoch + ); + } else if status.is_active_in_previous_epoch { + safe_sub_assign!(balance, base_reward); + }; + } else { + let inactivity_penalty = get_inactivity_penalty( + state, + index, + epochs_since_finality.as_u64(), + total_balances.previous_epoch, + spec, + ) + .expect( + "Cannot fail to access a validator balance when iterating validator balances.", + ); + + if status.is_active_in_previous_epoch { + if !status.is_previous_epoch_attester { + safe_sub_assign!(balance, inactivity_penalty); + } + if !status.is_previous_epoch_boundary_attester { + safe_sub_assign!(balance, inactivity_penalty); + } + if !status.is_previous_epoch_head_attester { + safe_sub_assign!(balance, inactivity_penalty); + } + + if state.validator_registry[index].slashed { + let base_reward = + get_base_reward(state, index, total_balances.previous_epoch, spec).expect( + "Cannot fail to access a validator balance when iterating validator balances.", + ); + safe_sub_assign!(balance, 2 * inactivity_penalty + base_reward); + } } } - } - for &index in &attesters.previous_epoch.indices { - let base_reward = state.base_reward(index, base_reward_quotient, spec); - let inclusion_distance = - inclusion_distance(state, &previous_epoch_attestations, index, spec)?; + // Crosslinks - safe_sub_assign!( - state.validator_balances[index], - base_reward - - base_reward * spec.min_attestation_inclusion_delay / inclusion_distance - ); - } - } + if let Some(ref info) = status.winning_root_info { + safe_add_assign!( + balance, + base_reward * info.total_attesting_balance / info.total_committee_balance + ); + } else { + safe_sub_assign!(balance, base_reward); + } + + balance + }) + .collect(); // Attestation inclusion - for &index in &attesters.previous_epoch.indices { - let inclusion_slot = inclusion_slot(state, &previous_epoch_attestations[..], index, spec)?; - - let proposer_index = state - .get_beacon_proposer_index(inclusion_slot, spec) - .map_err(|_| Error::UnableToDetermineProducer)?; - - let base_reward = state.base_reward(proposer_index, base_reward_quotient, spec); - - safe_add_assign!( - state.validator_balances[proposer_index], - base_reward / spec.attestation_inclusion_reward_quotient - ); + // Guard against an out-of-bounds during the attester inclusion balance update. + if statuses.statuses.len() != state.validator_registry.len() { + return Err(Error::ValidatorStatusesInconsistent); } - //Crosslinks + for (index, _validator) in state.validator_registry.iter().enumerate() { + let status = &statuses.statuses[index]; - for slot in state.previous_epoch(spec).slot_iter(spec.slots_per_epoch) { - // Clone removes the borrow which becomes an issue when mutating `state.balances`. - let crosslink_committees_at_slot = - state.get_crosslink_committees_at_slot(slot, spec)?.clone(); + if status.is_previous_epoch_attester { + let proposer_index = status.inclusion_info.proposer_index; + let inclusion_distance = status.inclusion_info.distance; - for (crosslink_committee, shard) in crosslink_committees_at_slot { - let shard = shard as u64; + let base_reward = + get_base_reward(state, proposer_index, total_balances.previous_epoch, spec).expect( + "Cannot fail to access a validator balance when iterating validator balances.", + ); - // Note: I'm a little uncertain of the logic here -- I am waiting for spec v0.5.0 to - // clear it up. - // - // What happens here is: - // - // - If there was some crosslink root elected by the super-majority of this committee, - // then we reward all who voted for that root and penalize all that did not. - // - However, if there _was not_ some super-majority-voted crosslink root, then penalize - // all the validators. - // - // I'm not quite sure that the second case (no super-majority crosslink) is correct. - if let Some(winning_root) = winning_root_for_shards.get(&shard) { - // Hash set de-dedups and (hopefully) offers a speed improvement from faster - // lookups. - let attesting_validator_indices: HashSet = - HashSet::from_iter(winning_root.attesting_validator_indices.iter().cloned()); - - for &index in &crosslink_committee { - let base_reward = state.base_reward(index, base_reward_quotient, spec); - - let total_balance = state.get_total_balance(&crosslink_committee, spec); - - if attesting_validator_indices.contains(&index) { - safe_add_assign!( - state.validator_balances[index], - base_reward * winning_root.total_attesting_balance / total_balance - ); - } else { - safe_sub_assign!(state.validator_balances[index], base_reward); - } - } - } else { - for &index in &crosslink_committee { - let base_reward = state.base_reward(index, base_reward_quotient, spec); - - safe_sub_assign!(state.validator_balances[index], base_reward); - } + if inclusion_distance > 0 && inclusion_distance < Slot::max_value() { + safe_add_assign!( + state.validator_balances[proposer_index], + base_reward * spec.min_attestation_inclusion_delay + / inclusion_distance.as_u64() + ) } } } @@ -443,49 +402,76 @@ fn process_rewards_and_penalities( Ok(()) } -// Spec v0.4.0 -fn process_validator_registry(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> { - let current_epoch = state.current_epoch(spec); +/// Returns the base reward for some validator. +/// +/// Spec v0.5.0 +pub fn get_base_reward( + state: &BeaconState, + index: usize, + previous_total_balance: u64, + spec: &ChainSpec, +) -> Result { + if previous_total_balance == 0 { + Ok(0) + } else { + let adjusted_quotient = previous_total_balance.integer_sqrt() / spec.base_reward_quotient; + Ok(state.get_effective_balance(index, spec)? / adjusted_quotient / 5) + } +} + +/// Returns the inactivity penalty for some validator. +/// +/// Spec v0.5.0 +pub fn get_inactivity_penalty( + state: &BeaconState, + index: usize, + epochs_since_finality: u64, + previous_total_balance: u64, + spec: &ChainSpec, +) -> Result { + Ok(get_base_reward(state, index, previous_total_balance, spec)? + + state.get_effective_balance(index, spec)? * epochs_since_finality + / spec.inactivity_penalty_quotient + / 2) +} + +/// Updates the state's `latest_active_index_roots` field with a tree hash the active validator +/// indices for the next epoch. +/// +/// Spec v0.4.0 +pub fn update_active_tree_index_roots( + state: &mut BeaconState, + spec: &ChainSpec, +) -> Result<(), Error> { let next_epoch = state.next_epoch(spec); - state.previous_shuffling_epoch = state.current_shuffling_epoch; - state.previous_shuffling_start_shard = state.current_shuffling_start_shard; + let active_tree_root = get_active_validator_indices( + &state.validator_registry, + next_epoch + Epoch::from(spec.activation_exit_delay), + ) + .hash_tree_root(); - state.previous_shuffling_seed = state.current_shuffling_seed; - - let should_update_validator_registy = if state.finalized_epoch - > state.validator_registry_update_epoch - { - (0..state.get_current_epoch_committee_count(spec)).all(|i| { - let shard = (state.current_shuffling_start_shard + i as u64) % spec.shard_count; - state.latest_crosslinks[shard as usize].epoch > state.validator_registry_update_epoch - }) - } else { - false - }; - - if should_update_validator_registy { - state.update_validator_registry(spec); - - state.current_shuffling_epoch = next_epoch; - state.current_shuffling_start_shard = (state.current_shuffling_start_shard - + state.get_current_epoch_committee_count(spec) as u64) - % spec.shard_count; - state.current_shuffling_seed = state.generate_seed(state.current_shuffling_epoch, spec)? - } else { - let epochs_since_last_registry_update = - current_epoch - state.validator_registry_update_epoch; - if (epochs_since_last_registry_update > 1) - & epochs_since_last_registry_update.is_power_of_two() - { - state.current_shuffling_epoch = next_epoch; - state.current_shuffling_seed = - state.generate_seed(state.current_shuffling_epoch, spec)? - } - } - - state.process_slashings(spec); - state.process_exit_queue(spec); + state.latest_active_index_roots[(next_epoch.as_usize() + + spec.activation_exit_delay as usize) + % spec.latest_active_index_roots_length] = Hash256::from_slice(&active_tree_root[..]); Ok(()) } + +/// Advances the state's `latest_slashed_balances` field. +/// +/// Spec v0.4.0 +pub fn update_latest_slashed_balances(state: &mut BeaconState, spec: &ChainSpec) { + let current_epoch = state.current_epoch(spec); + let next_epoch = state.next_epoch(spec); + + state.latest_slashed_balances[next_epoch.as_usize() % spec.latest_slashed_exit_length] = + state.latest_slashed_balances[current_epoch.as_usize() % spec.latest_slashed_exit_length]; +} + +/// Removes all pending attestations from the previous epoch. +/// +/// Spec v0.4.0 +pub fn clean_attestations(state: &mut BeaconState) { + state.previous_epoch_attestations = vec![]; +} diff --git a/eth2/state_processing/src/per_epoch_processing/attester_sets.rs b/eth2/state_processing/src/per_epoch_processing/attester_sets.rs deleted file mode 100644 index 2b674e1bc..000000000 --- a/eth2/state_processing/src/per_epoch_processing/attester_sets.rs +++ /dev/null @@ -1,98 +0,0 @@ -use std::collections::HashSet; -use types::*; - -#[derive(Default)] -pub struct Attesters { - pub indices: HashSet, - pub balance: u64, -} - -impl Attesters { - fn add(&mut self, additional_indices: &[usize], additional_balance: u64) { - self.indices.reserve(additional_indices.len()); - for i in additional_indices { - self.indices.insert(*i); - } - self.balance.saturating_add(additional_balance); - } -} - -pub struct AttesterSets { - pub current_epoch: Attesters, - pub current_epoch_boundary: Attesters, - pub previous_epoch: Attesters, - pub previous_epoch_boundary: Attesters, - pub previous_epoch_head: Attesters, -} - -impl AttesterSets { - pub fn new(state: &BeaconState, spec: &ChainSpec) -> Result { - let mut current_epoch = Attesters::default(); - let mut current_epoch_boundary = Attesters::default(); - let mut previous_epoch = Attesters::default(); - let mut previous_epoch_boundary = Attesters::default(); - let mut previous_epoch_head = Attesters::default(); - - for a in &state.latest_attestations { - let attesting_indices = - state.get_attestation_participants(&a.data, &a.aggregation_bitfield, spec)?; - let attesting_balance = state.get_total_balance(&attesting_indices, spec); - - if is_from_epoch(a, state.current_epoch(spec), spec) { - current_epoch.add(&attesting_indices, attesting_balance); - - if has_common_epoch_boundary_root(a, state, state.current_epoch(spec), spec)? { - current_epoch_boundary.add(&attesting_indices, attesting_balance); - } - } else if is_from_epoch(a, state.previous_epoch(spec), spec) { - previous_epoch.add(&attesting_indices, attesting_balance); - - if has_common_epoch_boundary_root(a, state, state.previous_epoch(spec), spec)? { - previous_epoch_boundary.add(&attesting_indices, attesting_balance); - } - - if has_common_beacon_block_root(a, state, spec)? { - previous_epoch_head.add(&attesting_indices, attesting_balance); - } - } - } - - Ok(Self { - current_epoch, - current_epoch_boundary, - previous_epoch, - previous_epoch_boundary, - previous_epoch_head, - }) - } -} - -fn is_from_epoch(a: &PendingAttestation, epoch: Epoch, spec: &ChainSpec) -> bool { - a.data.slot.epoch(spec.slots_per_epoch) == epoch -} - -fn has_common_epoch_boundary_root( - a: &PendingAttestation, - state: &BeaconState, - epoch: Epoch, - spec: &ChainSpec, -) -> Result { - let slot = epoch.start_slot(spec.slots_per_epoch); - let state_boundary_root = *state - .get_block_root(slot, spec) - .ok_or_else(|| BeaconStateError::InsufficientBlockRoots)?; - - Ok(a.data.epoch_boundary_root == state_boundary_root) -} - -fn has_common_beacon_block_root( - a: &PendingAttestation, - state: &BeaconState, - spec: &ChainSpec, -) -> Result { - let state_block_root = *state - .get_block_root(a.data.slot, spec) - .ok_or_else(|| BeaconStateError::InsufficientBlockRoots)?; - - Ok(a.data.beacon_block_root == state_block_root) -} diff --git a/eth2/state_processing/src/per_epoch_processing/errors.rs b/eth2/state_processing/src/per_epoch_processing/errors.rs index 51e9b253c..94fc0cca5 100644 --- a/eth2/state_processing/src/per_epoch_processing/errors.rs +++ b/eth2/state_processing/src/per_epoch_processing/errors.rs @@ -6,6 +6,14 @@ pub enum EpochProcessingError { NoBlockRoots, BaseRewardQuotientIsZero, NoRandaoSeed, + PreviousTotalBalanceIsZero, + InclusionDistanceZero, + ValidatorStatusesInconsistent, + /// Unable to get the inclusion distance for a validator that should have an inclusion + /// distance. This indicates an internal inconsistency. + /// + /// (validator_index) + InclusionSlotsInconsistent(usize), BeaconStateError(BeaconStateError), InclusionError(InclusionError), } diff --git a/eth2/state_processing/src/per_epoch_processing/get_attestation_participants.rs b/eth2/state_processing/src/per_epoch_processing/get_attestation_participants.rs new file mode 100644 index 000000000..d822e434d --- /dev/null +++ b/eth2/state_processing/src/per_epoch_processing/get_attestation_participants.rs @@ -0,0 +1,37 @@ +use types::{beacon_state::helpers::verify_bitfield_length, *}; + +/// Returns validator indices which participated in the attestation. +/// +/// Spec v0.5.0 +pub fn get_attestation_participants( + state: &BeaconState, + attestation_data: &AttestationData, + bitfield: &Bitfield, + spec: &ChainSpec, +) -> Result, BeaconStateError> { + let epoch = attestation_data.slot.epoch(spec.slots_per_epoch); + + let crosslink_committee = + state.get_crosslink_committee_for_shard(epoch, attestation_data.shard, spec)?; + + if crosslink_committee.slot != attestation_data.slot { + return Err(BeaconStateError::NoCommitteeForShard); + } + + let committee = &crosslink_committee.committee; + + if !verify_bitfield_length(&bitfield, committee.len()) { + return Err(BeaconStateError::InvalidBitfield); + } + + let mut participants = Vec::with_capacity(committee.len()); + for (i, validator_index) in committee.iter().enumerate() { + match bitfield.get(i) { + Ok(bit) if bit == true => participants.push(*validator_index), + _ => {} + } + } + participants.shrink_to_fit(); + + Ok(participants) +} diff --git a/eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs b/eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs index 243dc67f0..b52485947 100644 --- a/eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs +++ b/eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs @@ -1,12 +1,11 @@ use super::errors::InclusionError; +use super::get_attestation_participants::get_attestation_participants; use types::*; /// Returns the distance between the first included attestation for some validator and this /// slot. /// -/// Note: In the spec this is defined "inline", not as a helper function. -/// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn inclusion_distance( state: &BeaconState, attestations: &[&PendingAttestation], @@ -19,9 +18,7 @@ pub fn inclusion_distance( /// Returns the slot of the earliest included attestation for some validator. /// -/// Note: In the spec this is defined "inline", not as a helper function. -/// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn inclusion_slot( state: &BeaconState, attestations: &[&PendingAttestation], @@ -34,9 +31,7 @@ pub fn inclusion_slot( /// Finds the earliest included attestation for some validator. /// -/// Note: In the spec this is defined "inline", not as a helper function. -/// -/// Spec v0.4.0 +/// Spec v0.5.0 fn earliest_included_attestation( state: &BeaconState, attestations: &[&PendingAttestation], @@ -47,7 +42,7 @@ fn earliest_included_attestation( for (i, a) in attestations.iter().enumerate() { let participants = - state.get_attestation_participants(&a.data, &a.aggregation_bitfield, spec)?; + get_attestation_participants(state, &a.data, &a.aggregation_bitfield, spec)?; if participants.iter().any(|i| *i == validator_index) { included_attestations.push(i); } diff --git a/eth2/state_processing/src/per_epoch_processing/process_validator_registry.rs b/eth2/state_processing/src/per_epoch_processing/process_validator_registry.rs new file mode 100644 index 000000000..26ebd60b3 --- /dev/null +++ b/eth2/state_processing/src/per_epoch_processing/process_validator_registry.rs @@ -0,0 +1,72 @@ +use super::Error; +use types::*; + +/// Peforms a validator registry update, if required. +/// +/// Spec v0.4.0 +pub fn process_validator_registry(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> { + let current_epoch = state.current_epoch(spec); + let next_epoch = state.next_epoch(spec); + + state.previous_shuffling_epoch = state.current_shuffling_epoch; + state.previous_shuffling_start_shard = state.current_shuffling_start_shard; + + state.previous_shuffling_seed = state.current_shuffling_seed; + + if should_update_validator_registry(state, spec)? { + state.update_validator_registry(spec)?; + + state.current_shuffling_epoch = next_epoch; + state.current_shuffling_start_shard = (state.current_shuffling_start_shard + + spec.get_epoch_committee_count( + state + .get_active_validator_indices(current_epoch, spec)? + .len(), + ) as u64) + % spec.shard_count; + state.current_shuffling_seed = state.generate_seed(state.current_shuffling_epoch, spec)? + } else { + let epochs_since_last_registry_update = + current_epoch - state.validator_registry_update_epoch; + if (epochs_since_last_registry_update > 1) + & epochs_since_last_registry_update.is_power_of_two() + { + state.current_shuffling_epoch = next_epoch; + state.current_shuffling_seed = + state.generate_seed(state.current_shuffling_epoch, spec)? + } + } + + state.process_slashings(spec)?; + state.process_exit_queue(spec); + + Ok(()) +} + +/// Returns `true` if the validator registry should be updated during an epoch processing. +/// +/// Spec v0.5.0 +pub fn should_update_validator_registry( + state: &BeaconState, + spec: &ChainSpec, +) -> Result { + if state.finalized_epoch <= state.validator_registry_update_epoch { + return Ok(false); + } + + let num_active_validators = state + .get_active_validator_indices(state.current_epoch(spec), spec)? + .len(); + let current_epoch_committee_count = spec.get_epoch_committee_count(num_active_validators); + + for shard in (0..current_epoch_committee_count) + .into_iter() + .map(|i| (state.current_shuffling_start_shard + i as u64) % spec.shard_count) + { + if state.latest_crosslinks[shard as usize].epoch <= state.validator_registry_update_epoch { + return Ok(false); + } + } + + Ok(true) +} diff --git a/eth2/state_processing/src/per_epoch_processing/tests.rs b/eth2/state_processing/src/per_epoch_processing/tests.rs index 627df858b..69450edcd 100644 --- a/eth2/state_processing/src/per_epoch_processing/tests.rs +++ b/eth2/state_processing/src/per_epoch_processing/tests.rs @@ -1,21 +1,21 @@ #![cfg(test)] use crate::per_epoch_processing; use env_logger::{Builder, Env}; -use types::beacon_state::BeaconStateBuilder; +use types::test_utils::TestingBeaconStateBuilder; use types::*; #[test] fn runs_without_error() { Builder::from_env(Env::default().default_filter_or("error")).init(); - let mut builder = BeaconStateBuilder::new(8); - builder.spec = ChainSpec::few_validators(); + let spec = ChainSpec::few_validators(); - builder.build().unwrap(); - builder.teleport_to_end_of_epoch(builder.spec.genesis_epoch + 4); + let mut builder = TestingBeaconStateBuilder::from_deterministic_keypairs(8, &spec); - let mut state = builder.cloned_state(); + let target_slot = (spec.genesis_epoch + 4).end_slot(spec.slots_per_epoch); + builder.teleport_to_slot(target_slot, &spec); - let spec = &builder.spec; - per_epoch_processing(&mut state, spec).unwrap(); + let (mut state, _keypairs) = builder.build(); + + per_epoch_processing(&mut state, &spec).unwrap(); } diff --git a/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs b/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs new file mode 100644 index 000000000..bcbca8244 --- /dev/null +++ b/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs @@ -0,0 +1,323 @@ +use super::get_attestation_participants::get_attestation_participants; +use super::WinningRootHashSet; +use types::*; + +/// Sets the boolean `var` on `self` to be true if it is true on `other`. Otherwise leaves `self` +/// as is. +macro_rules! set_self_if_other_is_true { + ($self_: ident, $other: ident, $var: ident) => { + if $other.$var { + $self_.$var = true; + } + }; +} + +/// The information required to reward some validator for their participation in a "winning" +/// crosslink root. +#[derive(Default, Clone)] +pub struct WinningRootInfo { + /// The total balance of the crosslink committee. + pub total_committee_balance: u64, + /// The total balance of the crosslink committee that attested for the "winning" root. + pub total_attesting_balance: u64, +} + +/// The information required to reward a block producer for including an attestation in a block. +#[derive(Clone)] +pub struct InclusionInfo { + /// The earliest slot a validator had an attestation included in the previous epoch. + pub slot: Slot, + /// The distance between the attestation slot and the slot that attestation was included in a + /// block. + pub distance: Slot, + /// The index of the proposer at the slot where the attestation was included. + pub proposer_index: usize, +} + +impl Default for InclusionInfo { + /// Defaults to `slot` and `distance` at their maximum values and `proposer_index` at zero. + fn default() -> Self { + Self { + slot: Slot::max_value(), + distance: Slot::max_value(), + proposer_index: 0, + } + } +} + +impl InclusionInfo { + /// Tests if some `other` `InclusionInfo` has a lower inclusion slot than `self`. If so, + /// replaces `self` with `other`. + pub fn update(&mut self, other: &Self) { + if other.slot < self.slot { + self.slot = other.slot; + self.distance = other.distance; + self.proposer_index = other.proposer_index; + } + } +} + +/// Information required to reward some validator during the current and previous epoch. +#[derive(Default, Clone)] +pub struct AttesterStatus { + /// True if the validator was active in the state's _current_ epoch. + pub is_active_in_current_epoch: bool, + /// True if the validator was active in the state's _previous_ epoch. + pub is_active_in_previous_epoch: bool, + + /// True if the validator had an attestation included in the _current_ epoch. + pub is_current_epoch_attester: bool, + /// True if the validator's beacon block root attestation for the first slot of the _current_ + /// epoch matches the block root known to the state. + pub is_current_epoch_boundary_attester: bool, + /// True if the validator had an attestation included in the _previous_ epoch. + pub is_previous_epoch_attester: bool, + /// True if the validator's beacon block root attestation for the first slot of the _previous_ + /// epoch matches the block root known to the state. + pub is_previous_epoch_boundary_attester: bool, + /// True if the validator's beacon block root attestation in the _previous_ epoch at the + /// attestation's slot (`attestation_data.slot`) matches the block root known to the state. + pub is_previous_epoch_head_attester: bool, + + /// Information used to reward the block producer of this validators earliest-included + /// attestation. + pub inclusion_info: InclusionInfo, + /// Information used to reward/penalize the validator if they voted in the super-majority for + /// some shard block. + pub winning_root_info: Option, +} + +impl AttesterStatus { + /// Accepts some `other` `AttesterStatus` and updates `self` if required. + /// + /// Will never set one of the `bool` fields to `false`, it will only set it to `true` if other + /// contains a `true` field. + /// + /// Note: does not update the winning root info, this is done manually. + pub fn update(&mut self, other: &Self) { + // Update all the bool fields, only updating `self` if `other` is true (never setting + // `self` to false). + set_self_if_other_is_true!(self, other, is_active_in_current_epoch); + set_self_if_other_is_true!(self, other, is_active_in_previous_epoch); + set_self_if_other_is_true!(self, other, is_current_epoch_attester); + set_self_if_other_is_true!(self, other, is_current_epoch_boundary_attester); + set_self_if_other_is_true!(self, other, is_previous_epoch_attester); + set_self_if_other_is_true!(self, other, is_previous_epoch_boundary_attester); + set_self_if_other_is_true!(self, other, is_previous_epoch_head_attester); + + self.inclusion_info.update(&other.inclusion_info); + } +} + +/// The total effective balances for different sets of validators during the previous and current +/// epochs. +#[derive(Default, Clone)] +pub struct TotalBalances { + /// The total effective balance of all active validators during the _current_ epoch. + pub current_epoch: u64, + /// The total effective balance of all active validators during the _previous_ epoch. + pub previous_epoch: u64, + /// The total effective balance of all validators who attested during the _current_ epoch. + pub current_epoch_attesters: u64, + /// The total effective balance of all validators who attested during the _current_ epoch and + /// agreed with the state about the beacon block at the first slot of the _current_ epoch. + pub current_epoch_boundary_attesters: u64, + /// The total effective balance of all validators who attested during the _previous_ epoch. + pub previous_epoch_attesters: u64, + /// The total effective balance of all validators who attested during the _previous_ epoch and + /// agreed with the state about the beacon block at the first slot of the _previous_ epoch. + pub previous_epoch_boundary_attesters: u64, + /// The total effective balance of all validators who attested during the _previous_ epoch and + /// agreed with the state about the beacon block at the time of attestation. + pub previous_epoch_head_attesters: u64, +} + +/// Summarised information about validator participation in the _previous and _current_ epochs of +/// some `BeaconState`. +#[derive(Clone)] +pub struct ValidatorStatuses { + /// Information about each individual validator from the state's validator registy. + pub statuses: Vec, + /// Summed balances for various sets of validators. + pub total_balances: TotalBalances, +} + +impl ValidatorStatuses { + /// Initializes a new instance, determining: + /// + /// - Active validators + /// - Total balances for the current and previous epochs. + /// + /// Spec v0.5.0 + pub fn new(state: &BeaconState, spec: &ChainSpec) -> Result { + let mut statuses = Vec::with_capacity(state.validator_registry.len()); + let mut total_balances = TotalBalances::default(); + + for (i, validator) in state.validator_registry.iter().enumerate() { + let mut status = AttesterStatus::default(); + + if validator.is_active_at(state.current_epoch(spec)) { + status.is_active_in_current_epoch = true; + total_balances.current_epoch += state.get_effective_balance(i, spec)?; + } + + if validator.is_active_at(state.previous_epoch(spec)) { + status.is_active_in_previous_epoch = true; + total_balances.previous_epoch += state.get_effective_balance(i, spec)?; + } + + statuses.push(status); + } + + Ok(Self { + statuses, + total_balances, + }) + } + + /// Process some attestations from the given `state` updating the `statuses` and + /// `total_balances` fields. + /// + /// Spec v0.5.0 + pub fn process_attestations( + &mut self, + state: &BeaconState, + spec: &ChainSpec, + ) -> Result<(), BeaconStateError> { + for a in state + .previous_epoch_attestations + .iter() + .chain(state.current_epoch_attestations.iter()) + { + let attesting_indices = + get_attestation_participants(state, &a.data, &a.aggregation_bitfield, spec)?; + let attesting_balance = state.get_total_balance(&attesting_indices, spec)?; + + let mut status = AttesterStatus::default(); + + // Profile this attestation, updating the total balances and generating an + // `AttesterStatus` object that applies to all participants in the attestation. + if is_from_epoch(a, state.current_epoch(spec), spec) { + self.total_balances.current_epoch_attesters += attesting_balance; + status.is_current_epoch_attester = true; + + if has_common_epoch_boundary_root(a, state, state.current_epoch(spec), spec)? { + self.total_balances.current_epoch_boundary_attesters += attesting_balance; + status.is_current_epoch_boundary_attester = true; + } + } else if is_from_epoch(a, state.previous_epoch(spec), spec) { + self.total_balances.previous_epoch_attesters += attesting_balance; + status.is_previous_epoch_attester = true; + + // The inclusion slot and distance are only required for previous epoch attesters. + let relative_epoch = RelativeEpoch::from_slot(state.slot, a.data.slot, spec)?; + status.inclusion_info = InclusionInfo { + slot: a.inclusion_slot, + distance: inclusion_distance(a), + proposer_index: state.get_beacon_proposer_index( + a.inclusion_slot, + relative_epoch, + spec, + )?, + }; + + if has_common_epoch_boundary_root(a, state, state.previous_epoch(spec), spec)? { + self.total_balances.previous_epoch_boundary_attesters += attesting_balance; + status.is_previous_epoch_boundary_attester = true; + } + + if has_common_beacon_block_root(a, state, spec)? { + self.total_balances.previous_epoch_head_attesters += attesting_balance; + status.is_previous_epoch_head_attester = true; + } + } + + // Loop through the participating validator indices and update the status vec. + for validator_index in attesting_indices { + self.statuses[validator_index].update(&status); + } + } + + Ok(()) + } + + /// Update the `statuses` for each validator based upon whether or not they attested to the + /// "winning" shard block root for the previous epoch. + /// + /// Spec v0.5.0 + pub fn process_winning_roots( + &mut self, + state: &BeaconState, + winning_roots: &WinningRootHashSet, + spec: &ChainSpec, + ) -> Result<(), BeaconStateError> { + // Loop through each slot in the previous epoch. + for slot in state.previous_epoch(spec).slot_iter(spec.slots_per_epoch) { + let crosslink_committees_at_slot = + state.get_crosslink_committees_at_slot(slot, spec)?; + + // Loop through each committee in the slot. + for c in crosslink_committees_at_slot { + // If there was some winning crosslink root for the committee's shard. + if let Some(winning_root) = winning_roots.get(&c.shard) { + let total_committee_balance = state.get_total_balance(&c.committee, spec)?; + for &validator_index in &winning_root.attesting_validator_indices { + // Take note of the balance information for the winning root, it will be + // used later to calculate rewards for that validator. + self.statuses[validator_index].winning_root_info = Some(WinningRootInfo { + total_committee_balance, + total_attesting_balance: winning_root.total_attesting_balance, + }) + } + } + } + } + + Ok(()) + } +} + +/// Returns the distance between when the attestation was created and when it was included in a +/// block. +/// +/// Spec v0.5.0 +fn inclusion_distance(a: &PendingAttestation) -> Slot { + a.inclusion_slot - a.data.slot +} + +/// Returns `true` if some `PendingAttestation` is from the supplied `epoch`. +/// +/// Spec v0.5.0 +fn is_from_epoch(a: &PendingAttestation, epoch: Epoch, spec: &ChainSpec) -> bool { + a.data.slot.epoch(spec.slots_per_epoch) == epoch +} + +/// Returns `true` if a `PendingAttestation` and `BeaconState` share the same beacon block hash for +/// the first slot of the given epoch. +/// +/// Spec v0.5.0 +fn has_common_epoch_boundary_root( + a: &PendingAttestation, + state: &BeaconState, + epoch: Epoch, + spec: &ChainSpec, +) -> Result { + let slot = epoch.start_slot(spec.slots_per_epoch); + let state_boundary_root = *state.get_block_root(slot, spec)?; + + Ok(a.data.target_root == state_boundary_root) +} + +/// Returns `true` if a `PendingAttestation` and `BeaconState` share the same beacon block hash for +/// the current slot of the `PendingAttestation`. +/// +/// Spec v0.5.0 +fn has_common_beacon_block_root( + a: &PendingAttestation, + state: &BeaconState, + spec: &ChainSpec, +) -> Result { + let state_block_root = *state.get_block_root(a.data.slot, spec)?; + + Ok(a.data.beacon_block_root == state_block_root) +} diff --git a/eth2/state_processing/src/per_epoch_processing/winning_root.rs b/eth2/state_processing/src/per_epoch_processing/winning_root.rs index 07678f93b..97cff3e13 100644 --- a/eth2/state_processing/src/per_epoch_processing/winning_root.rs +++ b/eth2/state_processing/src/per_epoch_processing/winning_root.rs @@ -1,3 +1,4 @@ +use super::get_attestation_participants::get_attestation_participants; use std::collections::HashSet; use std::iter::FromIterator; use types::*; @@ -13,14 +14,14 @@ impl WinningRoot { /// Returns `true` if `self` is a "better" candidate than `other`. /// /// A winning root is "better" than another if it has a higher `total_attesting_balance`. Ties - /// are broken by favouring the lower `crosslink_data_root` value. + /// are broken by favouring the higher `crosslink_data_root` value. /// - /// Spec v0.4.0 + /// Spec v0.5.0 pub fn is_better_than(&self, other: &Self) -> bool { if self.total_attesting_balance > other.total_attesting_balance { true } else if self.total_attesting_balance == other.total_attesting_balance { - self.crosslink_data_root < other.crosslink_data_root + self.crosslink_data_root > other.crosslink_data_root } else { false } @@ -33,22 +34,21 @@ impl WinningRoot { /// The `WinningRoot` object also contains additional fields that are useful in later stages of /// per-epoch processing. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn winning_root( state: &BeaconState, shard: u64, - current_epoch_attestations: &[&PendingAttestation], - previous_epoch_attestations: &[&PendingAttestation], spec: &ChainSpec, ) -> Result, BeaconStateError> { let mut winning_root: Option = None; let crosslink_data_roots: HashSet = HashSet::from_iter( - previous_epoch_attestations + state + .previous_epoch_attestations .iter() - .chain(current_epoch_attestations.iter()) + .chain(state.current_epoch_attestations.iter()) .filter_map(|a| { - if a.data.shard == shard { + if is_eligible_for_winning_root(state, a, shard) { Some(a.data.crosslink_data_root) } else { None @@ -57,18 +57,17 @@ pub fn winning_root( ); for crosslink_data_root in crosslink_data_roots { - let attesting_validator_indices = get_attesting_validator_indices( - state, - shard, - current_epoch_attestations, - previous_epoch_attestations, - &crosslink_data_root, - spec, - )?; + let attesting_validator_indices = + get_attesting_validator_indices(state, shard, &crosslink_data_root, spec)?; - let total_attesting_balance: u64 = attesting_validator_indices - .iter() - .fold(0, |acc, i| acc + state.get_effective_balance(*i, spec)); + let total_attesting_balance: u64 = + attesting_validator_indices + .iter() + .try_fold(0_u64, |acc, i| { + state + .get_effective_balance(*i, spec) + .and_then(|bal| Ok(acc + bal)) + })?; let candidate = WinningRoot { crosslink_data_root, @@ -88,25 +87,36 @@ pub fn winning_root( Ok(winning_root) } -/// Returns all indices which voted for a given crosslink. May contain duplicates. +/// Returns `true` if pending attestation `a` is eligible to become a winning root. /// -/// Spec v0.4.0 +/// Spec v0.5.0 +fn is_eligible_for_winning_root(state: &BeaconState, a: &PendingAttestation, shard: Shard) -> bool { + if shard >= state.latest_crosslinks.len() as u64 { + return false; + } + + a.data.previous_crosslink == state.latest_crosslinks[shard as usize] +} + +/// Returns all indices which voted for a given crosslink. Does not contain duplicates. +/// +/// Spec v0.5.0 fn get_attesting_validator_indices( state: &BeaconState, shard: u64, - current_epoch_attestations: &[&PendingAttestation], - previous_epoch_attestations: &[&PendingAttestation], crosslink_data_root: &Hash256, spec: &ChainSpec, ) -> Result, BeaconStateError> { let mut indices = vec![]; - for a in current_epoch_attestations + for a in state + .current_epoch_attestations .iter() - .chain(previous_epoch_attestations.iter()) + .chain(state.previous_epoch_attestations.iter()) { if (a.data.shard == shard) && (a.data.crosslink_data_root == *crosslink_data_root) { - indices.append(&mut state.get_attestation_participants( + indices.append(&mut get_attestation_participants( + state, &a.data, &a.aggregation_bitfield, spec, @@ -114,5 +124,41 @@ fn get_attesting_validator_indices( } } + // Sort the list (required for dedup). "Unstable" means the sort may re-order equal elements, + // this causes no issue here. + // + // These sort + dedup ops are potentially good CPU time optimisation targets. + indices.sort_unstable(); + // Remove all duplicate indices (requires a sorted list). + indices.dedup(); + Ok(indices) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn is_better_than() { + let worse = WinningRoot { + crosslink_data_root: Hash256::from_slice(&[1; 32]), + attesting_validator_indices: vec![], + total_attesting_balance: 42, + }; + + let better = WinningRoot { + crosslink_data_root: Hash256::from_slice(&[2; 32]), + ..worse.clone() + }; + + assert!(better.is_better_than(&worse)); + + let better = WinningRoot { + total_attesting_balance: worse.total_attesting_balance + 1, + ..worse.clone() + }; + + assert!(better.is_better_than(&worse)); + } +} diff --git a/eth2/state_processing/src/per_slot_processing.rs b/eth2/state_processing/src/per_slot_processing.rs index 0bb405c98..a90c5b408 100644 --- a/eth2/state_processing/src/per_slot_processing.rs +++ b/eth2/state_processing/src/per_slot_processing.rs @@ -1,5 +1,6 @@ use crate::*; -use types::{BeaconState, BeaconStateError, ChainSpec, Hash256}; +use ssz::TreeHash; +use types::*; #[derive(Debug, PartialEq)] pub enum Error { @@ -9,12 +10,14 @@ pub enum Error { /// Advances a state forward by one slot, performing per-epoch processing if required. /// -/// Spec v0.4.0 +/// Spec v0.5.0 pub fn per_slot_processing( state: &mut BeaconState, - previous_block_root: Hash256, + latest_block_header: &BeaconBlockHeader, spec: &ChainSpec, ) -> Result<(), Error> { + cache_state(state, latest_block_header, spec)?; + if (state.slot + 1) % spec.slots_per_epoch == 0 { per_epoch_processing(state, spec)?; state.advance_caches(); @@ -22,27 +25,38 @@ pub fn per_slot_processing( state.slot += 1; - update_block_roots(state, previous_block_root, spec); + let latest_block_root = Hash256::from_slice(&state.latest_block_header.hash_tree_root()[..]); + state.set_block_root(state.slot - 1, latest_block_root, spec)?; Ok(()) } -/// Updates the state's block roots as per-slot processing is performed. -/// -/// Spec v0.4.0 -pub fn update_block_roots(state: &mut BeaconState, previous_block_root: Hash256, spec: &ChainSpec) { - state.latest_block_roots[(state.slot.as_usize() - 1) % spec.latest_block_roots_length] = - previous_block_root; +fn cache_state( + state: &mut BeaconState, + latest_block_header: &BeaconBlockHeader, + spec: &ChainSpec, +) -> Result<(), Error> { + let previous_slot_state_root = Hash256::from_slice(&state.hash_tree_root()[..]); - if state.slot.as_usize() % spec.latest_block_roots_length == 0 { - let root = merkle_root(&state.latest_block_roots[..]); - state.batched_block_roots.push(root); + // Note: increment the state slot here to allow use of our `state_root` and `block_root` + // getter/setter functions. + // + // This is a bit hacky, however it gets the job safely without lots of code. + let previous_slot = state.slot; + state.slot += 1; + + // Store the previous slot's post-state transition root. + if state.latest_block_header.state_root == spec.zero_hash { + state.latest_block_header.state_root = previous_slot_state_root } -} -fn merkle_root(_input: &[Hash256]) -> Hash256 { - // TODO: implement correctly. - Hash256::zero() + let latest_block_root = Hash256::from_slice(&latest_block_header.hash_tree_root()[..]); + state.set_block_root(previous_slot, latest_block_root, spec)?; + + // Set the state slot back to what it should be. + state.slot -= 1; + + Ok(()) } impl From for Error { diff --git a/eth2/state_processing/tests/tests.rs b/eth2/state_processing/tests/tests.rs new file mode 100644 index 000000000..9cee7b34c --- /dev/null +++ b/eth2/state_processing/tests/tests.rs @@ -0,0 +1,42 @@ +use serde_derive::Deserialize; +use types::*; + +#[derive(Debug, Deserialize)] +pub struct TestCase { + pub name: String, + pub config: ChainSpec, + pub verify_signatures: bool, + pub initial_state: BeaconState, + pub blocks: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct TestDoc { + pub title: String, + pub summary: String, + pub fork: String, + pub version: String, + pub test_cases: Vec, +} + +#[test] +#[ignore] +fn yaml() { + use serde_yaml; + use std::{fs::File, io::prelude::*, path::PathBuf}; + + let mut file = { + let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + file_path_buf.push("specs/example.yml"); + + File::open(file_path_buf).unwrap() + }; + + let mut yaml_str = String::new(); + + file.read_to_string(&mut yaml_str).unwrap(); + + let yaml_str = yaml_str.to_lowercase(); + + let _doc: TestDoc = serde_yaml::from_str(&yaml_str.as_str()).unwrap(); +} diff --git a/eth2/types/Cargo.toml b/eth2/types/Cargo.toml index f8120d520..1b5f79d23 100644 --- a/eth2/types/Cargo.toml +++ b/eth2/types/Cargo.toml @@ -7,6 +7,7 @@ edition = "2018" [dependencies] bls = { path = "../utils/bls" } boolean-bitfield = { path = "../utils/boolean-bitfield" } +dirs = "1.0" ethereum-types = "0.5" hashing = { path = "../utils/hashing" } honey-badger-split = { path = "../utils/honey-badger-split" } @@ -17,6 +18,7 @@ rand = "0.5.5" serde = "1.0" serde_derive = "1.0" serde_json = "1.0" +serde_yaml = "0.8" slog = "^2.2.3" ssz = { path = "../utils/ssz" } ssz_derive = { path = "../utils/ssz_derive" } diff --git a/eth2/types/src/attestation.rs b/eth2/types/src/attestation.rs index 03ef8ce48..0b660466e 100644 --- a/eth2/types/src/attestation.rs +++ b/eth2/types/src/attestation.rs @@ -1,15 +1,26 @@ use super::{AggregateSignature, AttestationData, Bitfield}; use crate::test_utils::TestRandom; use rand::RngCore; -use serde_derive::Serialize; +use serde_derive::{Deserialize, Serialize}; use ssz::TreeHash; use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; use test_random_derive::TestRandom; /// Details an attestation that can be slashable. /// -/// Spec v0.4.0 -#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)] +/// Spec v0.5.0 +#[derive( + Debug, + Clone, + PartialEq, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + TestRandom, + SignedRoot, +)] pub struct Attestation { pub aggregation_bitfield: Bitfield, pub data: AttestationData, @@ -20,29 +31,6 @@ pub struct Attestation { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Attestation::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Attestation::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(Attestation); } diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data.rs index 1dfadfb1d..4a6b57823 100644 --- a/eth2/types/src/attestation_data.rs +++ b/eth2/types/src/attestation_data.rs @@ -1,20 +1,21 @@ use crate::test_utils::TestRandom; use crate::{Crosslink, Epoch, Hash256, Slot}; use rand::RngCore; -use serde_derive::Serialize; +use serde_derive::{Deserialize, Serialize}; use ssz::TreeHash; use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; use test_random_derive::TestRandom; /// The data upon which an attestation is based. /// -/// Spec v0.4.0 +/// Spec v0.5.0 #[derive( Debug, Clone, PartialEq, Default, Serialize, + Deserialize, Hash, Encode, Decode, @@ -23,14 +24,19 @@ use test_random_derive::TestRandom; SignedRoot, )] pub struct AttestationData { + // LMD GHOST vote pub slot: Slot, - pub shard: u64, pub beacon_block_root: Hash256, - pub epoch_boundary_root: Hash256, + + // FFG Vote + pub source_epoch: Epoch, + pub source_root: Hash256, + pub target_root: Hash256, + + // Crosslink Vote + pub shard: u64, + pub previous_crosslink: Crosslink, pub crosslink_data_root: Hash256, - pub latest_crosslink: Crosslink, - pub justified_epoch: Epoch, - pub justified_block_root: Hash256, } impl Eq for AttestationData {} @@ -38,29 +44,6 @@ impl Eq for AttestationData {} #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = AttestationData::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = AttestationData::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(AttestationData); } diff --git a/eth2/types/src/attestation_data_and_custody_bit.rs b/eth2/types/src/attestation_data_and_custody_bit.rs index 83018c194..2cc6bc80c 100644 --- a/eth2/types/src/attestation_data_and_custody_bit.rs +++ b/eth2/types/src/attestation_data_and_custody_bit.rs @@ -6,7 +6,7 @@ use ssz_derive::{Decode, Encode, TreeHash}; /// Used for pairing an attestation with a proof-of-custody. /// -/// Spec v0.4.0 +/// Spec v0.5.0 #[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode, TreeHash)] pub struct AttestationDataAndCustodyBit { pub data: AttestationData, @@ -25,31 +25,6 @@ impl TestRandom for AttestationDataAndCustodyBit { #[cfg(test)] mod test { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - - let original = AttestationDataAndCustodyBit::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = AttestationDataAndCustodyBit::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(AttestationDataAndCustodyBit); } diff --git a/eth2/types/src/attestation_duty.rs b/eth2/types/src/attestation_duty.rs new file mode 100644 index 000000000..f6e86d263 --- /dev/null +++ b/eth2/types/src/attestation_duty.rs @@ -0,0 +1,9 @@ +use crate::*; +use serde_derive::{Deserialize, Serialize}; + +#[derive(Debug, PartialEq, Clone, Default, Serialize, Deserialize)] +pub struct AttestationDuty { + pub slot: Slot, + pub shard: Shard, + pub committee_index: usize, +} diff --git a/eth2/types/src/attester_slashing.rs b/eth2/types/src/attester_slashing.rs index 1cb671960..6fc404f42 100644 --- a/eth2/types/src/attester_slashing.rs +++ b/eth2/types/src/attester_slashing.rs @@ -1,17 +1,13 @@ use crate::{test_utils::TestRandom, SlashableAttestation}; use rand::RngCore; -use serde_derive::Serialize; +use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; -mod builder; - -pub use builder::AttesterSlashingBuilder; - /// Two conflicting attestations. /// -/// Spec v0.4.0 -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] +/// Spec v0.5.0 +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] pub struct AttesterSlashing { pub slashable_attestation_1: SlashableAttestation, pub slashable_attestation_2: SlashableAttestation, @@ -20,29 +16,6 @@ pub struct AttesterSlashing { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = AttesterSlashing::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = AttesterSlashing::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(AttesterSlashing); } diff --git a/eth2/types/src/beacon_block.rs b/eth2/types/src/beacon_block.rs index 2e1e24ef7..b966751ed 100644 --- a/eth2/types/src/beacon_block.rs +++ b/eth2/types/src/beacon_block.rs @@ -1,40 +1,50 @@ use crate::test_utils::TestRandom; -use crate::{BeaconBlockBody, ChainSpec, Eth1Data, Hash256, Slot}; +use crate::*; use bls::Signature; use rand::RngCore; -use serde_derive::Serialize; +use serde_derive::{Deserialize, Serialize}; use ssz::TreeHash; use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; use test_random_derive::TestRandom; /// A block of the `BeaconChain`. /// -/// Spec v0.4.0 -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)] +/// Spec v0.5.0 +#[derive( + Debug, + PartialEq, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + TestRandom, + SignedRoot, +)] pub struct BeaconBlock { pub slot: Slot, - pub parent_root: Hash256, + pub previous_block_root: Hash256, pub state_root: Hash256, - pub randao_reveal: Signature, - pub eth1_data: Eth1Data, pub body: BeaconBlockBody, pub signature: Signature, } impl BeaconBlock { - /// Produce the first block of the Beacon Chain. - pub fn genesis(state_root: Hash256, spec: &ChainSpec) -> BeaconBlock { + /// Returns an empty block to be used during genesis. + /// + /// Spec v0.5.0 + pub fn empty(spec: &ChainSpec) -> BeaconBlock { BeaconBlock { slot: spec.genesis_slot, - parent_root: spec.zero_hash, - state_root, - randao_reveal: spec.empty_signature.clone(), - eth1_data: Eth1Data { - deposit_root: spec.zero_hash, - block_hash: spec.zero_hash, - }, - signature: spec.empty_signature.clone(), + previous_block_root: spec.zero_hash, + state_root: spec.zero_hash, body: BeaconBlockBody { + randao_reveal: spec.empty_signature.clone(), + eth1_data: Eth1Data { + deposit_root: spec.zero_hash, + block_hash: spec.zero_hash, + }, proposer_slashings: vec![], attester_slashings: vec![], attestations: vec![], @@ -42,41 +52,50 @@ impl BeaconBlock { voluntary_exits: vec![], transfers: vec![], }, + signature: spec.empty_signature.clone(), } } /// Returns the `hash_tree_root` of the block. + /// + /// Spec v0.5.0 pub fn canonical_root(&self) -> Hash256 { Hash256::from_slice(&self.hash_tree_root()[..]) } + + /// Returns a full `BeaconBlockHeader` of this block. + /// + /// Note: This method is used instead of an `Into` impl to avoid a `Clone` of an entire block + /// when you want to have the block _and_ the header. + /// + /// Note: performs a full tree-hash of `self.body`. + /// + /// Spec v0.5.0 + pub fn into_header(&self) -> BeaconBlockHeader { + BeaconBlockHeader { + slot: self.slot, + previous_block_root: self.previous_block_root, + state_root: self.state_root, + block_body_root: Hash256::from_slice(&self.body.hash_tree_root()[..]), + signature: self.signature.clone(), + } + } + + /// Returns a "temporary" header, where the `state_root` is `spec.zero_hash`. + /// + /// Spec v0.5.0 + pub fn into_temporary_header(&self, spec: &ChainSpec) -> BeaconBlockHeader { + BeaconBlockHeader { + state_root: spec.zero_hash, + signature: spec.empty_signature.clone(), + ..self.into_header() + } + } } #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = BeaconBlock::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = BeaconBlock::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(BeaconBlock); } diff --git a/eth2/types/src/beacon_block_body.rs b/eth2/types/src/beacon_block_body.rs index e7dec2e4b..677e24cec 100644 --- a/eth2/types/src/beacon_block_body.rs +++ b/eth2/types/src/beacon_block_body.rs @@ -1,15 +1,17 @@ -use super::{Attestation, AttesterSlashing, Deposit, ProposerSlashing, Transfer, VoluntaryExit}; use crate::test_utils::TestRandom; +use crate::*; use rand::RngCore; -use serde_derive::Serialize; +use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; /// The body of a `BeaconChain` block, containing operations. /// -/// Spec v0.4.0 -#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, TreeHash, TestRandom)] +/// Spec v0.5.0 +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] pub struct BeaconBlockBody { + pub randao_reveal: Signature, + pub eth1_data: Eth1Data, pub proposer_slashings: Vec, pub attester_slashings: Vec, pub attestations: Vec, @@ -21,29 +23,6 @@ pub struct BeaconBlockBody { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = BeaconBlockBody::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = BeaconBlockBody::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(BeaconBlockBody); } diff --git a/eth2/types/src/beacon_block_header.rs b/eth2/types/src/beacon_block_header.rs new file mode 100644 index 000000000..3d8b08cc8 --- /dev/null +++ b/eth2/types/src/beacon_block_header.rs @@ -0,0 +1,47 @@ +use crate::test_utils::TestRandom; +use crate::*; +use bls::Signature; +use rand::RngCore; +use serde_derive::{Deserialize, Serialize}; +use ssz::TreeHash; +use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; +use test_random_derive::TestRandom; + +/// A header of a `BeaconBlock`. +/// +/// Spec v0.5.0 +#[derive( + Debug, + PartialEq, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + TestRandom, + SignedRoot, +)] +pub struct BeaconBlockHeader { + pub slot: Slot, + pub previous_block_root: Hash256, + pub state_root: Hash256, + pub block_body_root: Hash256, + pub signature: Signature, +} + +impl BeaconBlockHeader { + /// Returns the `hash_tree_root` of the header. + /// + /// Spec v0.5.0 + pub fn canonical_root(&self) -> Hash256 { + Hash256::from_slice(&self.hash_tree_root()[..]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + ssz_tests!(BeaconBlockHeader); +} diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 809408b32..1b2424774 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -1,41 +1,20 @@ -use self::epoch_cache::EpochCache; +use self::epoch_cache::{EpochCache, Error as EpochCacheError}; use crate::test_utils::TestRandom; -use crate::{validator_registry::get_active_validator_indices, *}; -use bls::verify_proof_of_possession; -use helpers::*; -use honey_badger_split::SplitExt; +use crate::*; use int_to_bytes::int_to_bytes32; -use log::{debug, error, trace}; +use pubkey_cache::PubkeyCache; use rand::RngCore; -use rayon::prelude::*; -use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; -use std::collections::HashMap; -use swap_or_not_shuffle::shuffle_list; +use serde_derive::{Deserialize, Serialize}; +use ssz::{hash, ssz_encode, TreeHash}; +use ssz_derive::{Decode, Encode, TreeHash}; +use test_random_derive::TestRandom; -pub use builder::BeaconStateBuilder; - -mod builder; mod epoch_cache; pub mod helpers; +mod pubkey_cache; mod tests; -pub type Committee = Vec; -pub type CrosslinkCommittees = Vec<(Committee, u64)>; -pub type Shard = u64; -pub type CommitteeIndex = u64; -pub type AttestationDuty = (Slot, Shard, CommitteeIndex); -pub type AttestationDutyMap = HashMap; -pub type ShardCommitteeIndexMap = HashMap; - -pub const CACHED_EPOCHS: usize = 3; - -#[derive(Debug, PartialEq, Clone, Copy)] -pub enum RelativeEpoch { - Previous, - Current, - Next, -} +pub const CACHED_EPOCHS: usize = 4; #[derive(Debug, PartialEq)] pub enum Error { @@ -47,13 +26,25 @@ pub enum Error { UnableToShuffle, UnknownValidator, InvalidBitfield, + ValidatorIsWithdrawable, InsufficientRandaoMixes, - InsufficientValidators, + NoValidators, + UnableToDetermineProducer, InsufficientBlockRoots, InsufficientIndexRoots, InsufficientAttestations, InsufficientCommittees, + InsufficientSlashedBalances, + InsufficientStateRoots, + NoCommitteeForShard, EpochCacheUninitialized(RelativeEpoch), + PubkeyCacheInconsistent, + PubkeyCacheIncomplete { + cache_len: usize, + registry_len: usize, + }, + RelativeEpochError(RelativeEpochError), + EpochCacheError(EpochCacheError), } macro_rules! safe_add_assign { @@ -67,7 +58,10 @@ macro_rules! safe_sub_assign { }; } -#[derive(Debug, PartialEq, Clone, Default, Serialize)] +/// The state of the `BeaconChain` at some slot. +/// +/// Spec v0.5.0 +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, TestRandom, Encode, Decode, TreeHash)] pub struct BeaconState { // Misc pub slot: Slot, @@ -89,18 +83,24 @@ pub struct BeaconState { pub current_shuffling_seed: Hash256, // Finality + pub previous_epoch_attestations: Vec, + pub current_epoch_attestations: Vec, pub previous_justified_epoch: Epoch, - pub justified_epoch: Epoch, + pub current_justified_epoch: Epoch, + pub previous_justified_root: Hash256, + pub current_justified_root: Hash256, pub justification_bitfield: u64, pub finalized_epoch: Epoch, + pub finalized_root: Hash256, // Recent state pub latest_crosslinks: Vec, pub latest_block_roots: Vec, + pub latest_state_roots: Vec, pub latest_active_index_roots: Vec, pub latest_slashed_balances: Vec, - pub latest_attestations: Vec, - pub batched_block_roots: Vec, + pub latest_block_header: BeaconBlockHeader, + pub historical_roots: Vec, // Ethereum 1.0 chain data pub latest_eth1_data: Eth1Data, @@ -108,45 +108,51 @@ pub struct BeaconState { pub deposit_index: u64, // Caching (not in the spec) + #[serde(default)] + #[ssz(skip_serializing)] + #[ssz(skip_deserializing)] + #[tree_hash(skip_hashing)] + #[test_random(default)] pub cache_index_offset: usize, - pub caches: Vec, + #[serde(default)] + #[ssz(skip_serializing)] + #[ssz(skip_deserializing)] + #[tree_hash(skip_hashing)] + #[test_random(default)] + pub caches: [EpochCache; CACHED_EPOCHS], + #[serde(default)] + #[ssz(skip_serializing)] + #[ssz(skip_deserializing)] + #[tree_hash(skip_hashing)] + #[test_random(default)] + pub pubkey_cache: PubkeyCache, } impl BeaconState { /// Produce the first state of the Beacon Chain. - pub fn genesis_without_validators( - genesis_time: u64, - latest_eth1_data: Eth1Data, - spec: &ChainSpec, - ) -> Result { - debug!("Creating genesis state (without validator processing)."); + /// + /// This does not fully build a genesis beacon state, it omits processing of initial validator + /// deposits. To obtain a full genesis beacon state, use the `BeaconStateBuilder`. + /// + /// Spec v0.5.0 + pub fn genesis(genesis_time: u64, latest_eth1_data: Eth1Data, spec: &ChainSpec) -> BeaconState { let initial_crosslink = Crosslink { epoch: spec.genesis_epoch, crosslink_data_root: spec.zero_hash, }; - Ok(BeaconState { - /* - * Misc - */ + BeaconState { + // Misc slot: spec.genesis_slot, genesis_time, - fork: Fork { - previous_version: spec.genesis_fork_version, - current_version: spec.genesis_fork_version, - epoch: spec.genesis_epoch, - }, + fork: Fork::genesis(spec), - /* - * Validator registry - */ + // Validator registry validator_registry: vec![], // Set later in the function. validator_balances: vec![], // Set later in the function. validator_registry_update_epoch: spec.genesis_epoch, - /* - * Randomness and committees - */ + // Randomness and committees latest_randao_mixes: vec![spec.zero_hash; spec.latest_randao_mixes_length as usize], previous_shuffling_start_shard: spec.genesis_start_shard, current_shuffling_start_shard: spec.genesis_start_shard, @@ -155,26 +161,25 @@ impl BeaconState { previous_shuffling_seed: spec.zero_hash, current_shuffling_seed: spec.zero_hash, - /* - * Finality - */ + // Finality + previous_epoch_attestations: vec![], + current_epoch_attestations: vec![], previous_justified_epoch: spec.genesis_epoch, - justified_epoch: spec.genesis_epoch, + current_justified_epoch: spec.genesis_epoch, + previous_justified_root: spec.zero_hash, + current_justified_root: spec.zero_hash, justification_bitfield: 0, finalized_epoch: spec.genesis_epoch, + finalized_root: spec.zero_hash, - /* - * Recent state - */ + // Recent state latest_crosslinks: vec![initial_crosslink; spec.shard_count as usize], - latest_block_roots: vec![spec.zero_hash; spec.latest_block_roots_length as usize], - latest_active_index_roots: vec![ - spec.zero_hash; - spec.latest_active_index_roots_length as usize - ], - latest_slashed_balances: vec![0; spec.latest_slashed_exit_length as usize], - latest_attestations: vec![], - batched_block_roots: vec![], + latest_block_roots: vec![spec.zero_hash; spec.slots_per_historical_root], + latest_state_roots: vec![spec.zero_hash; spec.slots_per_historical_root], + latest_active_index_roots: vec![spec.zero_hash; spec.latest_active_index_roots_length], + latest_slashed_balances: vec![0; spec.latest_slashed_exit_length], + latest_block_header: BeaconBlock::empty(spec).into_temporary_header(spec), + historical_roots: vec![], /* * PoW receipt root @@ -187,53 +192,19 @@ impl BeaconState { * Caching (not in spec) */ cache_index_offset: 0, - caches: vec![EpochCache::empty(); CACHED_EPOCHS], - }) - } - - /// Produce the first state of the Beacon Chain. - pub fn genesis( - genesis_time: u64, - initial_validator_deposits: Vec, - latest_eth1_data: Eth1Data, - spec: &ChainSpec, - ) -> Result { - let mut genesis_state = - BeaconState::genesis_without_validators(genesis_time, latest_eth1_data, spec)?; - - debug!("Processing genesis deposits..."); - - let deposit_data = initial_validator_deposits - .par_iter() - .map(|deposit| &deposit.deposit_data) - .collect(); - - genesis_state.process_deposits(deposit_data, spec); - - trace!("Processed genesis deposits."); - - for validator_index in 0..genesis_state.validator_registry.len() { - if genesis_state.get_effective_balance(validator_index, spec) >= spec.max_deposit_amount - { - genesis_state.activate_validator(validator_index, true, spec); - } + caches: [ + EpochCache::default(), + EpochCache::default(), + EpochCache::default(), + EpochCache::default(), + ], + pubkey_cache: PubkeyCache::default(), } - - genesis_state.deposit_index = initial_validator_deposits.len() as u64; - - let genesis_active_index_root = hash_tree_root(get_active_validator_indices( - &genesis_state.validator_registry, - spec.genesis_epoch, - )); - genesis_state.latest_active_index_roots = - vec![genesis_active_index_root; spec.latest_active_index_roots_length]; - genesis_state.current_shuffling_seed = - genesis_state.generate_seed(spec.genesis_epoch, spec)?; - - Ok(genesis_state) } /// Returns the `hash_tree_root` of the state. + /// + /// Spec v0.5.0 pub fn canonical_root(&self) -> Hash256 { Hash256::from_slice(&self.hash_tree_root()[..]) } @@ -246,7 +217,8 @@ impl BeaconState { ) -> Result<(), Error> { let cache_index = self.cache_index(relative_epoch); - if self.caches[cache_index].initialized { + if self.caches[cache_index].initialized_epoch == Some(self.slot.epoch(spec.slots_per_epoch)) + { Ok(()) } else { self.force_build_epoch_cache(relative_epoch, spec) @@ -259,36 +231,13 @@ impl BeaconState { relative_epoch: RelativeEpoch, spec: &ChainSpec, ) -> Result<(), Error> { - let epoch = self.absolute_epoch(relative_epoch, spec); let cache_index = self.cache_index(relative_epoch); - self.caches[cache_index] = EpochCache::initialized(&self, epoch, spec)?; + self.caches[cache_index] = EpochCache::initialized(&self, relative_epoch, spec)?; Ok(()) } - /// Converts a `RelativeEpoch` into an `Epoch` with respect to the epoch of this state. - fn absolute_epoch(&self, relative_epoch: RelativeEpoch, spec: &ChainSpec) -> Epoch { - match relative_epoch { - RelativeEpoch::Previous => self.previous_epoch(spec), - RelativeEpoch::Current => self.current_epoch(spec), - RelativeEpoch::Next => self.next_epoch(spec), - } - } - - /// Converts an `Epoch` into a `RelativeEpoch` with respect to the epoch of this state. - /// - /// Returns an error if the given `epoch` not "previous", "current" or "next" compared to the - /// epoch of this tate. - fn relative_epoch(&self, epoch: Epoch, spec: &ChainSpec) -> Result { - match epoch { - e if e == self.current_epoch(spec) => Ok(RelativeEpoch::Current), - e if e == self.previous_epoch(spec) => Ok(RelativeEpoch::Previous), - e if e == self.next_epoch(spec) => Ok(RelativeEpoch::Next), - _ => Err(Error::EpochOutOfBounds), - } - } - /// Advances the cache for this state into the next epoch. /// /// This should be used if the `slot` of this state is advanced beyond an epoch boundary. @@ -309,15 +258,16 @@ impl BeaconState { /// Removes the specified cache and sets it to uninitialized. pub fn drop_cache(&mut self, relative_epoch: RelativeEpoch) { let previous_cache_index = self.cache_index(relative_epoch); - self.caches[previous_cache_index] = EpochCache::empty(); + self.caches[previous_cache_index] = EpochCache::default(); } /// Returns the index of `self.caches` for some `RelativeEpoch`. fn cache_index(&self, relative_epoch: RelativeEpoch) -> usize { let base_index = match relative_epoch { - RelativeEpoch::Current => 1, RelativeEpoch::Previous => 0, - RelativeEpoch::Next => 2, + RelativeEpoch::Current => 1, + RelativeEpoch::NextWithoutRegistryChange => 2, + RelativeEpoch::NextWithRegistryChange => 3, }; (base_index + self.cache_index_offset) % CACHED_EPOCHS @@ -325,19 +275,61 @@ impl BeaconState { /// Returns the cache for some `RelativeEpoch`. Returns an error if the cache has not been /// initialized. - fn cache(&self, relative_epoch: RelativeEpoch) -> Result<&EpochCache, Error> { + fn cache(&self, relative_epoch: RelativeEpoch, spec: &ChainSpec) -> Result<&EpochCache, Error> { let cache = &self.caches[self.cache_index(relative_epoch)]; - if cache.initialized { + let epoch = relative_epoch.into_epoch(self.slot.epoch(spec.slots_per_epoch)); + + if cache.initialized_epoch == Some(epoch) { Ok(cache) } else { Err(Error::EpochCacheUninitialized(relative_epoch)) } } + /// Updates the pubkey cache, if required. + /// + /// Adds all `pubkeys` from the `validator_registry` which are not already in the cache. Will + /// never re-add a pubkey. + pub fn update_pubkey_cache(&mut self) -> Result<(), Error> { + for (i, validator) in self + .validator_registry + .iter() + .enumerate() + .skip(self.pubkey_cache.len()) + { + let success = self.pubkey_cache.insert(validator.pubkey.clone(), i); + if !success { + return Err(Error::PubkeyCacheInconsistent); + } + } + + Ok(()) + } + + /// Completely drops the `pubkey_cache`, replacing it with a new, empty cache. + pub fn drop_pubkey_cache(&mut self) { + self.pubkey_cache = PubkeyCache::default() + } + + /// If a validator pubkey exists in the validator registry, returns `Some(i)`, otherwise + /// returns `None`. + /// + /// Requires a fully up-to-date `pubkey_cache`, returns an error if this is not the case. + pub fn get_validator_index(&self, pubkey: &PublicKey) -> Result, Error> { + if self.pubkey_cache.len() == self.validator_registry.len() { + Ok(self.pubkey_cache.get(pubkey)) + } else { + Err(Error::PubkeyCacheIncomplete { + cache_len: self.pubkey_cache.len(), + registry_len: self.validator_registry.len(), + }) + } + } + /// The epoch corresponding to `self.slot`. /// - /// Spec v0.4.0 + /// Spec v0.5.0 pub fn current_epoch(&self, spec: &ChainSpec) -> Epoch { self.slot.epoch(spec.slots_per_epoch) } @@ -346,95 +338,166 @@ impl BeaconState { /// /// If the current epoch is the genesis epoch, the genesis_epoch is returned. /// - /// Spec v0.4.0 + /// Spec v0.5.0 pub fn previous_epoch(&self, spec: &ChainSpec) -> Epoch { - let current_epoch = self.current_epoch(&spec); - std::cmp::max(current_epoch - 1, spec.genesis_epoch) + self.current_epoch(&spec) - 1 } /// The epoch following `self.current_epoch()`. /// - /// Spec v0.4.0 + /// Spec v0.5.0 pub fn next_epoch(&self, spec: &ChainSpec) -> Epoch { - self.current_epoch(spec).saturating_add(1_u64) + self.current_epoch(spec) + 1 } - /// The first slot of the epoch corresponding to `self.slot`. + /// Returns the active validator indices for the given epoch, assuming there is no validator + /// registry update in the next epoch. /// - /// Spec v0.4.0 - pub fn current_epoch_start_slot(&self, spec: &ChainSpec) -> Slot { - self.current_epoch(spec).start_slot(spec.slots_per_epoch) - } + /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. + /// + /// Spec v0.5.0 + pub fn get_active_validator_indices( + &self, + epoch: Epoch, + spec: &ChainSpec, + ) -> Result<&[usize], Error> { + // If the slot is in the next epoch, assume there was no validator registry update. + let relative_epoch = + match RelativeEpoch::from_epoch(self.slot.epoch(spec.slots_per_epoch), epoch) { + Err(RelativeEpochError::AmbiguiousNextEpoch) => { + Ok(RelativeEpoch::NextWithoutRegistryChange) + } + e => e, + }?; - /// The first slot of the epoch preceding the one corresponding to `self.slot`. - /// - /// Spec v0.4.0 - pub fn previous_epoch_start_slot(&self, spec: &ChainSpec) -> Slot { - self.previous_epoch(spec).start_slot(spec.slots_per_epoch) - } + let cache = self.cache(relative_epoch, spec)?; - /// Return the number of committees in the previous epoch. - /// - /// Spec v0.4.0 - fn get_previous_epoch_committee_count(&self, spec: &ChainSpec) -> u64 { - let previous_active_validators = - get_active_validator_indices(&self.validator_registry, self.previous_shuffling_epoch); - spec.get_epoch_committee_count(previous_active_validators.len()) - } - - /// Return the number of committees in the current epoch. - /// - /// Spec v0.4.0 - pub fn get_current_epoch_committee_count(&self, spec: &ChainSpec) -> u64 { - let current_active_validators = - get_active_validator_indices(&self.validator_registry, self.current_shuffling_epoch); - spec.get_epoch_committee_count(current_active_validators.len()) - } - - /// Return the number of committees in the next epoch. - /// - /// Spec v0.4.0 - pub fn get_next_epoch_committee_count(&self, spec: &ChainSpec) -> u64 { - let next_active_validators = - get_active_validator_indices(&self.validator_registry, self.next_epoch(spec)); - spec.get_epoch_committee_count(next_active_validators.len()) + Ok(&cache.active_validator_indices) } /// Returns the crosslink committees for some slot. /// /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. /// - /// Spec v0.4.0 + /// Spec v0.5.0 pub fn get_crosslink_committees_at_slot( &self, slot: Slot, spec: &ChainSpec, - ) -> Result<&CrosslinkCommittees, Error> { - let epoch = slot.epoch(spec.slots_per_epoch); - let relative_epoch = self.relative_epoch(epoch, spec)?; - let cache = self.cache(relative_epoch)?; + ) -> Result<&Vec, Error> { + // If the slot is in the next epoch, assume there was no validator registry update. + let relative_epoch = match RelativeEpoch::from_slot(self.slot, slot, spec) { + Err(RelativeEpochError::AmbiguiousNextEpoch) => { + Ok(RelativeEpoch::NextWithoutRegistryChange) + } + e => e, + }?; - let slot_offset = slot - epoch.start_slot(spec.slots_per_epoch); + let cache = self.cache(relative_epoch, spec)?; - Ok(&cache.committees[slot_offset.as_usize()]) + Ok(cache + .get_crosslink_committees_at_slot(slot, spec) + .ok_or_else(|| Error::SlotOutOfBounds)?) + } + + /// Returns the crosslink committees for some shard in an epoch. + /// + /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. + /// + /// Spec v0.4.0 + pub fn get_crosslink_committee_for_shard( + &self, + epoch: Epoch, + shard: Shard, + spec: &ChainSpec, + ) -> Result<&CrosslinkCommittee, Error> { + // If the slot is in the next epoch, assume there was no validator registry update. + let relative_epoch = match RelativeEpoch::from_epoch(self.current_epoch(spec), epoch) { + Err(RelativeEpochError::AmbiguiousNextEpoch) => { + Ok(RelativeEpoch::NextWithoutRegistryChange) + } + e => e, + }?; + + let cache = self.cache(relative_epoch, spec)?; + + Ok(cache + .get_crosslink_committee_for_shard(shard, spec) + .ok_or_else(|| Error::NoCommitteeForShard)?) + } + + /// Safely obtains the index for latest block roots, given some `slot`. + /// + /// Spec v0.5.0 + fn get_latest_block_roots_index(&self, slot: Slot, spec: &ChainSpec) -> Result { + if (slot < self.slot) && (self.slot <= slot + spec.slots_per_historical_root as u64) { + let i = slot.as_usize() % spec.slots_per_historical_root; + if i >= self.latest_block_roots.len() { + Err(Error::InsufficientStateRoots) + } else { + Ok(i) + } + } else { + Err(BeaconStateError::SlotOutOfBounds) + } } /// Return the block root at a recent `slot`. /// - /// Spec v0.4.0 - pub fn get_block_root(&self, slot: Slot, spec: &ChainSpec) -> Option<&Hash256> { - if (self.slot <= slot + spec.latest_block_roots_length as u64) && (slot < self.slot) { - self.latest_block_roots - .get(slot.as_usize() % spec.latest_block_roots_length) - } else { - None - } + /// Spec v0.5.0 + pub fn get_block_root( + &self, + slot: Slot, + spec: &ChainSpec, + ) -> Result<&Hash256, BeaconStateError> { + let i = self.get_latest_block_roots_index(slot, spec)?; + Ok(&self.latest_block_roots[i]) + } + + /// Sets the block root for some given slot. + /// + /// Spec v0.5.0 + pub fn set_block_root( + &mut self, + slot: Slot, + block_root: Hash256, + spec: &ChainSpec, + ) -> Result<(), BeaconStateError> { + let i = self.get_latest_block_roots_index(slot, spec)?; + Ok(self.latest_block_roots[i] = block_root) + } + + /// XOR-assigns the existing `epoch` randao mix with the hash of the `signature`. + /// + /// # Errors: + /// + /// See `Self::get_randao_mix`. + /// + /// Spec v0.5.0 + pub fn update_randao_mix( + &mut self, + epoch: Epoch, + signature: &Signature, + spec: &ChainSpec, + ) -> Result<(), Error> { + let i = epoch.as_usize() % spec.latest_randao_mixes_length; + + let signature_hash = Hash256::from_slice(&hash(&ssz_encode(signature))); + + self.latest_randao_mixes[i] = *self.get_randao_mix(epoch, spec)? ^ signature_hash; + + Ok(()) } /// Return the randao mix at a recent ``epoch``. /// - /// Spec v0.4.0 - pub fn get_randao_mix(&self, epoch: Epoch, spec: &ChainSpec) -> Option<&Hash256> { + /// # Errors: + /// - `InsufficientRandaoMixes` if `self.latest_randao_mixes` is shorter than + /// `spec.latest_randao_mixes_length`. + /// - `EpochOutOfBounds` if the state no longer stores randao mixes for the given `epoch`. + /// + /// Spec v0.5.0 + pub fn get_randao_mix(&self, epoch: Epoch, spec: &ChainSpec) -> Result<&Hash256, Error> { let current_epoch = self.current_epoch(spec); if (current_epoch - (spec.latest_randao_mixes_length as u64) < epoch) @@ -442,8 +505,9 @@ impl BeaconState { { self.latest_randao_mixes .get(epoch.as_usize() % spec.latest_randao_mixes_length) + .ok_or_else(|| Error::InsufficientRandaoMixes) } else { - None + Err(Error::EpochOutOfBounds) } } @@ -467,13 +531,49 @@ impl BeaconState { } } + /// Safely obtains the index for latest state roots, given some `slot`. + /// + /// Spec v0.5.0 + fn get_latest_state_roots_index(&self, slot: Slot, spec: &ChainSpec) -> Result { + if (slot < self.slot) && (self.slot <= slot + spec.slots_per_historical_root as u64) { + let i = slot.as_usize() % spec.slots_per_historical_root; + if i >= self.latest_state_roots.len() { + Err(Error::InsufficientStateRoots) + } else { + Ok(i) + } + } else { + Err(BeaconStateError::SlotOutOfBounds) + } + } + + /// Gets the state root for some slot. + /// + /// Spec v0.5.0 + pub fn get_state_root(&mut self, slot: Slot, spec: &ChainSpec) -> Result<&Hash256, Error> { + let i = self.get_latest_state_roots_index(slot, spec)?; + Ok(&self.latest_state_roots[i]) + } + + /// Sets the latest state root for slot. + /// + /// Spec v0.5.0 + pub fn set_state_root( + &mut self, + slot: Slot, + state_root: Hash256, + spec: &ChainSpec, + ) -> Result<(), Error> { + let i = self.get_latest_state_roots_index(slot, spec)?; + Ok(self.latest_state_roots[i] = state_root) + } + /// Generate a seed for the given `epoch`. /// /// Spec v0.4.0 pub fn generate_seed(&self, epoch: Epoch, spec: &ChainSpec) -> Result { let mut input = self - .get_randao_mix(epoch - spec.min_seed_lookahead, spec) - .ok_or_else(|| Error::InsufficientRandaoMixes)? + .get_randao_mix(epoch - spec.min_seed_lookahead, spec)? .as_bytes() .to_vec(); @@ -494,80 +594,46 @@ impl BeaconState { /// /// If the state does not contain an index for a beacon proposer at the requested `slot`, then `None` is returned. /// - /// Spec v0.4.0 - pub fn get_beacon_proposer_index(&self, slot: Slot, spec: &ChainSpec) -> Result { - let committees = self.get_crosslink_committees_at_slot(slot, spec)?; - trace!( - "get_beacon_proposer_index: slot: {}, committees_count: {}", - slot, - committees.len() - ); + /// Spec v0.5.0 + pub fn get_beacon_proposer_index( + &self, + slot: Slot, + relative_epoch: RelativeEpoch, + spec: &ChainSpec, + ) -> Result { + let cache = self.cache(relative_epoch, spec)?; + + let committees = cache + .get_crosslink_committees_at_slot(slot, spec) + .ok_or_else(|| Error::SlotOutOfBounds)?; + + let epoch = slot.epoch(spec.slots_per_epoch); + committees .first() - .ok_or(Error::InsufficientValidators) - .and_then(|(first_committee, _)| { - let index = slot + .ok_or(Error::UnableToDetermineProducer) + .and_then(|first| { + let index = epoch .as_usize() - .checked_rem(first_committee.len()) - .ok_or(Error::InsufficientValidators)?; - Ok(first_committee[index]) + .checked_rem(first.committee.len()) + .ok_or(Error::UnableToDetermineProducer)?; + Ok(first.committee[index]) }) } - /// Returns the list of validator indices which participiated in the attestation. - /// - /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. - /// - /// Spec v0.4.0 - pub fn get_attestation_participants( - &self, - attestation_data: &AttestationData, - bitfield: &Bitfield, - spec: &ChainSpec, - ) -> Result, Error> { - let epoch = attestation_data.slot.epoch(spec.slots_per_epoch); - let relative_epoch = self.relative_epoch(epoch, spec)?; - let cache = self.cache(relative_epoch)?; - - let (committee_slot_index, committee_index) = cache - .shard_committee_index_map - .get(&attestation_data.shard) - .ok_or_else(|| Error::ShardOutOfBounds)?; - let (committee, shard) = &cache.committees[*committee_slot_index][*committee_index]; - - assert_eq!(*shard, attestation_data.shard, "Bad epoch cache build."); - - if !verify_bitfield_length(&bitfield, committee.len()) { - return Err(Error::InvalidBitfield); - } - - let mut participants = vec![]; - for (i, validator_index) in committee.iter().enumerate() { - if bitfield.get(i).unwrap() { - participants.push(*validator_index); - } - } - - Ok(participants) - } - /// Return the effective balance (also known as "balance at stake") for a validator with the given ``index``. /// /// Spec v0.4.0 - pub fn get_effective_balance(&self, validator_index: usize, spec: &ChainSpec) -> u64 { - std::cmp::min( - self.validator_balances[validator_index], - spec.max_deposit_amount, - ) - } - - /// Return the combined effective balance of an array of validators. - /// - /// Spec v0.4.0 - pub fn get_total_balance(&self, validator_indices: &[usize], spec: &ChainSpec) -> u64 { - validator_indices - .iter() - .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)) + pub fn get_effective_balance( + &self, + validator_index: usize, + spec: &ChainSpec, + ) -> Result { + let balance = self + .validator_balances + .get(validator_index) + .ok_or_else(|| Error::UnknownValidator)?; + Ok(std::cmp::min(*balance, spec.max_deposit_amount)) } /// Return the epoch at which an activation or exit triggered in ``epoch`` takes effect. @@ -577,99 +643,9 @@ impl BeaconState { epoch + 1 + spec.activation_exit_delay } - /// Process multiple deposits in sequence. - /// - /// Builds a hashmap of validator pubkeys to validator index and passes it to each successive - /// call to `process_deposit(..)`. This requires much less computation than successive calls to - /// `process_deposits(..)` without the hashmap. - /// - /// Spec v0.4.0 - pub fn process_deposits( - &mut self, - deposits: Vec<&DepositData>, - spec: &ChainSpec, - ) -> Vec { - let mut added_indices = vec![]; - let mut pubkey_map: HashMap = HashMap::new(); - - for (i, validator) in self.validator_registry.iter().enumerate() { - pubkey_map.insert(validator.pubkey.clone(), i); - } - - for deposit_data in deposits { - let result = self.process_deposit( - deposit_data.deposit_input.pubkey.clone(), - deposit_data.amount, - deposit_data.deposit_input.proof_of_possession.clone(), - deposit_data.deposit_input.withdrawal_credentials, - Some(&pubkey_map), - spec, - ); - if let Ok(index) = result { - added_indices.push(index); - } - } - added_indices - } - - /// Process a validator deposit, returning the validator index if the deposit is valid. - /// - /// Optionally accepts a hashmap of all validator pubkeys to their validator index. Without - /// this hashmap, each call to `process_deposits` requires an iteration though - /// `self.validator_registry`. This becomes highly inefficient at scale. - /// - /// Spec v0.4.0 - pub fn process_deposit( - &mut self, - pubkey: PublicKey, - amount: u64, - proof_of_possession: Signature, - withdrawal_credentials: Hash256, - pubkey_map: Option<&HashMap>, - spec: &ChainSpec, - ) -> Result { - // TODO: update proof of possession to function written above ( - // requires bls::create_proof_of_possession to be updated - // - // https://github.com/sigp/lighthouse/issues/239 - if !verify_proof_of_possession(&proof_of_possession, &pubkey) { - return Err(()); - } - - let validator_index = if let Some(pubkey_map) = pubkey_map { - pubkey_map.get(&pubkey).and_then(|i| Some(*i)) - } else { - self.validator_registry - .iter() - .position(|v| v.pubkey == pubkey) - }; - - if let Some(index) = validator_index { - if self.validator_registry[index].withdrawal_credentials == withdrawal_credentials { - safe_add_assign!(self.validator_balances[index], amount); - Ok(index) - } else { - Err(()) - } - } else { - let validator = Validator { - pubkey, - withdrawal_credentials, - activation_epoch: spec.far_future_epoch, - exit_epoch: spec.far_future_epoch, - withdrawable_epoch: spec.far_future_epoch, - initiated_exit: false, - slashed: false, - }; - self.validator_registry.push(validator); - self.validator_balances.push(amount); - Ok(self.validator_registry.len() - 1) - } - } - /// Activate the validator of the given ``index``. /// - /// Spec v0.4.0 + /// Spec v0.5.0 pub fn activate_validator( &mut self, validator_index: usize, @@ -687,7 +663,7 @@ impl BeaconState { /// Initiate an exit for the validator of the given `index`. /// - /// Spec v0.4.0 + /// Spec v0.5.0 pub fn initiate_validator_exit(&mut self, validator_index: usize) { self.validator_registry[validator_index].initiated_exit = true; } @@ -708,7 +684,7 @@ impl BeaconState { /// Slash the validator with index ``index``. /// - /// Spec v0.4.0 + /// Spec v0.5.0 pub fn slash_validator( &mut self, validator_index: usize, @@ -720,22 +696,27 @@ impl BeaconState { .validator_registry .get(validator_index) .ok_or_else(|| Error::UnknownValidator)?; + let effective_balance = self.get_effective_balance(validator_index, spec)?; + // A validator that is withdrawn cannot be slashed. + // + // This constraint will be lifted in Phase 0. if self.slot >= validator .withdrawable_epoch .start_slot(spec.slots_per_epoch) { - return Err(Error::SlotOutOfBounds); + return Err(Error::ValidatorIsWithdrawable); } self.exit_validator(validator_index, spec); - self.latest_slashed_balances[current_epoch.as_usize() % spec.latest_slashed_exit_length] += - self.get_effective_balance(validator_index, spec); + self.increment_current_epoch_slashed_balances(effective_balance, spec)?; + + let whistleblower_index = + self.get_beacon_proposer_index(self.slot, RelativeEpoch::Current, spec)?; + let whistleblower_reward = effective_balance / spec.whistleblower_reward_quotient; - let whistleblower_index = self.get_beacon_proposer_index(self.slot, spec)?; - let whistleblower_reward = self.get_effective_balance(validator_index, spec); safe_add_assign!( self.validator_balances[whistleblower_index as usize], whistleblower_reward @@ -744,14 +725,31 @@ impl BeaconState { self.validator_balances[validator_index], whistleblower_reward ); + self.validator_registry[validator_index].slashed = true; + self.validator_registry[validator_index].withdrawable_epoch = current_epoch + Epoch::from(spec.latest_slashed_exit_length); - debug!( - "Whistleblower {} penalized validator {}.", - whistleblower_index, validator_index - ); + Ok(()) + } + + /// Increment `self.latest_slashed_balances` with a slashing from the current epoch. + /// + /// Spec v0.5.0. + fn increment_current_epoch_slashed_balances( + &mut self, + increment: u64, + spec: &ChainSpec, + ) -> Result<(), Error> { + let current_epoch = self.current_epoch(spec); + + let slashed_balances_index = current_epoch.as_usize() % spec.latest_slashed_exit_length; + if slashed_balances_index >= self.latest_slashed_balances.len() { + return Err(Error::InsufficientSlashedBalances); + } + + self.latest_slashed_balances[slashed_balances_index] += increment; Ok(()) } @@ -765,175 +763,6 @@ impl BeaconState { self.current_epoch(spec) + spec.min_validator_withdrawability_delay; } - /// Returns the crosslink committees for some slot. - /// - /// Utilizes the cache and will fail if the appropriate cache is not initialized. - /// - /// Spec v0.4.0 - pub(crate) fn get_shuffling_for_slot( - &self, - slot: Slot, - registry_change: bool, - - spec: &ChainSpec, - ) -> Result>, Error> { - let (_committees_per_epoch, seed, shuffling_epoch, _shuffling_start_shard) = - self.get_committee_params_at_slot(slot, registry_change, spec)?; - - self.get_shuffling(seed, shuffling_epoch, spec) - } - - /// Shuffle ``validators`` into crosslink committees seeded by ``seed`` and ``epoch``. - /// - /// Return a list of ``committees_per_epoch`` committees where each - /// committee is itself a list of validator indices. - /// - /// Spec v0.4.0 - pub(crate) fn get_shuffling( - &self, - seed: Hash256, - epoch: Epoch, - spec: &ChainSpec, - ) -> Result>, Error> { - let active_validator_indices = - get_active_validator_indices(&self.validator_registry, epoch); - - if active_validator_indices.is_empty() { - error!("get_shuffling: no validators."); - return Err(Error::InsufficientValidators); - } - - debug!("Shuffling {} validators...", active_validator_indices.len()); - - let committees_per_epoch = spec.get_epoch_committee_count(active_validator_indices.len()); - - trace!( - "get_shuffling: active_validator_indices.len() == {}, committees_per_epoch: {}", - active_validator_indices.len(), - committees_per_epoch - ); - - let active_validator_indices: Vec = active_validator_indices.to_vec(); - - let shuffled_active_validator_indices = shuffle_list( - active_validator_indices, - spec.shuffle_round_count, - &seed[..], - true, - ) - .ok_or_else(|| Error::UnableToShuffle)?; - - Ok(shuffled_active_validator_indices - .honey_badger_split(committees_per_epoch as usize) - .map(|slice: &[usize]| slice.to_vec()) - .collect()) - } - - /// Returns the following params for the given slot: - /// - /// - epoch committee count - /// - epoch seed - /// - calculation epoch - /// - start shard - /// - /// In the spec, this functionality is included in the `get_crosslink_committees_at_slot(..)` - /// function. It is separated here to allow the division of shuffling and committee building, - /// as is required for efficient operations. - /// - /// Spec v0.4.0 - pub(crate) fn get_committee_params_at_slot( - &self, - slot: Slot, - registry_change: bool, - spec: &ChainSpec, - ) -> Result<(u64, Hash256, Epoch, u64), Error> { - let epoch = slot.epoch(spec.slots_per_epoch); - let current_epoch = self.current_epoch(spec); - let previous_epoch = self.previous_epoch(spec); - let next_epoch = self.next_epoch(spec); - - if epoch == current_epoch { - Ok(( - self.get_current_epoch_committee_count(spec), - self.current_shuffling_seed, - self.current_shuffling_epoch, - self.current_shuffling_start_shard, - )) - } else if epoch == previous_epoch { - Ok(( - self.get_previous_epoch_committee_count(spec), - self.previous_shuffling_seed, - self.previous_shuffling_epoch, - self.previous_shuffling_start_shard, - )) - } else if epoch == next_epoch { - let current_committees_per_epoch = self.get_current_epoch_committee_count(spec); - let epochs_since_last_registry_update = - current_epoch - self.validator_registry_update_epoch; - let (seed, shuffling_start_shard) = if registry_change { - let next_seed = self.generate_seed(next_epoch, spec)?; - ( - next_seed, - (self.current_shuffling_start_shard + current_committees_per_epoch) - % spec.shard_count, - ) - } else if (epochs_since_last_registry_update > 1) - & epochs_since_last_registry_update.is_power_of_two() - { - let next_seed = self.generate_seed(next_epoch, spec)?; - (next_seed, self.current_shuffling_start_shard) - } else { - ( - self.current_shuffling_seed, - self.current_shuffling_start_shard, - ) - }; - Ok(( - self.get_next_epoch_committee_count(spec), - seed, - next_epoch, - shuffling_start_shard, - )) - } else { - Err(Error::EpochOutOfBounds) - } - } - - /// Return the list of ``(committee, shard)`` tuples for the ``slot``. - /// - /// Note: There are two possible shufflings for crosslink committees for a - /// `slot` in the next epoch: with and without a `registry_change` - /// - /// Note: does not utilize the cache, `get_crosslink_committees_at_slot` is an equivalent - /// function which uses the cache. - /// - /// Spec v0.4.0 - pub(crate) fn calculate_crosslink_committees_at_slot( - &self, - slot: Slot, - registry_change: bool, - shuffling: Vec>, - spec: &ChainSpec, - ) -> Result, u64)>, Error> { - let (committees_per_epoch, _seed, _shuffling_epoch, shuffling_start_shard) = - self.get_committee_params_at_slot(slot, registry_change, spec)?; - - let offset = slot.as_u64() % spec.slots_per_epoch; - let committees_per_slot = committees_per_epoch / spec.slots_per_epoch; - let slot_start_shard = - (shuffling_start_shard + committees_per_slot * offset) % spec.shard_count; - - let mut crosslinks_at_slot = vec![]; - for i in 0..committees_per_slot { - let tuple = ( - shuffling[(committees_per_slot * offset + i) as usize].clone(), - (slot_start_shard + i) % spec.shard_count, - ); - crosslinks_at_slot.push(tuple) - } - Ok(crosslinks_at_slot) - } - /// Returns the `slot`, `shard` and `committee_index` for which a validator must produce an /// attestation. /// @@ -942,27 +771,28 @@ impl BeaconState { /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. /// /// Spec v0.4.0 - pub fn attestation_slot_and_shard_for_validator( + pub fn get_attestation_duties( &self, validator_index: usize, - _spec: &ChainSpec, - ) -> Result, Error> { - let cache = self.cache(RelativeEpoch::Current)?; + spec: &ChainSpec, + ) -> Result<&Option, Error> { + let cache = self.cache(RelativeEpoch::Current, spec)?; Ok(cache - .attestation_duty_map - .get(&(validator_index as u64)) - .and_then(|tuple| Some(*tuple))) + .attestation_duties + .get(validator_index) + .ok_or_else(|| Error::UnknownValidator)?) } - /// Process the slashings. + /// Process slashings. + /// + /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. /// /// Spec v0.4.0 - pub fn process_slashings(&mut self, spec: &ChainSpec) { + pub fn process_slashings(&mut self, spec: &ChainSpec) -> Result<(), Error> { let current_epoch = self.current_epoch(spec); - let active_validator_indices = - get_active_validator_indices(&self.validator_registry, current_epoch); - let total_balance = self.get_total_balance(&active_validator_indices[..], spec); + let active_validator_indices = self.get_active_validator_indices(current_epoch, spec)?; + let total_balance = self.get_total_balance(&active_validator_indices[..], spec)?; for (index, validator) in self.validator_registry.iter().enumerate() { if validator.slashed @@ -976,16 +806,19 @@ impl BeaconState { [(epoch_index + 1) % spec.latest_slashed_exit_length]; let total_at_end = self.latest_slashed_balances[epoch_index]; let total_penalities = total_at_end.saturating_sub(total_at_start); + + let effective_balance = self.get_effective_balance(index, spec)?; let penalty = std::cmp::max( - self.get_effective_balance(index, spec) - * std::cmp::min(total_penalities * 3, total_balance) + effective_balance * std::cmp::min(total_penalities * 3, total_balance) / total_balance, - self.get_effective_balance(index, spec) / spec.min_penalty_quotient, + effective_balance / spec.min_penalty_quotient, ); safe_sub_assign!(self.validator_balances[index], penalty); } } + + Ok(()) } /// Process the exit queue. @@ -1019,12 +852,13 @@ impl BeaconState { /// Update validator registry, activating/exiting validators if possible. /// + /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. + /// /// Spec v0.4.0 - pub fn update_validator_registry(&mut self, spec: &ChainSpec) { + pub fn update_validator_registry(&mut self, spec: &ChainSpec) -> Result<(), Error> { let current_epoch = self.current_epoch(spec); - let active_validator_indices = - get_active_validator_indices(&self.validator_registry, current_epoch); - let total_balance = self.get_total_balance(&active_validator_indices[..], spec); + let active_validator_indices = self.get_active_validator_indices(current_epoch, spec)?; + let total_balance = self.get_total_balance(&active_validator_indices[..], spec)?; let max_balance_churn = std::cmp::max( spec.max_deposit_amount, @@ -1038,7 +872,7 @@ impl BeaconState { if (validator.activation_epoch == spec.far_future_epoch) & (self.validator_balances[index] == spec.max_deposit_amount) { - balance_churn += self.get_effective_balance(index, spec); + balance_churn += self.get_effective_balance(index, spec)?; if balance_churn > max_balance_churn { break; } @@ -1051,7 +885,7 @@ impl BeaconState { let validator = &self.validator_registry[index]; if (validator.exit_epoch == spec.far_future_epoch) & (validator.initiated_exit) { - balance_churn += self.get_effective_balance(index, spec); + balance_churn += self.get_effective_balance(index, spec)?; if balance_churn > max_balance_churn { break; } @@ -1061,278 +895,59 @@ impl BeaconState { } self.validator_registry_update_epoch = current_epoch; - } - /// Confirm validator owns PublicKey - /// - /// Spec v0.4.0 - pub fn validate_proof_of_possession( - &self, - pubkey: PublicKey, - proof_of_possession: Signature, - withdrawal_credentials: Hash256, - spec: &ChainSpec, - ) -> bool { - let proof_of_possession_data = DepositInput { - pubkey: pubkey.clone(), - withdrawal_credentials, - proof_of_possession: Signature::empty_signature(), - }; - - proof_of_possession.verify( - &proof_of_possession_data.hash_tree_root(), - spec.get_domain( - self.slot.epoch(spec.slots_per_epoch), - Domain::Deposit, - &self.fork, - ), - &pubkey, - ) + Ok(()) } /// Iterate through the validator registry and eject active validators with balance below /// ``EJECTION_BALANCE``. /// - /// Spec v0.4.0 - pub fn process_ejections(&mut self, spec: &ChainSpec) { - for validator_index in - get_active_validator_indices(&self.validator_registry, self.current_epoch(spec)) - { - if self.validator_balances[validator_index] < spec.ejection_balance { - self.exit_validator(validator_index, spec) - } - } - } - - /// Returns the penality that should be applied to some validator for inactivity. - /// - /// Note: this is defined "inline" in the spec, not as a helper function. - /// - /// Spec v0.4.0 - pub fn inactivity_penalty( - &self, - validator_index: usize, - epochs_since_finality: Epoch, - base_reward_quotient: u64, - spec: &ChainSpec, - ) -> u64 { - let effective_balance = self.get_effective_balance(validator_index, spec); - self.base_reward(validator_index, base_reward_quotient, spec) - + effective_balance * epochs_since_finality.as_u64() - / spec.inactivity_penalty_quotient - / 2 - } - - /// Returns the base reward for some validator. - /// - /// Note: In the spec this is defined "inline", not as a helper function. - /// - /// Spec v0.4.0 - pub fn base_reward( - &self, - validator_index: usize, - base_reward_quotient: u64, - spec: &ChainSpec, - ) -> u64 { - self.get_effective_balance(validator_index, spec) / base_reward_quotient / 5 - } - - /// Returns the union of all participants in the provided attestations - /// - /// Spec v0.4.0 - pub fn get_attestation_participants_union( - &self, - attestations: &[&PendingAttestation], - spec: &ChainSpec, - ) -> Result, Error> { - let mut all_participants = attestations + /// Spec v0.5.0 + pub fn process_ejections(&mut self, spec: &ChainSpec) -> Result<(), Error> { + // There is an awkward double (triple?) loop here because we can't loop across the borrowed + // active validator indices and mutate state in the one loop. + let exitable: Vec = self + .get_active_validator_indices(self.current_epoch(spec), spec)? .iter() - .try_fold::<_, _, Result, Error>>(vec![], |mut acc, a| { - acc.append(&mut self.get_attestation_participants( - &a.data, - &a.aggregation_bitfield, - spec, - )?); - Ok(acc) - })?; - all_participants.sort_unstable(); - all_participants.dedup(); - Ok(all_participants) - } -} + .filter_map(|&i| { + if self.validator_balances[i as usize] < spec.ejection_balance { + Some(i) + } else { + None + } + }) + .collect(); -fn hash_tree_root(input: Vec) -> Hash256 { - Hash256::from_slice(&input.hash_tree_root()[..]) -} - -impl Encodable for BeaconState { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.slot); - s.append(&self.genesis_time); - s.append(&self.fork); - s.append(&self.validator_registry); - s.append(&self.validator_balances); - s.append(&self.validator_registry_update_epoch); - s.append(&self.latest_randao_mixes); - s.append(&self.previous_shuffling_start_shard); - s.append(&self.current_shuffling_start_shard); - s.append(&self.previous_shuffling_epoch); - s.append(&self.current_shuffling_epoch); - s.append(&self.previous_shuffling_seed); - s.append(&self.current_shuffling_seed); - s.append(&self.previous_justified_epoch); - s.append(&self.justified_epoch); - s.append(&self.justification_bitfield); - s.append(&self.finalized_epoch); - s.append(&self.latest_crosslinks); - s.append(&self.latest_block_roots); - s.append(&self.latest_active_index_roots); - s.append(&self.latest_slashed_balances); - s.append(&self.latest_attestations); - s.append(&self.batched_block_roots); - s.append(&self.latest_eth1_data); - s.append(&self.eth1_data_votes); - s.append(&self.deposit_index); - } -} - -impl Decodable for BeaconState { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (slot, i) = <_>::ssz_decode(bytes, i)?; - let (genesis_time, i) = <_>::ssz_decode(bytes, i)?; - let (fork, i) = <_>::ssz_decode(bytes, i)?; - let (validator_registry, i) = <_>::ssz_decode(bytes, i)?; - let (validator_balances, i) = <_>::ssz_decode(bytes, i)?; - let (validator_registry_update_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (latest_randao_mixes, i) = <_>::ssz_decode(bytes, i)?; - let (previous_shuffling_start_shard, i) = <_>::ssz_decode(bytes, i)?; - let (current_shuffling_start_shard, i) = <_>::ssz_decode(bytes, i)?; - let (previous_shuffling_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (current_shuffling_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (previous_shuffling_seed, i) = <_>::ssz_decode(bytes, i)?; - let (current_shuffling_seed, i) = <_>::ssz_decode(bytes, i)?; - let (previous_justified_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (justified_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (justification_bitfield, i) = <_>::ssz_decode(bytes, i)?; - let (finalized_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (latest_crosslinks, i) = <_>::ssz_decode(bytes, i)?; - let (latest_block_roots, i) = <_>::ssz_decode(bytes, i)?; - let (latest_active_index_roots, i) = <_>::ssz_decode(bytes, i)?; - let (latest_slashed_balances, i) = <_>::ssz_decode(bytes, i)?; - let (latest_attestations, i) = <_>::ssz_decode(bytes, i)?; - let (batched_block_roots, i) = <_>::ssz_decode(bytes, i)?; - let (latest_eth1_data, i) = <_>::ssz_decode(bytes, i)?; - let (eth1_data_votes, i) = <_>::ssz_decode(bytes, i)?; - let (deposit_index, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - Self { - slot, - genesis_time, - fork, - validator_registry, - validator_balances, - validator_registry_update_epoch, - latest_randao_mixes, - previous_shuffling_start_shard, - current_shuffling_start_shard, - previous_shuffling_epoch, - current_shuffling_epoch, - previous_shuffling_seed, - current_shuffling_seed, - previous_justified_epoch, - justified_epoch, - justification_bitfield, - finalized_epoch, - latest_crosslinks, - latest_block_roots, - latest_active_index_roots, - latest_slashed_balances, - latest_attestations, - batched_block_roots, - latest_eth1_data, - eth1_data_votes, - deposit_index, - cache_index_offset: 0, - caches: vec![EpochCache::empty(); CACHED_EPOCHS], - }, - i, - )) - } -} - -impl TreeHash for BeaconState { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.slot.hash_tree_root_internal()); - result.append(&mut self.genesis_time.hash_tree_root_internal()); - result.append(&mut self.fork.hash_tree_root_internal()); - result.append(&mut self.validator_registry.hash_tree_root_internal()); - result.append(&mut self.validator_balances.hash_tree_root_internal()); - result.append( - &mut self - .validator_registry_update_epoch - .hash_tree_root_internal(), - ); - result.append(&mut self.latest_randao_mixes.hash_tree_root_internal()); - result.append( - &mut self - .previous_shuffling_start_shard - .hash_tree_root_internal(), - ); - result.append(&mut self.current_shuffling_start_shard.hash_tree_root_internal()); - result.append(&mut self.previous_shuffling_epoch.hash_tree_root_internal()); - result.append(&mut self.current_shuffling_epoch.hash_tree_root_internal()); - result.append(&mut self.previous_shuffling_seed.hash_tree_root_internal()); - result.append(&mut self.current_shuffling_seed.hash_tree_root_internal()); - result.append(&mut self.previous_justified_epoch.hash_tree_root_internal()); - result.append(&mut self.justified_epoch.hash_tree_root_internal()); - result.append(&mut self.justification_bitfield.hash_tree_root_internal()); - result.append(&mut self.finalized_epoch.hash_tree_root_internal()); - result.append(&mut self.latest_crosslinks.hash_tree_root_internal()); - result.append(&mut self.latest_block_roots.hash_tree_root_internal()); - result.append(&mut self.latest_active_index_roots.hash_tree_root_internal()); - result.append(&mut self.latest_slashed_balances.hash_tree_root_internal()); - result.append(&mut self.latest_attestations.hash_tree_root_internal()); - result.append(&mut self.batched_block_roots.hash_tree_root_internal()); - result.append(&mut self.latest_eth1_data.hash_tree_root_internal()); - result.append(&mut self.eth1_data_votes.hash_tree_root_internal()); - result.append(&mut self.deposit_index.hash_tree_root_internal()); - hash(&result) - } -} - -impl TestRandom for BeaconState { - fn random_for_test(rng: &mut T) -> Self { - Self { - slot: <_>::random_for_test(rng), - genesis_time: <_>::random_for_test(rng), - fork: <_>::random_for_test(rng), - validator_registry: <_>::random_for_test(rng), - validator_balances: <_>::random_for_test(rng), - validator_registry_update_epoch: <_>::random_for_test(rng), - latest_randao_mixes: <_>::random_for_test(rng), - previous_shuffling_start_shard: <_>::random_for_test(rng), - current_shuffling_start_shard: <_>::random_for_test(rng), - previous_shuffling_epoch: <_>::random_for_test(rng), - current_shuffling_epoch: <_>::random_for_test(rng), - previous_shuffling_seed: <_>::random_for_test(rng), - current_shuffling_seed: <_>::random_for_test(rng), - previous_justified_epoch: <_>::random_for_test(rng), - justified_epoch: <_>::random_for_test(rng), - justification_bitfield: <_>::random_for_test(rng), - finalized_epoch: <_>::random_for_test(rng), - latest_crosslinks: <_>::random_for_test(rng), - latest_block_roots: <_>::random_for_test(rng), - latest_active_index_roots: <_>::random_for_test(rng), - latest_slashed_balances: <_>::random_for_test(rng), - latest_attestations: <_>::random_for_test(rng), - batched_block_roots: <_>::random_for_test(rng), - latest_eth1_data: <_>::random_for_test(rng), - eth1_data_votes: <_>::random_for_test(rng), - deposit_index: <_>::random_for_test(rng), - cache_index_offset: 0, - caches: vec![EpochCache::empty(); CACHED_EPOCHS], + for validator_index in exitable { + self.exit_validator(validator_index, spec) } + + Ok(()) + } + + /// Return the combined effective balance of an array of validators. + /// + /// Spec v0.5.0 + pub fn get_total_balance( + &self, + validator_indices: &[usize], + spec: &ChainSpec, + ) -> Result { + validator_indices.iter().try_fold(0_u64, |acc, i| { + self.get_effective_balance(*i, spec) + .and_then(|bal| Ok(bal + acc)) + }) + } +} + +impl From for Error { + fn from(e: RelativeEpochError) -> Error { + Error::RelativeEpochError(e) + } +} + +impl From for Error { + fn from(e: EpochCacheError) -> Error { + Error::EpochCacheError(e) } } diff --git a/eth2/types/src/beacon_state/builder.rs b/eth2/types/src/beacon_state/builder.rs deleted file mode 100644 index 0be297db7..000000000 --- a/eth2/types/src/beacon_state/builder.rs +++ /dev/null @@ -1,263 +0,0 @@ -use crate::*; -use bls::create_proof_of_possession; - -/// Builds a `BeaconState` for use in testing or benchmarking. -/// -/// Building the `BeaconState` is a three step processes: -/// -/// 1. Create a new `BeaconStateBuilder`. -/// 2. Call `Self::build()` or `Self::build_fast()` generate a `BeaconState`. -/// 3. (Optional) Use builder functions to modify the `BeaconState`. -/// 4. Call `Self::cloned_state()` to obtain a `BeaconState` cloned from this struct. -/// -/// Step (2) happens prior to step (3) because some functionality requires an existing -/// `BeaconState`. -/// -/// Step (4) produces a clone of the BeaconState and doesn't consume the `BeaconStateBuilder` to -/// allow access to `self.keypairs` and `self.spec`. -pub struct BeaconStateBuilder { - pub validator_count: usize, - pub state: Option, - pub genesis_time: u64, - pub latest_eth1_data: Eth1Data, - pub spec: ChainSpec, - pub keypairs: Vec, -} - -impl BeaconStateBuilder { - /// Create a new builder with the given number of validators. - pub fn new(validator_count: usize) -> Self { - let genesis_time = 10_000_000; - - let latest_eth1_data = Eth1Data { - deposit_root: Hash256::zero(), - block_hash: Hash256::zero(), - }; - - let spec = ChainSpec::foundation(); - - Self { - validator_count, - state: None, - genesis_time, - latest_eth1_data, - spec, - keypairs: vec![], - } - } - - /// Builds a `BeaconState` using the `BeaconState::genesis(..)` function. - /// - /// Each validator is assigned a unique, randomly-generated keypair and all - /// proof-of-possessions are verified during genesis. - pub fn build(&mut self) -> Result<(), BeaconStateError> { - self.keypairs = (0..self.validator_count) - .collect::>() - .iter() - .map(|_| Keypair::random()) - .collect(); - - let initial_validator_deposits = self - .keypairs - .iter() - .map(|keypair| Deposit { - branch: vec![], // branch verification is not specified. - index: 0, // index verification is not specified. - deposit_data: DepositData { - amount: 32_000_000_000, // 32 ETH (in Gwei) - timestamp: self.genesis_time - 1, - deposit_input: DepositInput { - pubkey: keypair.pk.clone(), - withdrawal_credentials: Hash256::zero(), // Withdrawal not possible. - proof_of_possession: create_proof_of_possession(&keypair), - }, - }, - }) - .collect(); - - let state = BeaconState::genesis( - self.genesis_time, - initial_validator_deposits, - self.latest_eth1_data.clone(), - &self.spec, - )?; - - self.state = Some(state); - - Ok(()) - } - - /// Builds a `BeaconState` using the `BeaconState::genesis(..)` function, without supplying any - /// validators. Instead validators are added to the state post-genesis. - /// - /// One keypair is randomly generated and all validators are assigned this same keypair. - /// Proof-of-possessions are not created (or validated). - /// - /// This function runs orders of magnitude faster than `Self::build()`, however it will be - /// erroneous for functions which use a validators public key as an identifier (e.g., - /// deposits). - pub fn build_fast(&mut self) -> Result<(), BeaconStateError> { - let common_keypair = Keypair::random(); - - let mut validator_registry = Vec::with_capacity(self.validator_count); - let mut validator_balances = Vec::with_capacity(self.validator_count); - self.keypairs = Vec::with_capacity(self.validator_count); - - for _ in 0..self.validator_count { - self.keypairs.push(common_keypair.clone()); - validator_balances.push(32_000_000_000); - validator_registry.push(Validator { - pubkey: common_keypair.pk.clone(), - withdrawal_credentials: Hash256::zero(), - activation_epoch: self.spec.genesis_epoch, - ..Validator::default() - }) - } - - let state = BeaconState { - validator_registry, - validator_balances, - ..BeaconState::genesis( - self.genesis_time, - vec![], - self.latest_eth1_data.clone(), - &self.spec, - )? - }; - - self.state = Some(state); - - Ok(()) - } - - /// Sets the `BeaconState` to be in the last slot of the given epoch. - /// - /// Sets all justification/finalization parameters to be be as "perfect" as possible (i.e., - /// highest justified and finalized slots, full justification bitfield, etc). - pub fn teleport_to_end_of_epoch(&mut self, epoch: Epoch) { - let state = self.state.as_mut().expect("Genesis required"); - - let slot = epoch.end_slot(self.spec.slots_per_epoch); - - state.slot = slot; - state.validator_registry_update_epoch = epoch - 1; - - state.previous_shuffling_epoch = epoch - 1; - state.current_shuffling_epoch = epoch; - - state.previous_shuffling_seed = Hash256::from_low_u64_le(0); - state.current_shuffling_seed = Hash256::from_low_u64_le(1); - - state.previous_justified_epoch = epoch - 2; - state.justified_epoch = epoch - 1; - state.justification_bitfield = u64::max_value(); - state.finalized_epoch = epoch - 1; - } - - /// Creates a full set of attestations for the `BeaconState`. Each attestation has full - /// participation from its committee and references the expected beacon_block hashes. - /// - /// These attestations should be fully conducive to justification and finalization. - pub fn insert_attestations(&mut self) { - let state = self.state.as_mut().expect("Genesis required"); - - state - .build_epoch_cache(RelativeEpoch::Previous, &self.spec) - .unwrap(); - state - .build_epoch_cache(RelativeEpoch::Current, &self.spec) - .unwrap(); - - let current_epoch = state.current_epoch(&self.spec); - let previous_epoch = state.previous_epoch(&self.spec); - let current_epoch_depth = - (state.slot - current_epoch.end_slot(self.spec.slots_per_epoch)).as_usize(); - - let previous_epoch_slots = previous_epoch.slot_iter(self.spec.slots_per_epoch); - let current_epoch_slots = current_epoch - .slot_iter(self.spec.slots_per_epoch) - .take(current_epoch_depth); - - for slot in previous_epoch_slots.chain(current_epoch_slots) { - let committees = state - .get_crosslink_committees_at_slot(slot, &self.spec) - .unwrap() - .clone(); - - for (committee, shard) in committees { - state - .latest_attestations - .push(committee_to_pending_attestation( - state, &committee, shard, slot, &self.spec, - )) - } - } - } - - /// Returns a cloned `BeaconState`. - pub fn cloned_state(&self) -> BeaconState { - self.state.as_ref().expect("Genesis required").clone() - } -} - -/// Builds a valid PendingAttestation with full participation for some committee. -fn committee_to_pending_attestation( - state: &BeaconState, - committee: &[usize], - shard: u64, - slot: Slot, - spec: &ChainSpec, -) -> PendingAttestation { - let current_epoch = state.current_epoch(spec); - let previous_epoch = state.previous_epoch(spec); - - let mut aggregation_bitfield = Bitfield::new(); - let mut custody_bitfield = Bitfield::new(); - - for (i, _) in committee.iter().enumerate() { - aggregation_bitfield.set(i, true); - custody_bitfield.set(i, true); - } - - let is_previous_epoch = - state.slot.epoch(spec.slots_per_epoch) != slot.epoch(spec.slots_per_epoch); - - let justified_epoch = if is_previous_epoch { - state.previous_justified_epoch - } else { - state.justified_epoch - }; - - let epoch_boundary_root = if is_previous_epoch { - *state - .get_block_root(previous_epoch.start_slot(spec.slots_per_epoch), spec) - .unwrap() - } else { - *state - .get_block_root(current_epoch.start_slot(spec.slots_per_epoch), spec) - .unwrap() - }; - - let justified_block_root = *state - .get_block_root(justified_epoch.start_slot(spec.slots_per_epoch), &spec) - .unwrap(); - - PendingAttestation { - aggregation_bitfield, - data: AttestationData { - slot, - shard, - beacon_block_root: *state.get_block_root(slot, spec).unwrap(), - epoch_boundary_root, - crosslink_data_root: Hash256::zero(), - latest_crosslink: Crosslink { - epoch: slot.epoch(spec.slots_per_epoch), - crosslink_data_root: Hash256::zero(), - }, - justified_epoch, - justified_block_root, - }, - custody_bitfield, - inclusion_slot: slot, - } -} diff --git a/eth2/types/src/beacon_state/epoch_cache.rs b/eth2/types/src/beacon_state/epoch_cache.rs index bbc991646..ca8bcc70e 100644 --- a/eth2/types/src/beacon_state/epoch_cache.rs +++ b/eth2/types/src/beacon_state/epoch_cache.rs @@ -1,84 +1,305 @@ -use super::{AttestationDutyMap, BeaconState, CrosslinkCommittees, Error, ShardCommitteeIndexMap}; -use crate::{ChainSpec, Epoch}; -use log::trace; -use serde_derive::Serialize; -use std::collections::HashMap; +use super::BeaconState; +use crate::*; +use honey_badger_split::SplitExt; +use serde_derive::{Deserialize, Serialize}; +use swap_or_not_shuffle::shuffle_list; -#[derive(Debug, PartialEq, Clone, Serialize)] +#[derive(Debug, PartialEq)] +pub enum Error { + UnableToShuffle, + UnableToGenerateSeed, +} + +mod tests; + +#[derive(Debug, Default, PartialEq, Clone, Serialize, Deserialize)] pub struct EpochCache { - /// True if this cache has been initialized. - pub initialized: bool, - /// The crosslink committees for an epoch. - pub committees: Vec, + /// `Some(epoch)` if the cache is initialized, where `epoch` is the cache it holds. + pub initialized_epoch: Option, + /// All crosslink committees for an epoch. + pub epoch_crosslink_committees: EpochCrosslinkCommittees, /// Maps validator index to a slot, shard and committee index for attestation. - pub attestation_duty_map: AttestationDutyMap, + pub attestation_duties: Vec>, /// Maps a shard to an index of `self.committees`. - pub shard_committee_index_map: ShardCommitteeIndexMap, + pub shard_committee_indices: Vec>, + /// Indices of all active validators in the epoch + pub active_validator_indices: Vec, } impl EpochCache { - pub fn empty() -> EpochCache { - EpochCache { - initialized: false, - committees: vec![], - attestation_duty_map: AttestationDutyMap::new(), - shard_committee_index_map: ShardCommitteeIndexMap::new(), - } - } - + /// Return a new, fully initialized cache. pub fn initialized( state: &BeaconState, - epoch: Epoch, + relative_epoch: RelativeEpoch, spec: &ChainSpec, ) -> Result { - let mut epoch_committees: Vec = - Vec::with_capacity(spec.slots_per_epoch as usize); - let mut attestation_duty_map: AttestationDutyMap = HashMap::new(); - let mut shard_committee_index_map: ShardCommitteeIndexMap = HashMap::new(); + let epoch = relative_epoch.into_epoch(state.slot.epoch(spec.slots_per_epoch)); - let shuffling = - state.get_shuffling_for_slot(epoch.start_slot(spec.slots_per_epoch), false, spec)?; + let active_validator_indices = + get_active_validator_indices(&state.validator_registry, epoch); - for (epoch_committeess_index, slot) in epoch.slot_iter(spec.slots_per_epoch).enumerate() { - let slot_committees = state.calculate_crosslink_committees_at_slot( - slot, - false, - shuffling.clone(), + let builder = match relative_epoch { + RelativeEpoch::Previous => EpochCrosslinkCommitteesBuilder::for_previous_epoch( + state, + active_validator_indices.clone(), spec, - )?; + ), + RelativeEpoch::Current => EpochCrosslinkCommitteesBuilder::for_current_epoch( + state, + active_validator_indices.clone(), + spec, + ), + RelativeEpoch::NextWithRegistryChange => { + EpochCrosslinkCommitteesBuilder::for_next_epoch( + state, + active_validator_indices.clone(), + true, + spec, + )? + } + RelativeEpoch::NextWithoutRegistryChange => { + EpochCrosslinkCommitteesBuilder::for_next_epoch( + state, + active_validator_indices.clone(), + false, + spec, + )? + } + }; + let epoch_crosslink_committees = builder.build(spec)?; - for (slot_committees_index, (committee, shard)) in slot_committees.iter().enumerate() { - // Empty committees are not permitted. - if committee.is_empty() { - return Err(Error::InsufficientValidators); - } + // Loop through all the validators in the committees and create the following maps: + // + // 1. `attestation_duties`: maps `ValidatorIndex` to `AttestationDuty`. + // 2. `shard_committee_indices`: maps `Shard` into a `CrosslinkCommittee` in + // `EpochCrosslinkCommittees`. + let mut attestation_duties = vec![None; state.validator_registry.len()]; + let mut shard_committee_indices = vec![None; spec.shard_count as usize]; + for (i, slot_committees) in epoch_crosslink_committees + .crosslink_committees + .iter() + .enumerate() + { + let slot = epoch.start_slot(spec.slots_per_epoch) + i as u64; - trace!( - "shard: {}, epoch_i: {}, slot_i: {}", - shard, - epoch_committeess_index, - slot_committees_index - ); + for (j, crosslink_committee) in slot_committees.iter().enumerate() { + let shard = crosslink_committee.shard; - shard_committee_index_map - .insert(*shard, (epoch_committeess_index, slot_committees_index)); + shard_committee_indices[shard as usize] = Some((slot, j)); - for (committee_index, validator_index) in committee.iter().enumerate() { - attestation_duty_map.insert( - *validator_index as u64, - (slot, *shard, committee_index as u64), - ); + for (k, validator_index) in crosslink_committee.committee.iter().enumerate() { + let attestation_duty = AttestationDuty { + slot, + shard, + committee_index: k, + }; + attestation_duties[*validator_index] = Some(attestation_duty) } } - - epoch_committees.push(slot_committees) } Ok(EpochCache { - initialized: true, - committees: epoch_committees, - attestation_duty_map, - shard_committee_index_map, + initialized_epoch: Some(epoch), + epoch_crosslink_committees, + attestation_duties, + shard_committee_indices, + active_validator_indices, }) } + + pub fn get_crosslink_committees_at_slot( + &self, + slot: Slot, + spec: &ChainSpec, + ) -> Option<&Vec> { + self.epoch_crosslink_committees + .get_crosslink_committees_at_slot(slot, spec) + } + + pub fn get_crosslink_committee_for_shard( + &self, + shard: Shard, + spec: &ChainSpec, + ) -> Option<&CrosslinkCommittee> { + if shard > self.shard_committee_indices.len() as u64 { + None + } else { + let (slot, committee) = self.shard_committee_indices[shard as usize]?; + let slot_committees = self.get_crosslink_committees_at_slot(slot, spec)?; + slot_committees.get(committee) + } + } +} + +pub fn get_active_validator_indices(validators: &[Validator], epoch: Epoch) -> Vec { + let mut active = Vec::with_capacity(validators.len()); + + for (index, validator) in validators.iter().enumerate() { + if validator.is_active_at(epoch) { + active.push(index) + } + } + + active.shrink_to_fit(); + + active +} + +#[derive(Debug, Default, PartialEq, Clone, Serialize, Deserialize)] +pub struct EpochCrosslinkCommittees { + epoch: Epoch, + pub crosslink_committees: Vec>, +} + +impl EpochCrosslinkCommittees { + fn new(epoch: Epoch, spec: &ChainSpec) -> Self { + Self { + epoch, + crosslink_committees: vec![vec![]; spec.slots_per_epoch as usize], + } + } + + fn get_crosslink_committees_at_slot( + &self, + slot: Slot, + spec: &ChainSpec, + ) -> Option<&Vec> { + let epoch_start_slot = self.epoch.start_slot(spec.slots_per_epoch); + let epoch_end_slot = self.epoch.end_slot(spec.slots_per_epoch); + + if (epoch_start_slot <= slot) && (slot <= epoch_end_slot) { + let index = slot - epoch_start_slot; + self.crosslink_committees.get(index.as_usize()) + } else { + None + } + } +} + +pub struct EpochCrosslinkCommitteesBuilder { + epoch: Epoch, + shuffling_start_shard: Shard, + shuffling_seed: Hash256, + active_validator_indices: Vec, + committees_per_epoch: u64, +} + +impl EpochCrosslinkCommitteesBuilder { + pub fn for_previous_epoch( + state: &BeaconState, + active_validator_indices: Vec, + spec: &ChainSpec, + ) -> Self { + Self { + epoch: state.previous_epoch(spec), + shuffling_start_shard: state.previous_shuffling_start_shard, + shuffling_seed: state.previous_shuffling_seed, + committees_per_epoch: spec.get_epoch_committee_count(active_validator_indices.len()), + active_validator_indices, + } + } + + pub fn for_current_epoch( + state: &BeaconState, + active_validator_indices: Vec, + spec: &ChainSpec, + ) -> Self { + Self { + epoch: state.current_epoch(spec), + shuffling_start_shard: state.current_shuffling_start_shard, + shuffling_seed: state.current_shuffling_seed, + committees_per_epoch: spec.get_epoch_committee_count(active_validator_indices.len()), + active_validator_indices, + } + } + + pub fn for_next_epoch( + state: &BeaconState, + active_validator_indices: Vec, + registry_change: bool, + spec: &ChainSpec, + ) -> Result { + let current_epoch = state.current_epoch(spec); + let next_epoch = state.next_epoch(spec); + let committees_per_epoch = spec.get_epoch_committee_count(active_validator_indices.len()); + + let epochs_since_last_registry_update = + current_epoch - state.validator_registry_update_epoch; + + let (seed, shuffling_start_shard) = if registry_change { + let next_seed = state + .generate_seed(next_epoch, spec) + .map_err(|_| Error::UnableToGenerateSeed)?; + ( + next_seed, + (state.current_shuffling_start_shard + committees_per_epoch) % spec.shard_count, + ) + } else if (epochs_since_last_registry_update > 1) + & epochs_since_last_registry_update.is_power_of_two() + { + let next_seed = state + .generate_seed(next_epoch, spec) + .map_err(|_| Error::UnableToGenerateSeed)?; + (next_seed, state.current_shuffling_start_shard) + } else { + ( + state.current_shuffling_seed, + state.current_shuffling_start_shard, + ) + }; + + Ok(Self { + epoch: state.next_epoch(spec), + shuffling_start_shard, + shuffling_seed: seed, + active_validator_indices, + committees_per_epoch, + }) + } + + pub fn build(self, spec: &ChainSpec) -> Result { + // The shuffler fails on a empty list, so if there are no active validator indices, simply + // return an empty list. + let shuffled_active_validator_indices = if self.active_validator_indices.is_empty() { + vec![] + } else { + shuffle_list( + self.active_validator_indices, + spec.shuffle_round_count, + &self.shuffling_seed[..], + true, + ) + .ok_or_else(|| Error::UnableToShuffle)? + }; + + let mut committees: Vec> = shuffled_active_validator_indices + .honey_badger_split(self.committees_per_epoch as usize) + .map(|slice: &[usize]| slice.to_vec()) + .collect(); + + let mut epoch_crosslink_committees = EpochCrosslinkCommittees::new(self.epoch, spec); + let mut shard = self.shuffling_start_shard; + + let committees_per_slot = (self.committees_per_epoch / spec.slots_per_epoch) as usize; + + for (i, slot) in self.epoch.slot_iter(spec.slots_per_epoch).enumerate() { + for j in (0..committees.len()) + .into_iter() + .skip(i * committees_per_slot) + .take(committees_per_slot) + { + let crosslink_committee = CrosslinkCommittee { + slot, + shard, + committee: committees[j].drain(..).collect(), + }; + epoch_crosslink_committees.crosslink_committees[i].push(crosslink_committee); + + shard += 1; + shard %= spec.shard_count; + } + } + + Ok(epoch_crosslink_committees) + } } diff --git a/eth2/types/src/beacon_state/epoch_cache/tests.rs b/eth2/types/src/beacon_state/epoch_cache/tests.rs new file mode 100644 index 000000000..10df635f2 --- /dev/null +++ b/eth2/types/src/beacon_state/epoch_cache/tests.rs @@ -0,0 +1,142 @@ +#![cfg(test)] + +use super::*; +use crate::test_utils::*; +use swap_or_not_shuffle::shuffle_list; + +fn do_sane_cache_test( + state: BeaconState, + epoch: Epoch, + validator_count: usize, + expected_seed: Hash256, + expected_shuffling_start: u64, + spec: &ChainSpec, +) { + let active_indices: Vec = (0..validator_count).collect(); + assert_eq!( + &active_indices[..], + state.get_active_validator_indices(epoch, &spec).unwrap(), + "Validator indices mismatch" + ); + + let shuffling = shuffle_list( + active_indices, + spec.shuffle_round_count, + &expected_seed[..], + true, + ) + .unwrap(); + + let committees_per_epoch = spec.get_epoch_committee_count(shuffling.len()); + let committees_per_slot = committees_per_epoch / spec.slots_per_epoch; + + let mut expected_indices_iter = shuffling.iter(); + let mut shard_counter = expected_shuffling_start; + + for (i, slot) in epoch.slot_iter(spec.slots_per_epoch).enumerate() { + let crosslink_committees_at_slot = + state.get_crosslink_committees_at_slot(slot, &spec).unwrap(); + + assert_eq!( + crosslink_committees_at_slot.len(), + committees_per_slot as usize, + "Bad committees per slot ({})", + i + ); + + for c in crosslink_committees_at_slot { + assert_eq!(c.shard, shard_counter, "Bad shard"); + shard_counter += 1; + shard_counter %= spec.shard_count; + + for &i in &c.committee { + assert_eq!( + i, + *expected_indices_iter.next().unwrap(), + "Non-sequential validators." + ); + } + } + } +} + +fn setup_sane_cache_test(validator_count: usize, spec: &ChainSpec) -> BeaconState { + let mut builder = + TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, spec); + + let epoch = spec.genesis_epoch + 4; + let slot = epoch.start_slot(spec.slots_per_epoch); + builder.teleport_to_slot(slot, spec); + + let (mut state, _keypairs) = builder.build(); + + state.current_shuffling_start_shard = 0; + state.current_shuffling_seed = Hash256::from_slice(&[1; 32]); + + state.previous_shuffling_start_shard = spec.shard_count - 1; + state.previous_shuffling_seed = Hash256::from_slice(&[2; 32]); + + state + .build_epoch_cache(RelativeEpoch::Previous, spec) + .unwrap(); + state + .build_epoch_cache(RelativeEpoch::Current, spec) + .unwrap(); + state + .build_epoch_cache(RelativeEpoch::NextWithRegistryChange, spec) + .unwrap(); + state + .build_epoch_cache(RelativeEpoch::NextWithoutRegistryChange, spec) + .unwrap(); + + state +} + +#[test] +fn builds_sane_current_epoch_cache() { + let mut spec = ChainSpec::few_validators(); + spec.shard_count = 4; + let validator_count = (spec.shard_count * spec.target_committee_size) + 1; + let state = setup_sane_cache_test(validator_count as usize, &spec); + do_sane_cache_test( + state.clone(), + state.current_epoch(&spec), + validator_count as usize, + state.current_shuffling_seed, + state.current_shuffling_start_shard, + &spec, + ); +} + +#[test] +fn builds_sane_previous_epoch_cache() { + let mut spec = ChainSpec::few_validators(); + spec.shard_count = 2; + let validator_count = (spec.shard_count * spec.target_committee_size) + 1; + let state = setup_sane_cache_test(validator_count as usize, &spec); + do_sane_cache_test( + state.clone(), + state.previous_epoch(&spec), + validator_count as usize, + state.previous_shuffling_seed, + state.previous_shuffling_start_shard, + &spec, + ); +} + +#[test] +fn builds_sane_next_without_update_epoch_cache() { + let mut spec = ChainSpec::few_validators(); + spec.shard_count = 2; + let validator_count = (spec.shard_count * spec.target_committee_size) + 1; + let mut state = setup_sane_cache_test(validator_count as usize, &spec); + state.validator_registry_update_epoch = state.slot.epoch(spec.slots_per_epoch); + do_sane_cache_test( + state.clone(), + state.next_epoch(&spec), + validator_count as usize, + state.current_shuffling_seed, + state.current_shuffling_start_shard, + &spec, + ); +} diff --git a/eth2/types/src/beacon_state/helpers.rs b/eth2/types/src/beacon_state/helpers.rs index c93b16f76..adae7bab4 100644 --- a/eth2/types/src/beacon_state/helpers.rs +++ b/eth2/types/src/beacon_state/helpers.rs @@ -11,7 +11,7 @@ pub fn verify_bitfield_length(bitfield: &Bitfield, committee_size: usize) -> boo } for i in committee_size..(bitfield.num_bytes() * 8) { - if bitfield.get(i).expect("Impossible due to previous check.") { + if bitfield.get(i).unwrap_or(false) { return false; } } diff --git a/eth2/types/src/beacon_state/pubkey_cache.rs b/eth2/types/src/beacon_state/pubkey_cache.rs new file mode 100644 index 000000000..340bdb311 --- /dev/null +++ b/eth2/types/src/beacon_state/pubkey_cache.rs @@ -0,0 +1,38 @@ +use crate::*; +use serde_derive::{Deserialize, Serialize}; +use std::collections::HashMap; + +type ValidatorIndex = usize; + +#[derive(Debug, PartialEq, Clone, Default, Serialize, Deserialize)] +pub struct PubkeyCache { + map: HashMap, +} + +impl PubkeyCache { + /// Returns the number of validator indices already in the map. + pub fn len(&self) -> ValidatorIndex { + self.map.len() + } + + /// Inserts a validator index into the map. + /// + /// The added index must equal the number of validators already added to the map. This ensures + /// that an index is never skipped. + pub fn insert(&mut self, pubkey: PublicKey, index: ValidatorIndex) -> bool { + if index == self.map.len() { + self.map.insert(pubkey, index); + true + } else { + false + } + } + + /// Inserts a validator index into the map. + /// + /// The added index must equal the number of validators already added to the map. This ensures + /// that an index is never skipped. + pub fn get(&self, pubkey: &PublicKey) -> Option { + self.map.get(pubkey).cloned() + } +} diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs index 40bfd146c..dc16a013b 100644 --- a/eth2/types/src/beacon_state/tests.rs +++ b/eth2/types/src/beacon_state/tests.rs @@ -1,84 +1,57 @@ #![cfg(test)] - use super::*; -use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; -use crate::{BeaconState, ChainSpec}; -use ssz::{ssz_encode, Decodable}; +use crate::test_utils::*; -#[test] -pub fn can_produce_genesis_block() { - let mut builder = BeaconStateBuilder::new(2); - builder.build().unwrap(); -} +ssz_tests!(BeaconState); -/// Tests that `get_attestation_participants` is consistent with the result of -/// get_crosslink_committees_at_slot` with a full bitfield. -#[test] -pub fn get_attestation_participants_consistency() { - let mut rng = XorShiftRng::from_seed([42; 16]); +/// Test that +/// +/// 1. Using the cache before it's built fails. +/// 2. Using the cache after it's build passes. +/// 3. Using the cache after it's dropped fails. +fn test_cache_initialization<'a>( + state: &'a mut BeaconState, + relative_epoch: RelativeEpoch, + spec: &ChainSpec, +) { + let slot = relative_epoch + .into_epoch(state.slot.epoch(spec.slots_per_epoch)) + .start_slot(spec.slots_per_epoch); - let mut builder = BeaconStateBuilder::new(8); - builder.spec = ChainSpec::few_validators(); + // Assuming the cache isn't already built, assert that a call to a cache-using function fails. + assert_eq!( + state.get_beacon_proposer_index(slot, relative_epoch, spec), + Err(BeaconStateError::EpochCacheUninitialized(relative_epoch)) + ); - builder.build().unwrap(); + // Build the cache. + state.build_epoch_cache(relative_epoch, spec).unwrap(); - let mut state = builder.cloned_state(); - let spec = builder.spec.clone(); - - state - .build_epoch_cache(RelativeEpoch::Previous, &spec) + // Assert a call to a cache-using function passes. + let _ = state + .get_beacon_proposer_index(slot, relative_epoch, spec) .unwrap(); - state - .build_epoch_cache(RelativeEpoch::Current, &spec) - .unwrap(); - state.build_epoch_cache(RelativeEpoch::Next, &spec).unwrap(); - for slot in state - .slot - .epoch(spec.slots_per_epoch) - .slot_iter(spec.slots_per_epoch) - { - let committees = state.get_crosslink_committees_at_slot(slot, &spec).unwrap(); + // Drop the cache. + state.drop_cache(relative_epoch); - for (committee, shard) in committees { - let mut attestation_data = AttestationData::random_for_test(&mut rng); - attestation_data.slot = slot; - attestation_data.shard = *shard; - - let mut bitfield = Bitfield::new(); - for (i, _) in committee.iter().enumerate() { - bitfield.set(i, true); - } - - assert_eq!( - state - .get_attestation_participants(&attestation_data, &bitfield, &spec) - .unwrap(), - *committee - ); - } - } + // Assert a call to a cache-using function fail. + assert_eq!( + state.get_beacon_proposer_index(slot, relative_epoch, spec), + Err(BeaconStateError::EpochCacheUninitialized(relative_epoch)) + ); } #[test] -pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = BeaconState::random_for_test(&mut rng); +fn cache_initialization() { + let spec = ChainSpec::few_validators(); + let (mut state, _keypairs) = + TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(16, &spec).build(); - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); + state.slot = (spec.genesis_epoch + 1).start_slot(spec.slots_per_epoch); - assert_eq!(original, decoded); -} - -#[test] -pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = BeaconState::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 + test_cache_initialization(&mut state, RelativeEpoch::Previous, &spec); + test_cache_initialization(&mut state, RelativeEpoch::Current, &spec); + test_cache_initialization(&mut state, RelativeEpoch::NextWithRegistryChange, &spec); + test_cache_initialization(&mut state, RelativeEpoch::NextWithoutRegistryChange, &spec); } diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs index 089d40385..1607b85aa 100644 --- a/eth2/types/src/chain_spec.rs +++ b/eth2/types/src/chain_spec.rs @@ -1,21 +1,27 @@ -use crate::{Address, Epoch, Fork, Hash256, Multiaddr, Slot}; +use crate::*; use bls::Signature; +use int_to_bytes::int_to_bytes4; +use serde_derive::Deserialize; const GWEI: u64 = 1_000_000_000; +/// Each of the BLS signature domains. +/// +/// Spec v0.5.0 pub enum Domain { - Deposit, - Attestation, - Proposal, - Exit, + BeaconBlock, Randao, + Attestation, + Deposit, + Exit, Transfer, } /// Holds all the "constants" for a BeaconChain. /// -/// Spec v0.4.0 -#[derive(PartialEq, Debug, Clone)] +/// Spec v0.5.0 +#[derive(PartialEq, Debug, Clone, Deserialize)] +#[serde(default)] pub struct ChainSpec { /* * Misc @@ -23,8 +29,7 @@ pub struct ChainSpec { pub shard_count: u64, pub target_committee_size: u64, pub max_balance_churn_quotient: u64, - pub beacon_chain_shard_number: u64, - pub max_indices_per_slashable_vote: u64, + pub max_indices_per_slashable_vote: usize, pub max_exit_dequeues_per_epoch: u64, pub shuffle_round_count: u8, @@ -45,7 +50,7 @@ pub struct ChainSpec { /* * Initial Values */ - pub genesis_fork_version: u64, + pub genesis_fork_version: u32, pub genesis_slot: Slot, pub genesis_epoch: Epoch, pub genesis_start_shard: u64, @@ -63,12 +68,13 @@ pub struct ChainSpec { pub min_seed_lookahead: Epoch, pub activation_exit_delay: u64, pub epochs_per_eth1_voting_period: u64, + pub slots_per_historical_root: usize, pub min_validator_withdrawability_delay: Epoch, + pub persistent_committee_period: u64, /* * State list lengths */ - pub latest_block_roots_length: usize, pub latest_randao_mixes_length: usize, pub latest_active_index_roots_length: usize, pub latest_slashed_exit_length: usize, @@ -100,12 +106,12 @@ pub struct ChainSpec { * * Use `ChainSpec::get_domain(..)` to access these values. */ - domain_deposit: u64, - domain_attestation: u64, - domain_proposal: u64, - domain_exit: u64, - domain_randao: u64, - domain_transfer: u64, + domain_beacon_block: u32, + domain_randao: u32, + domain_attestation: u32, + domain_deposit: u32, + domain_exit: u32, + domain_transfer: u32, /* * Network specific parameters @@ -130,24 +136,29 @@ impl ChainSpec { /// Get the domain number that represents the fork meta and signature domain. /// - /// Spec v0.4.0 + /// Spec v0.5.0 pub fn get_domain(&self, epoch: Epoch, domain: Domain, fork: &Fork) -> u64 { let domain_constant = match domain { - Domain::Deposit => self.domain_deposit, - Domain::Attestation => self.domain_attestation, - Domain::Proposal => self.domain_proposal, - Domain::Exit => self.domain_exit, + Domain::BeaconBlock => self.domain_beacon_block, Domain::Randao => self.domain_randao, + Domain::Attestation => self.domain_attestation, + Domain::Deposit => self.domain_deposit, + Domain::Exit => self.domain_exit, Domain::Transfer => self.domain_transfer, }; - let fork_version = fork.get_fork_version(epoch); - fork_version * u64::pow(2, 32) + domain_constant + let mut bytes: Vec = fork.get_fork_version(epoch).to_vec(); + bytes.append(&mut int_to_bytes4(domain_constant)); + + let mut fork_and_domain = [0; 8]; + fork_and_domain.copy_from_slice(&bytes); + + u64::from_le_bytes(fork_and_domain) } /// Returns a `ChainSpec` compatible with the Ethereum Foundation specification. /// - /// Spec v0.4.0 + /// Spec v0.5.0 pub fn foundation() -> Self { let genesis_slot = Slot::new(2_u64.pow(32)); let slots_per_epoch = 64; @@ -160,7 +171,6 @@ impl ChainSpec { shard_count: 1_024, target_committee_size: 128, max_balance_churn_quotient: 32, - beacon_chain_shard_number: u64::max_value(), max_indices_per_slashable_vote: 4_096, max_exit_dequeues_per_epoch: 4, shuffle_round_count: 90, @@ -200,12 +210,13 @@ impl ChainSpec { min_seed_lookahead: Epoch::new(1), activation_exit_delay: 4, epochs_per_eth1_voting_period: 16, + slots_per_historical_root: 8_192, min_validator_withdrawability_delay: Epoch::new(256), + persistent_committee_period: 2_048, /* * State list lengths */ - latest_block_roots_length: 8_192, latest_randao_mixes_length: 8_192, latest_active_index_roots_length: 8_192, latest_slashed_exit_length: 8_192, @@ -232,11 +243,11 @@ impl ChainSpec { /* * Signature domains */ - domain_deposit: 0, - domain_attestation: 1, - domain_proposal: 2, - domain_exit: 3, - domain_randao: 4, + domain_beacon_block: 0, + domain_randao: 1, + domain_attestation: 2, + domain_deposit: 3, + domain_exit: 4, domain_transfer: 5, /* @@ -264,8 +275,6 @@ impl ChainSpec { } /// Returns a `ChainSpec` compatible with the specification suitable for 8 validators. - /// - /// Spec v0.4.0 pub fn few_validators() -> Self { let genesis_slot = Slot::new(2_u64.pow(32)); let slots_per_epoch = 8; @@ -282,12 +291,43 @@ impl ChainSpec { } } +impl Default for ChainSpec { + fn default() -> Self { + Self::foundation() + } +} + #[cfg(test)] mod tests { use super::*; + use int_to_bytes::int_to_bytes8; #[test] fn test_foundation_spec_can_be_constructed() { let _ = ChainSpec::foundation(); } + + fn test_domain(domain_type: Domain, raw_domain: u32, spec: &ChainSpec) { + let fork = Fork::genesis(&spec); + let epoch = Epoch::new(0); + + let domain = spec.get_domain(epoch, domain_type, &fork); + + let mut expected = fork.get_fork_version(epoch).to_vec(); + expected.append(&mut int_to_bytes4(raw_domain)); + + assert_eq!(int_to_bytes8(domain), expected); + } + + #[test] + fn test_get_domain() { + let spec = ChainSpec::foundation(); + + test_domain(Domain::BeaconBlock, spec.domain_beacon_block, &spec); + test_domain(Domain::Randao, spec.domain_randao, &spec); + test_domain(Domain::Attestation, spec.domain_attestation, &spec); + test_domain(Domain::Deposit, spec.domain_deposit, &spec); + test_domain(Domain::Exit, spec.domain_exit, &spec); + test_domain(Domain::Transfer, spec.domain_transfer, &spec); + } } diff --git a/eth2/types/src/crosslink.rs b/eth2/types/src/crosslink.rs index f49195a75..f91680c75 100644 --- a/eth2/types/src/crosslink.rs +++ b/eth2/types/src/crosslink.rs @@ -1,15 +1,25 @@ use crate::test_utils::TestRandom; use crate::{Epoch, Hash256}; use rand::RngCore; -use serde_derive::Serialize; +use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; /// Specifies the block hash for a shard at an epoch. /// -/// Spec v0.4.0 +/// Spec v0.5.0 #[derive( - Debug, Clone, PartialEq, Default, Serialize, Hash, Encode, Decode, TreeHash, TestRandom, + Debug, + Clone, + PartialEq, + Default, + Serialize, + Deserialize, + Hash, + Encode, + Decode, + TreeHash, + TestRandom, )] pub struct Crosslink { pub epoch: Epoch, @@ -19,29 +29,6 @@ pub struct Crosslink { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Crosslink::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Crosslink::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(Crosslink); } diff --git a/eth2/types/src/crosslink_committee.rs b/eth2/types/src/crosslink_committee.rs new file mode 100644 index 000000000..af1778a1b --- /dev/null +++ b/eth2/types/src/crosslink_committee.rs @@ -0,0 +1,10 @@ +use crate::*; +use serde_derive::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode, TreeHash}; + +#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize, Decode, Encode, TreeHash)] +pub struct CrosslinkCommittee { + pub slot: Slot, + pub shard: Shard, + pub committee: Vec, +} diff --git a/eth2/types/src/deposit.rs b/eth2/types/src/deposit.rs index 2e69ea599..ff8d83d77 100644 --- a/eth2/types/src/deposit.rs +++ b/eth2/types/src/deposit.rs @@ -1,16 +1,16 @@ use super::{DepositData, Hash256}; use crate::test_utils::TestRandom; use rand::RngCore; -use serde_derive::Serialize; +use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; /// A deposit to potentially become a beacon chain validator. /// -/// Spec v0.4.0 -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] +/// Spec v0.5.0 +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] pub struct Deposit { - pub branch: Vec, + pub proof: Vec, pub index: u64, pub deposit_data: DepositData, } @@ -18,29 +18,6 @@ pub struct Deposit { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Deposit::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Deposit::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(Deposit); } diff --git a/eth2/types/src/deposit_data.rs b/eth2/types/src/deposit_data.rs index 1eb2722a9..a1e30032f 100644 --- a/eth2/types/src/deposit_data.rs +++ b/eth2/types/src/deposit_data.rs @@ -1,14 +1,14 @@ use super::DepositInput; use crate::test_utils::TestRandom; use rand::RngCore; -use serde_derive::Serialize; +use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; /// Data generated by the deposit contract. /// -/// Spec v0.4.0 -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] +/// Spec v0.5.0 +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] pub struct DepositData { pub amount: u64, pub timestamp: u64, @@ -18,29 +18,6 @@ pub struct DepositData { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = DepositData::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = DepositData::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(DepositData); } diff --git a/eth2/types/src/deposit_input.rs b/eth2/types/src/deposit_input.rs index c4c79c3d1..3f8a6177a 100644 --- a/eth2/types/src/deposit_input.rs +++ b/eth2/types/src/deposit_input.rs @@ -1,47 +1,88 @@ -use super::Hash256; use crate::test_utils::TestRandom; +use crate::*; use bls::{PublicKey, Signature}; use rand::RngCore; -use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, TreeHash}; +use serde_derive::{Deserialize, Serialize}; +use ssz::{SignedRoot, TreeHash}; +use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; use test_random_derive::TestRandom; /// The data supplied by the user to the deposit contract. /// -/// Spec v0.4.0 -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] +/// Spec v0.5.0 +#[derive( + Debug, + PartialEq, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + SignedRoot, + TreeHash, + TestRandom, +)] pub struct DepositInput { pub pubkey: PublicKey, pub withdrawal_credentials: Hash256, pub proof_of_possession: Signature, } +impl DepositInput { + /// Generate the 'proof_of_posession' signature for a given DepositInput details. + /// + /// Spec v0.5.0 + pub fn create_proof_of_possession( + &self, + secret_key: &SecretKey, + epoch: Epoch, + fork: &Fork, + spec: &ChainSpec, + ) -> Signature { + let msg = self.signed_root(); + let domain = spec.get_domain(epoch, Domain::Deposit, fork); + + Signature::new(msg.as_slice(), domain, secret_key) + } + + /// Verify that proof-of-possession is valid. + /// + /// Spec v0.5.0 + pub fn validate_proof_of_possession( + &self, + epoch: Epoch, + fork: &Fork, + spec: &ChainSpec, + ) -> bool { + let msg = self.signed_root(); + let domain = spec.get_domain(epoch, Domain::Deposit, fork); + + self.proof_of_possession.verify(&msg, domain, &self.pubkey) + } +} + #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; + + ssz_tests!(DepositInput); #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = DepositInput::random_for_test(&mut rng); + fn can_create_and_validate() { + let spec = ChainSpec::foundation(); + let fork = Fork::genesis(&spec); + let keypair = Keypair::random(); + let epoch = Epoch::new(0); - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); + let mut deposit_input = DepositInput { + pubkey: keypair.pk.clone(), + withdrawal_credentials: Hash256::zero(), + proof_of_possession: Signature::empty_signature(), + }; - assert_eq!(original, decoded); - } + deposit_input.proof_of_possession = + deposit_input.create_proof_of_possession(&keypair.sk, epoch, &fork, &spec); - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = DepositInput::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 + assert!(deposit_input.validate_proof_of_possession(epoch, &fork, &spec)); } } diff --git a/eth2/types/src/epoch_cache.rs b/eth2/types/src/epoch_cache.rs new file mode 100644 index 000000000..e69de29bb diff --git a/eth2/types/src/eth1_data.rs b/eth2/types/src/eth1_data.rs index 2c817ca38..deced19fb 100644 --- a/eth2/types/src/eth1_data.rs +++ b/eth2/types/src/eth1_data.rs @@ -1,14 +1,16 @@ use super::Hash256; use crate::test_utils::TestRandom; use rand::RngCore; -use serde_derive::Serialize; +use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; /// Contains data obtained from the Eth1 chain. /// -/// Spec v0.4.0 -#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, TreeHash, TestRandom)] +/// Spec v0.5.0 +#[derive( + Debug, PartialEq, Clone, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, +)] pub struct Eth1Data { pub deposit_root: Hash256, pub block_hash: Hash256, @@ -17,29 +19,6 @@ pub struct Eth1Data { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Eth1Data::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Eth1Data::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(Eth1Data); } diff --git a/eth2/types/src/eth1_data_vote.rs b/eth2/types/src/eth1_data_vote.rs index 898145575..2f3a1ade1 100644 --- a/eth2/types/src/eth1_data_vote.rs +++ b/eth2/types/src/eth1_data_vote.rs @@ -1,14 +1,16 @@ use super::Eth1Data; use crate::test_utils::TestRandom; use rand::RngCore; -use serde_derive::Serialize; +use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; /// A summation of votes for some `Eth1Data`. /// -/// Spec v0.4.0 -#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, TreeHash, TestRandom)] +/// Spec v0.5.0 +#[derive( + Debug, PartialEq, Clone, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, +)] pub struct Eth1DataVote { pub eth1_data: Eth1Data, pub vote_count: u64, @@ -17,29 +19,6 @@ pub struct Eth1DataVote { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Eth1DataVote::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Eth1DataVote::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(Eth1DataVote); } diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs index 0acd6da90..b780b95ef 100644 --- a/eth2/types/src/fork.rs +++ b/eth2/types/src/fork.rs @@ -1,24 +1,41 @@ -use crate::{test_utils::TestRandom, Epoch}; +use crate::{test_utils::TestRandom, ChainSpec, Epoch}; +use int_to_bytes::int_to_bytes4; use rand::RngCore; -use serde_derive::Serialize; +use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; /// Specifies a fork of the `BeaconChain`, to prevent replay attacks. /// -/// Spec v0.4.0 -#[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode, TreeHash, TestRandom)] +/// Spec v0.5.0 +#[derive( + Debug, Clone, PartialEq, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, +)] pub struct Fork { - pub previous_version: u64, - pub current_version: u64, + pub previous_version: [u8; 4], + pub current_version: [u8; 4], pub epoch: Epoch, } impl Fork { + /// Initialize the `Fork` from the genesis parameters in the `spec`. + /// + /// Spec v0.5.0 + pub fn genesis(spec: &ChainSpec) -> Self { + let mut current_version: [u8; 4] = [0; 4]; + current_version.copy_from_slice(&int_to_bytes4(spec.genesis_fork_version)); + + Self { + previous_version: current_version, + current_version, + epoch: spec.genesis_epoch, + } + } + /// Return the fork version of the given ``epoch``. /// - /// Spec v0.4.0 - pub fn get_fork_version(&self, epoch: Epoch) -> u64 { + /// Spec v0.5.0 + pub fn get_fork_version(&self, epoch: Epoch) -> [u8; 4] { if epoch < self.epoch { return self.previous_version; } @@ -29,29 +46,51 @@ impl Fork { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Fork::random_for_test(&mut rng); + ssz_tests!(Fork); - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); + fn test_genesis(version: u32, epoch: Epoch) { + let mut spec = ChainSpec::foundation(); - assert_eq!(original, decoded); + spec.genesis_fork_version = version; + spec.genesis_epoch = epoch; + + let fork = Fork::genesis(&spec); + + assert_eq!(fork.epoch, spec.genesis_epoch, "epoch incorrect"); + assert_eq!( + fork.previous_version, fork.current_version, + "previous and current are not identical" + ); + assert_eq!( + fork.current_version, + version.to_le_bytes(), + "current version incorrect" + ); } #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Fork::random_for_test(&mut rng); + fn genesis() { + test_genesis(0, Epoch::new(0)); + test_genesis(9, Epoch::new(11)); + test_genesis(2_u32.pow(31), Epoch::new(2_u64.pow(63))); + test_genesis(u32::max_value(), Epoch::max_value()); + } - let result = original.hash_tree_root_internal(); + #[test] + fn get_fork_version() { + let previous_version = [1; 4]; + let current_version = [2; 4]; + let epoch = Epoch::new(10); - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 + let fork = Fork { + previous_version, + current_version, + epoch, + }; + + assert_eq!(fork.get_fork_version(epoch - 1), previous_version); + assert_eq!(fork.get_fork_version(epoch), current_version); + assert_eq!(fork.get_fork_version(epoch + 1), current_version); } } diff --git a/eth2/types/src/historical_batch.rs b/eth2/types/src/historical_batch.rs new file mode 100644 index 000000000..77859ed1a --- /dev/null +++ b/eth2/types/src/historical_batch.rs @@ -0,0 +1,22 @@ +use crate::test_utils::TestRandom; +use crate::Hash256; +use rand::RngCore; +use serde_derive::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode, TreeHash}; +use test_random_derive::TestRandom; + +/// Historical block and state roots. +/// +/// Spec v0.5.0 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] +pub struct HistoricalBatch { + pub block_roots: Vec, + pub state_roots: Vec, +} + +#[cfg(test)] +mod tests { + use super::*; + + ssz_tests!(HistoricalBatch); +} diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index 2cf61cfec..7d506bc54 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -1,14 +1,20 @@ +//! Ethereum 2.0 types + +#[macro_use] pub mod test_utils; pub mod attestation; pub mod attestation_data; pub mod attestation_data_and_custody_bit; +pub mod attestation_duty; pub mod attester_slashing; pub mod beacon_block; pub mod beacon_block_body; +pub mod beacon_block_header; pub mod beacon_state; pub mod chain_spec; pub mod crosslink; +pub mod crosslink_committee; pub mod deposit; pub mod deposit_data; pub mod deposit_input; @@ -16,16 +22,15 @@ pub mod eth1_data; pub mod eth1_data_vote; pub mod fork; pub mod free_attestation; +pub mod historical_batch; pub mod pending_attestation; -pub mod proposal; pub mod proposer_slashing; -pub mod readers; -pub mod shard_reassignment_record; pub mod slashable_attestation; pub mod transfer; pub mod voluntary_exit; #[macro_use] pub mod slot_epoch_macros; +pub mod relative_epoch; pub mod slot_epoch; pub mod slot_height; pub mod validator; @@ -37,12 +42,15 @@ use std::collections::HashMap; pub use crate::attestation::Attestation; pub use crate::attestation_data::AttestationData; pub use crate::attestation_data_and_custody_bit::AttestationDataAndCustodyBit; +pub use crate::attestation_duty::AttestationDuty; pub use crate::attester_slashing::AttesterSlashing; pub use crate::beacon_block::BeaconBlock; pub use crate::beacon_block_body::BeaconBlockBody; -pub use crate::beacon_state::{BeaconState, Error as BeaconStateError, RelativeEpoch}; +pub use crate::beacon_block_header::BeaconBlockHeader; +pub use crate::beacon_state::{BeaconState, Error as BeaconStateError}; pub use crate::chain_spec::{ChainSpec, Domain}; pub use crate::crosslink::Crosslink; +pub use crate::crosslink_committee::CrosslinkCommittee; pub use crate::deposit::Deposit; pub use crate::deposit_data::DepositData; pub use crate::deposit_input::DepositInput; @@ -50,9 +58,10 @@ pub use crate::eth1_data::Eth1Data; pub use crate::eth1_data_vote::Eth1DataVote; pub use crate::fork::Fork; pub use crate::free_attestation::FreeAttestation; +pub use crate::historical_batch::HistoricalBatch; pub use crate::pending_attestation::PendingAttestation; -pub use crate::proposal::Proposal; pub use crate::proposer_slashing::ProposerSlashing; +pub use crate::relative_epoch::{Error as RelativeEpochError, RelativeEpoch}; pub use crate::slashable_attestation::SlashableAttestation; pub use crate::slot_epoch::{Epoch, Slot}; pub use crate::slot_height::SlotHeight; @@ -60,6 +69,10 @@ pub use crate::transfer::Transfer; pub use crate::validator::Validator; pub use crate::voluntary_exit::VoluntaryExit; +pub type Shard = u64; +pub type Committee = Vec; +pub type CrosslinkCommittees = Vec<(Committee, u64)>; + pub type Hash256 = H256; pub type Address = H160; pub type EthBalance = U256; @@ -72,7 +85,7 @@ pub type AttesterMap = HashMap<(u64, u64), Vec>; /// Maps a slot to a block proposer. pub type ProposerMap = HashMap; -pub use bls::{AggregatePublicKey, AggregateSignature, Keypair, PublicKey, Signature}; +pub use bls::{AggregatePublicKey, AggregateSignature, Keypair, PublicKey, SecretKey, Signature}; pub use libp2p::floodsub::{Topic, TopicBuilder}; pub use libp2p::multiaddr; pub use libp2p::Multiaddr; diff --git a/eth2/types/src/pending_attestation.rs b/eth2/types/src/pending_attestation.rs index 0430d18ba..ca50b6d1c 100644 --- a/eth2/types/src/pending_attestation.rs +++ b/eth2/types/src/pending_attestation.rs @@ -1,14 +1,14 @@ use crate::test_utils::TestRandom; use crate::{AttestationData, Bitfield, Slot}; use rand::RngCore; -use serde_derive::Serialize; +use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; /// An attestation that has been included in the state but not yet fully processed. /// -/// Spec v0.4.0 -#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, TreeHash, TestRandom)] +/// Spec v0.5.0 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] pub struct PendingAttestation { pub aggregation_bitfield: Bitfield, pub data: AttestationData, @@ -19,29 +19,6 @@ pub struct PendingAttestation { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = PendingAttestation::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = PendingAttestation::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(PendingAttestation); } diff --git a/eth2/types/src/proposal.rs b/eth2/types/src/proposal.rs deleted file mode 100644 index b1fd737a0..000000000 --- a/eth2/types/src/proposal.rs +++ /dev/null @@ -1,78 +0,0 @@ -use crate::test_utils::TestRandom; -use crate::{Hash256, Slot}; -use bls::Signature; -use rand::RngCore; -use serde_derive::Serialize; -use ssz::TreeHash; -use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; -use test_random_derive::TestRandom; - -/// A proposal for some shard or beacon block. -/// -/// Spec v0.4.0 -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)] -pub struct Proposal { - pub slot: Slot, - /// Shard number (spec.beacon_chain_shard_number for beacon chain) - pub shard: u64, - pub block_root: Hash256, - pub signature: Signature, -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, SignedRoot, TreeHash}; - - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Proposal::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Proposal::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } - - #[derive(TreeHash)] - struct SignedProposal { - pub slot: Slot, - pub shard: u64, - pub block_root: Hash256, - } - - impl Into for Proposal { - fn into(self) -> SignedProposal { - SignedProposal { - slot: self.slot, - shard: self.shard, - block_root: self.block_root, - } - } - } - - #[test] - pub fn test_signed_root() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Proposal::random_for_test(&mut rng); - - let other: SignedProposal = original.clone().into(); - - assert_eq!(original.signed_root(), other.hash_tree_root()); - } - -} diff --git a/eth2/types/src/proposal_signed_data.rs b/eth2/types/src/proposal_signed_data.rs deleted file mode 100644 index 58f45a41d..000000000 --- a/eth2/types/src/proposal_signed_data.rs +++ /dev/null @@ -1,20 +0,0 @@ -use crate::test_utils::TestRandom; -use crate::{Hash256, Slot}; -use rand::RngCore; -use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, TreeHash}; -use test_random_derive::TestRandom; - -#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, TreeHash, TestRandom)] -pub struct ProposalSignedData { - pub slot: Slot, - pub shard: u64, - pub block_root: Hash256, -} - -#[cfg(test)] -mod tests { - use super::*; - - ssz_tests!(ProposalSignedData); -} diff --git a/eth2/types/src/proposer_slashing.rs b/eth2/types/src/proposer_slashing.rs index f86e7f3a8..02216a2fc 100644 --- a/eth2/types/src/proposer_slashing.rs +++ b/eth2/types/src/proposer_slashing.rs @@ -1,50 +1,23 @@ -use super::Proposal; +use super::BeaconBlockHeader; use crate::test_utils::TestRandom; use rand::RngCore; -use serde_derive::Serialize; +use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; -mod builder; - -pub use builder::ProposerSlashingBuilder; - /// Two conflicting proposals from the same proposer (validator). /// -/// Spec v0.4.0 -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] +/// Spec v0.5.0 +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)] pub struct ProposerSlashing { pub proposer_index: u64, - pub proposal_1: Proposal, - pub proposal_2: Proposal, + pub header_1: BeaconBlockHeader, + pub header_2: BeaconBlockHeader, } #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = ProposerSlashing::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = ProposerSlashing::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(ProposerSlashing); } diff --git a/eth2/types/src/readers/block_reader.rs b/eth2/types/src/readers/block_reader.rs deleted file mode 100644 index 93157a1a3..000000000 --- a/eth2/types/src/readers/block_reader.rs +++ /dev/null @@ -1,35 +0,0 @@ -use crate::{BeaconBlock, Hash256, Slot}; -use std::fmt::Debug; - -/// The `BeaconBlockReader` provides interfaces for reading a subset of fields of a `BeaconBlock`. -/// -/// The purpose of this trait is to allow reading from either; -/// - a standard `BeaconBlock` struct, or -/// - a SSZ serialized byte array. -/// -/// Note: presently, direct SSZ reading has not been implemented so this trait is being used for -/// "future proofing". -pub trait BeaconBlockReader: Debug + PartialEq { - fn slot(&self) -> Slot; - fn parent_root(&self) -> Hash256; - fn state_root(&self) -> Hash256; - fn into_beacon_block(self) -> Option; -} - -impl BeaconBlockReader for BeaconBlock { - fn slot(&self) -> Slot { - self.slot - } - - fn parent_root(&self) -> Hash256 { - self.parent_root - } - - fn state_root(&self) -> Hash256 { - self.state_root - } - - fn into_beacon_block(self) -> Option { - Some(self) - } -} diff --git a/eth2/types/src/readers/mod.rs b/eth2/types/src/readers/mod.rs deleted file mode 100644 index 4ccb14a8c..000000000 --- a/eth2/types/src/readers/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -mod block_reader; -mod state_reader; - -pub use self::block_reader::BeaconBlockReader; -pub use self::state_reader::BeaconStateReader; diff --git a/eth2/types/src/readers/state_reader.rs b/eth2/types/src/readers/state_reader.rs deleted file mode 100644 index e469bee57..000000000 --- a/eth2/types/src/readers/state_reader.rs +++ /dev/null @@ -1,25 +0,0 @@ -use crate::{BeaconState, Slot}; -use std::fmt::Debug; - -/// The `BeaconStateReader` provides interfaces for reading a subset of fields of a `BeaconState`. -/// -/// The purpose of this trait is to allow reading from either; -/// - a standard `BeaconState` struct, or -/// - a SSZ serialized byte array. -/// -/// Note: presently, direct SSZ reading has not been implemented so this trait is being used for -/// "future proofing". -pub trait BeaconStateReader: Debug + PartialEq { - fn slot(&self) -> Slot; - fn into_beacon_state(self) -> Option; -} - -impl BeaconStateReader for BeaconState { - fn slot(&self) -> Slot { - self.slot - } - - fn into_beacon_state(self) -> Option { - Some(self) - } -} diff --git a/eth2/types/src/relative_epoch.rs b/eth2/types/src/relative_epoch.rs new file mode 100644 index 000000000..6c135b1a6 --- /dev/null +++ b/eth2/types/src/relative_epoch.rs @@ -0,0 +1,134 @@ +use crate::*; + +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum Error { + EpochTooLow { base: Epoch, other: Epoch }, + EpochTooHigh { base: Epoch, other: Epoch }, + AmbiguiousNextEpoch, +} + +/// Defines the epochs relative to some epoch. Most useful when referring to the committees prior +/// to and following some epoch. +/// +/// Spec v0.5.0 +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum RelativeEpoch { + /// The prior epoch. + Previous, + /// The current epoch. + Current, + /// The next epoch if there _is_ a validator registry update. + /// + /// If the validator registry is updated during an epoch transition, a new shuffling seed is + /// generated, this changes the attestation and proposal roles. + NextWithRegistryChange, + /// The next epoch if there _is not_ a validator registry update. + /// + /// If the validator registry _is not_ updated during an epoch transition, the shuffling stays + /// the same. + NextWithoutRegistryChange, +} + +impl RelativeEpoch { + /// Returns the `epoch` that `self` refers to, with respect to the `base` epoch. + /// + /// Spec v0.5.0 + pub fn into_epoch(&self, base: Epoch) -> Epoch { + match self { + RelativeEpoch::Previous => base - 1, + RelativeEpoch::Current => base, + RelativeEpoch::NextWithoutRegistryChange => base + 1, + RelativeEpoch::NextWithRegistryChange => base + 1, + } + } + + /// Converts the `other` epoch into a `RelativeEpoch`, with respect to `base` + /// + /// ## Errors + /// Returns an error when: + /// - `EpochTooLow` when `other` is more than 1 prior to `base`. + /// - `EpochTooHigh` when `other` is more than 1 after `base`. + /// - `AmbiguiousNextEpoch` whenever `other` is one after `base`, because it's unknowable if + /// there will be a registry change. + /// + /// Spec v0.5.0 + pub fn from_epoch(base: Epoch, other: Epoch) -> Result { + if other == base - 1 { + Ok(RelativeEpoch::Previous) + } else if other == base { + Ok(RelativeEpoch::Current) + } else if other == base + 1 { + Err(Error::AmbiguiousNextEpoch) + } else if other < base { + Err(Error::EpochTooLow { base, other }) + } else { + Err(Error::EpochTooHigh { base, other }) + } + } + + /// Convenience function for `Self::from_epoch` where both slots are converted into epochs. + pub fn from_slot(base: Slot, other: Slot, spec: &ChainSpec) -> Result { + Self::from_epoch( + base.epoch(spec.slots_per_epoch), + other.epoch(spec.slots_per_epoch), + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_into_epoch() { + let base = Epoch::new(10); + + assert_eq!(RelativeEpoch::Current.into_epoch(base), base); + assert_eq!(RelativeEpoch::Previous.into_epoch(base), base - 1); + assert_eq!( + RelativeEpoch::NextWithRegistryChange.into_epoch(base), + base + 1 + ); + assert_eq!( + RelativeEpoch::NextWithoutRegistryChange.into_epoch(base), + base + 1 + ); + } + + #[test] + fn from_epoch() { + let base = Epoch::new(10); + + assert_eq!( + RelativeEpoch::from_epoch(base, base - 1), + Ok(RelativeEpoch::Previous) + ); + assert_eq!( + RelativeEpoch::from_epoch(base, base), + Ok(RelativeEpoch::Current) + ); + assert_eq!( + RelativeEpoch::from_epoch(base, base + 1), + Err(RelativeEpochError::AmbiguiousNextEpoch) + ); + } + + #[test] + fn from_slot() { + let spec = ChainSpec::foundation(); + let base = Epoch::new(10).start_slot(spec.slots_per_epoch); + + assert_eq!( + RelativeEpoch::from_slot(base, base - 1, &spec), + Ok(RelativeEpoch::Previous) + ); + assert_eq!( + RelativeEpoch::from_slot(base, base, &spec), + Ok(RelativeEpoch::Current) + ); + assert_eq!( + RelativeEpoch::from_slot(base, base + spec.slots_per_epoch, &spec), + Err(RelativeEpochError::AmbiguiousNextEpoch) + ); + } +} diff --git a/eth2/types/src/shard_reassignment_record.rs b/eth2/types/src/shard_reassignment_record.rs deleted file mode 100644 index f5dfa8676..000000000 --- a/eth2/types/src/shard_reassignment_record.rs +++ /dev/null @@ -1,42 +0,0 @@ -use crate::{test_utils::TestRandom, Slot}; -use rand::RngCore; -use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, TreeHash}; -use test_random_derive::TestRandom; - -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] -pub struct ShardReassignmentRecord { - pub validator_index: u64, - pub shard: u64, - pub slot: Slot, -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = ShardReassignmentRecord::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = ShardReassignmentRecord::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } -} diff --git a/eth2/types/src/slashable_attestation.rs b/eth2/types/src/slashable_attestation.rs index 20ba76cdb..05c41a72b 100644 --- a/eth2/types/src/slashable_attestation.rs +++ b/eth2/types/src/slashable_attestation.rs @@ -1,6 +1,6 @@ use crate::{test_utils::TestRandom, AggregateSignature, AttestationData, Bitfield, ChainSpec}; use rand::RngCore; -use serde_derive::Serialize; +use serde_derive::{Deserialize, Serialize}; use ssz::TreeHash; use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; use test_random_derive::TestRandom; @@ -9,8 +9,19 @@ use test_random_derive::TestRandom; /// /// To be included in an `AttesterSlashing`. /// -/// Spec v0.4.0 -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)] +/// Spec v0.5.0 +#[derive( + Debug, + PartialEq, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + TestRandom, + SignedRoot, +)] pub struct SlashableAttestation { /// Lists validator registry indices, not committee indices. pub validator_indices: Vec, @@ -22,17 +33,17 @@ pub struct SlashableAttestation { impl SlashableAttestation { /// Check if ``attestation_data_1`` and ``attestation_data_2`` have the same target. /// - /// Spec v0.4.0 + /// Spec v0.5.0 pub fn is_double_vote(&self, other: &SlashableAttestation, spec: &ChainSpec) -> bool { self.data.slot.epoch(spec.slots_per_epoch) == other.data.slot.epoch(spec.slots_per_epoch) } /// Check if ``attestation_data_1`` surrounds ``attestation_data_2``. /// - /// Spec v0.4.0 + /// Spec v0.5.0 pub fn is_surround_vote(&self, other: &SlashableAttestation, spec: &ChainSpec) -> bool { - let source_epoch_1 = self.data.justified_epoch; - let source_epoch_2 = other.data.justified_epoch; + let source_epoch_1 = self.data.source_epoch; + let source_epoch_2 = other.data.source_epoch; let target_epoch_1 = self.data.slot.epoch(spec.slots_per_epoch); let target_epoch_2 = other.data.slot.epoch(spec.slots_per_epoch); @@ -46,7 +57,6 @@ mod tests { use crate::chain_spec::ChainSpec; use crate::slot_epoch::{Epoch, Slot}; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_is_double_vote_true() { @@ -120,39 +130,18 @@ mod tests { ); } - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = SlashableAttestation::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = SlashableAttestation::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(SlashableAttestation); fn create_slashable_attestation( slot_factor: u64, - justified_epoch: u64, + source_epoch: u64, spec: &ChainSpec, ) -> SlashableAttestation { let mut rng = XorShiftRng::from_seed([42; 16]); let mut slashable_vote = SlashableAttestation::random_for_test(&mut rng); slashable_vote.data.slot = Slot::new(slot_factor * spec.slots_per_epoch); - slashable_vote.data.justified_epoch = Epoch::new(justified_epoch); + slashable_vote.data.source_epoch = Epoch::new(source_epoch); slashable_vote } } diff --git a/eth2/types/src/slashable_vote_data.rs b/eth2/types/src/slashable_vote_data.rs deleted file mode 100644 index 73cf91c61..000000000 --- a/eth2/types/src/slashable_vote_data.rs +++ /dev/null @@ -1,132 +0,0 @@ -use super::AttestationData; -use crate::chain_spec::ChainSpec; -use crate::test_utils::TestRandom; -use bls::AggregateSignature; -use rand::RngCore; -use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, TreeHash}; -use test_random_derive::TestRandom; - -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] -pub struct SlashableVoteData { - pub custody_bit_0_indices: Vec, - pub custody_bit_1_indices: Vec, - pub data: AttestationData, - pub aggregate_signature: AggregateSignature, -} - -impl SlashableVoteData { - /// Check if ``attestation_data_1`` and ``attestation_data_2`` have the same target. - /// - /// Spec v0.3.0 - pub fn is_double_vote(&self, other: &SlashableVoteData, spec: &ChainSpec) -> bool { - self.data.slot.epoch(spec.epoch_length) == other.data.slot.epoch(spec.epoch_length) - } - - /// Check if ``attestation_data_1`` surrounds ``attestation_data_2``. - /// - /// Spec v0.3.0 - pub fn is_surround_vote(&self, other: &SlashableVoteData, spec: &ChainSpec) -> bool { - let source_epoch_1 = self.data.justified_epoch; - let source_epoch_2 = other.data.justified_epoch; - let target_epoch_1 = self.data.slot.epoch(spec.epoch_length); - let target_epoch_2 = other.data.slot.epoch(spec.epoch_length); - - (source_epoch_1 < source_epoch_2) && (target_epoch_2 < target_epoch_1) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::chain_spec::ChainSpec; - use crate::slot_epoch::{Epoch, Slot}; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - - #[test] - pub fn test_is_double_vote_true() { - let spec = ChainSpec::foundation(); - let slashable_vote_first = create_slashable_vote_data(1, 1, &spec); - let slashable_vote_second = create_slashable_vote_data(1, 1, &spec); - - assert_eq!( - slashable_vote_first.is_double_vote(&slashable_vote_second, &spec), - true - ) - } - - #[test] - pub fn test_is_double_vote_false() { - let spec = ChainSpec::foundation(); - let slashable_vote_first = create_slashable_vote_data(1, 1, &spec); - let slashable_vote_second = create_slashable_vote_data(2, 1, &spec); - - assert_eq!( - slashable_vote_first.is_double_vote(&slashable_vote_second, &spec), - false - ); - } - - #[test] - pub fn test_is_surround_vote_true() { - let spec = ChainSpec::foundation(); - let slashable_vote_first = create_slashable_vote_data(2, 1, &spec); - let slashable_vote_second = create_slashable_vote_data(1, 2, &spec); - - assert_eq!( - slashable_vote_first.is_surround_vote(&slashable_vote_second, &spec), - true - ); - } - - #[test] - pub fn test_is_surround_vote_true_realistic() { - let spec = ChainSpec::foundation(); - let slashable_vote_first = create_slashable_vote_data(4, 1, &spec); - let slashable_vote_second = create_slashable_vote_data(3, 2, &spec); - - assert_eq!( - slashable_vote_first.is_surround_vote(&slashable_vote_second, &spec), - true - ); - } - - #[test] - pub fn test_is_surround_vote_false_source_epoch_fails() { - let spec = ChainSpec::foundation(); - let slashable_vote_first = create_slashable_vote_data(2, 2, &spec); - let slashable_vote_second = create_slashable_vote_data(1, 1, &spec); - - assert_eq!( - slashable_vote_first.is_surround_vote(&slashable_vote_second, &spec), - false - ); - } - - #[test] - pub fn test_is_surround_vote_false_target_epoch_fails() { - let spec = ChainSpec::foundation(); - let slashable_vote_first = create_slashable_vote_data(1, 1, &spec); - let slashable_vote_second = create_slashable_vote_data(2, 2, &spec); - - assert_eq!( - slashable_vote_first.is_surround_vote(&slashable_vote_second, &spec), - false - ); - } - - ssz_tests!(SlashableVoteData); - - fn create_slashable_vote_data( - slot_factor: u64, - justified_epoch: u64, - spec: &ChainSpec, - ) -> SlashableVoteData { - let mut rng = XorShiftRng::from_seed([42; 16]); - let mut slashable_vote = SlashableVoteData::random_for_test(&mut rng); - - slashable_vote.data.slot = Slot::new(slot_factor * spec.epoch_length); - slashable_vote.data.justified_epoch = Epoch::new(justified_epoch); - slashable_vote - } -} diff --git a/eth2/types/src/slot_epoch.rs b/eth2/types/src/slot_epoch.rs index 7753027a6..c40b6badf 100644 --- a/eth2/types/src/slot_epoch.rs +++ b/eth2/types/src/slot_epoch.rs @@ -12,7 +12,7 @@ use crate::slot_height::SlotHeight; /// may lead to programming errors which are not detected by the compiler. use crate::test_utils::TestRandom; use rand::RngCore; -use serde_derive::Serialize; +use serde_derive::{Deserialize, Serialize}; use slog; use ssz::{hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash}; use std::cmp::{Ord, Ordering}; @@ -21,10 +21,10 @@ use std::hash::{Hash, Hasher}; use std::iter::Iterator; use std::ops::{Add, AddAssign, Div, DivAssign, Mul, MulAssign, Rem, Sub, SubAssign}; -#[derive(Eq, Debug, Clone, Copy, Default, Serialize)] +#[derive(Eq, Debug, Clone, Copy, Default, Serialize, Deserialize)] pub struct Slot(u64); -#[derive(Eq, Debug, Clone, Copy, Default, Serialize)] +#[derive(Eq, Debug, Clone, Copy, Default, Serialize, Deserialize)] pub struct Epoch(u64); impl_common!(Slot); @@ -103,8 +103,6 @@ impl<'a> Iterator for SlotIter<'a> { #[cfg(test)] mod slot_tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; all_tests!(Slot); } @@ -112,8 +110,6 @@ mod slot_tests { #[cfg(test)] mod epoch_tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; all_tests!(Epoch); diff --git a/eth2/types/src/slot_epoch_macros.rs b/eth2/types/src/slot_epoch_macros.rs index 2148b6cc2..300ad3f6f 100644 --- a/eth2/types/src/slot_epoch_macros.rs +++ b/eth2/types/src/slot_epoch_macros.rs @@ -207,9 +207,9 @@ macro_rules! impl_ssz { } impl TreeHash for $type { - fn hash_tree_root_internal(&self) -> Vec { + fn hash_tree_root(&self) -> Vec { let mut result: Vec = vec![]; - result.append(&mut self.0.hash_tree_root_internal()); + result.append(&mut self.0.hash_tree_root()); hash(&result) } } @@ -248,7 +248,7 @@ macro_rules! impl_common { } // test macros -#[allow(unused_macros)] +#[cfg(test)] macro_rules! new_tests { ($type: ident) => { #[test] @@ -260,7 +260,7 @@ macro_rules! new_tests { }; } -#[allow(unused_macros)] +#[cfg(test)] macro_rules! from_into_tests { ($type: ident, $other: ident) => { #[test] @@ -286,7 +286,7 @@ macro_rules! from_into_tests { }; } -#[allow(unused_macros)] +#[cfg(test)] macro_rules! math_between_tests { ($type: ident, $other: ident) => { #[test] @@ -434,7 +434,7 @@ macro_rules! math_between_tests { }; } -#[allow(unused_macros)] +#[cfg(test)] macro_rules! math_tests { ($type: ident) => { #[test] @@ -528,35 +528,7 @@ macro_rules! math_tests { }; } -#[allow(unused_macros)] -macro_rules! ssz_tests { - ($type: ident) => { - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = $type::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = $type::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = $type::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } - }; -} - -#[allow(unused_macros)] +#[cfg(test)] macro_rules! all_tests { ($type: ident) => { new_tests!($type); diff --git a/eth2/types/src/slot_height.rs b/eth2/types/src/slot_height.rs index 1739227a4..4a783d4a0 100644 --- a/eth2/types/src/slot_height.rs +++ b/eth2/types/src/slot_height.rs @@ -33,11 +33,8 @@ impl SlotHeight { } #[cfg(test)] - mod slot_height_tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; all_tests!(SlotHeight); } diff --git a/eth2/types/src/test_utils/generate_deterministic_keypairs.rs b/eth2/types/src/test_utils/generate_deterministic_keypairs.rs new file mode 100644 index 000000000..f2ce8709e --- /dev/null +++ b/eth2/types/src/test_utils/generate_deterministic_keypairs.rs @@ -0,0 +1,30 @@ +use crate::*; +use int_to_bytes::int_to_bytes48; +use log::debug; +use rayon::prelude::*; + +/// Generates `validator_count` keypairs where the secret key is the index of the +/// validator. +/// +/// For example, the first validator has a secret key of `int_to_bytes48(1)`, the second has +/// `int_to_bytes48(2)` and so on. (We skip `0` as it generates a weird looking public key and is +/// probably invalid). +pub fn generate_deterministic_keypairs(validator_count: usize) -> Vec { + debug!( + "Generating {} deterministic validator keypairs...", + validator_count + ); + + let keypairs: Vec = (0..validator_count) + .collect::>() + .par_iter() + .map(|&i| { + let secret = int_to_bytes48(i as u64 + 1); + let sk = SecretKey::from_bytes(&secret).unwrap(); + let pk = PublicKey::from_secret_key(&sk); + Keypair { sk, pk } + }) + .collect(); + + keypairs +} diff --git a/eth2/types/src/test_utils/keypairs_file.rs b/eth2/types/src/test_utils/keypairs_file.rs new file mode 100644 index 000000000..a1ea4d928 --- /dev/null +++ b/eth2/types/src/test_utils/keypairs_file.rs @@ -0,0 +1,128 @@ +use crate::*; +use rayon::prelude::*; +use std::fs::File; +use std::io::{Error, ErrorKind, Read, Write}; +use std::path::Path; + +pub const PUBLIC_KEY_BYTES_LEN: usize = 96; +pub const SECRET_KEY_BYTES_LEN: usize = 48; + +pub const BATCH_SIZE: usize = 1_000; // ~15MB + +pub const KEYPAIR_BYTES_LEN: usize = PUBLIC_KEY_BYTES_LEN + SECRET_KEY_BYTES_LEN; +pub const BATCH_BYTE_LEN: usize = KEYPAIR_BYTES_LEN * BATCH_SIZE; + +/// Defines a trait that allows reading/writing a vec of `Keypair` from/to a file. +pub trait KeypairsFile { + /// Write to file, without guaranteeing interoperability with other clients. + fn to_raw_file(&self, path: &Path, keypairs: &[Keypair]) -> Result<(), Error>; + /// Read from file, without guaranteeing interoperability with other clients. + fn from_raw_file(path: &Path, count: usize) -> Result, Error>; +} + +impl KeypairsFile for Vec { + /// Write the keypairs to file, using the fastest possible method without guaranteeing + /// interoperability with other clients. + fn to_raw_file(&self, path: &Path, keypairs: &[Keypair]) -> Result<(), Error> { + let mut keypairs_file = File::create(path)?; + + for keypair_batch in keypairs.chunks(BATCH_SIZE) { + let mut buf = Vec::with_capacity(BATCH_BYTE_LEN); + + for keypair in keypair_batch { + buf.append(&mut keypair.sk.as_raw().as_bytes()); + buf.append(&mut keypair.pk.clone().as_uncompressed_bytes()); + } + + keypairs_file.write_all(&buf)?; + } + + Ok(()) + } + + /// Read the keypairs from file, using the fastest possible method without guaranteeing + /// interoperability with other clients. + fn from_raw_file(path: &Path, count: usize) -> Result, Error> { + let mut keypairs_file = File::open(path)?; + + let mut keypairs = Vec::with_capacity(count); + + let indices: Vec = (0..count).collect(); + + for batch in indices.chunks(BATCH_SIZE) { + let mut buf = vec![0; batch.len() * KEYPAIR_BYTES_LEN]; + keypairs_file.read_exact(&mut buf)?; + + let mut keypair_batch = batch + .par_iter() + .enumerate() + .map(|(i, _)| { + let sk_start = i * KEYPAIR_BYTES_LEN; + let sk_end = sk_start + SECRET_KEY_BYTES_LEN; + let sk = SecretKey::from_bytes(&buf[sk_start..sk_end]) + .map_err(|_| Error::new(ErrorKind::Other, "Invalid SecretKey bytes")) + .unwrap(); + + let pk_start = sk_end; + let pk_end = pk_start + PUBLIC_KEY_BYTES_LEN; + let pk = PublicKey::from_uncompressed_bytes(&buf[pk_start..pk_end]) + .map_err(|_| Error::new(ErrorKind::Other, "Invalid PublicKey bytes")) + .unwrap(); + + Keypair { sk, pk } + }) + .collect(); + + keypairs.append(&mut keypair_batch); + } + + Ok(keypairs) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rand::{distributions::Alphanumeric, thread_rng, Rng}; + use std::fs::remove_file; + + fn random_keypairs(n: usize) -> Vec { + (0..n).into_par_iter().map(|_| Keypair::random()).collect() + } + + fn random_tmp_file() -> String { + let mut rng = thread_rng(); + + rng.sample_iter(&Alphanumeric).take(7).collect() + } + + #[test] + #[ignore] + fn read_write_consistency_small_batch() { + let num_keypairs = 10; + let keypairs = random_keypairs(num_keypairs); + + let keypairs_path = Path::new("/tmp").join(random_tmp_file()); + keypairs.to_raw_file(&keypairs_path, &keypairs).unwrap(); + + let decoded = Vec::from_raw_file(&keypairs_path, num_keypairs).unwrap(); + remove_file(keypairs_path).unwrap(); + + assert_eq!(keypairs, decoded); + } + + #[test] + #[ignore] + fn read_write_consistency_big_batch() { + let num_keypairs = BATCH_SIZE + 1; + let keypairs = random_keypairs(num_keypairs); + + let keypairs_path = Path::new("/tmp").join(random_tmp_file()); + keypairs.to_raw_file(&keypairs_path, &keypairs).unwrap(); + + let decoded = Vec::from_raw_file(&keypairs_path, num_keypairs).unwrap(); + remove_file(keypairs_path).unwrap(); + + assert_eq!(keypairs, decoded); + } +} diff --git a/eth2/types/src/test_utils/macros.rs b/eth2/types/src/test_utils/macros.rs index b7c0a6522..f5b2fd87c 100644 --- a/eth2/types/src/test_utils/macros.rs +++ b/eth2/types/src/test_utils/macros.rs @@ -17,14 +17,14 @@ macro_rules! ssz_tests { } #[test] - pub fn test_hash_tree_root_internal() { + pub fn test_hash_tree_root() { use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; use ssz::TreeHash; let mut rng = XorShiftRng::from_seed([42; 16]); let original = $type::random_for_test(&mut rng); - let result = original.hash_tree_root_internal(); + let result = original.hash_tree_root(); assert_eq!(result.len(), 32); // TODO: Add further tests diff --git a/eth2/types/src/test_utils/mod.rs b/eth2/types/src/test_utils/mod.rs index 82e060fca..bc8da0548 100644 --- a/eth2/types/src/test_utils/mod.rs +++ b/eth2/types/src/test_utils/mod.rs @@ -1,55 +1,30 @@ -use rand::RngCore; +#[macro_use] +mod macros; +mod generate_deterministic_keypairs; +mod keypairs_file; +mod test_random; +mod testing_attestation_builder; +mod testing_attestation_data_builder; +mod testing_attester_slashing_builder; +mod testing_beacon_block_builder; +mod testing_beacon_state_builder; +mod testing_deposit_builder; +mod testing_pending_attestation_builder; +mod testing_proposer_slashing_builder; +mod testing_transfer_builder; +mod testing_voluntary_exit_builder; +pub use generate_deterministic_keypairs::generate_deterministic_keypairs; +pub use keypairs_file::KeypairsFile; pub use rand::{prng::XorShiftRng, SeedableRng}; - -pub mod address; -pub mod aggregate_signature; -pub mod bitfield; -pub mod hash256; -pub mod public_key; -pub mod secret_key; -pub mod signature; - -pub trait TestRandom -where - T: RngCore, -{ - fn random_for_test(rng: &mut T) -> Self; -} - -impl TestRandom for bool { - fn random_for_test(rng: &mut T) -> Self { - (rng.next_u32() % 2) == 1 - } -} - -impl TestRandom for u64 { - fn random_for_test(rng: &mut T) -> Self { - rng.next_u64() - } -} - -impl TestRandom for u32 { - fn random_for_test(rng: &mut T) -> Self { - rng.next_u32() - } -} - -impl TestRandom for usize { - fn random_for_test(rng: &mut T) -> Self { - rng.next_u32() as usize - } -} - -impl TestRandom for Vec -where - U: TestRandom, -{ - fn random_for_test(rng: &mut T) -> Self { - vec![ - ::random_for_test(rng), - ::random_for_test(rng), - ::random_for_test(rng), - ] - } -} +pub use test_random::TestRandom; +pub use testing_attestation_builder::TestingAttestationBuilder; +pub use testing_attestation_data_builder::TestingAttestationDataBuilder; +pub use testing_attester_slashing_builder::TestingAttesterSlashingBuilder; +pub use testing_beacon_block_builder::TestingBeaconBlockBuilder; +pub use testing_beacon_state_builder::{keypairs_path, TestingBeaconStateBuilder}; +pub use testing_deposit_builder::TestingDepositBuilder; +pub use testing_pending_attestation_builder::TestingPendingAttestationBuilder; +pub use testing_proposer_slashing_builder::TestingProposerSlashingBuilder; +pub use testing_transfer_builder::TestingTransferBuilder; +pub use testing_voluntary_exit_builder::TestingVoluntaryExitBuilder; diff --git a/eth2/types/src/test_utils/test_random.rs b/eth2/types/src/test_utils/test_random.rs new file mode 100644 index 000000000..cb7abe3a4 --- /dev/null +++ b/eth2/types/src/test_utils/test_random.rs @@ -0,0 +1,67 @@ +use rand::RngCore; + +mod address; +mod aggregate_signature; +mod bitfield; +mod hash256; +mod public_key; +mod secret_key; +mod signature; + +pub trait TestRandom +where + T: RngCore, +{ + fn random_for_test(rng: &mut T) -> Self; +} + +impl TestRandom for bool { + fn random_for_test(rng: &mut T) -> Self { + (rng.next_u32() % 2) == 1 + } +} + +impl TestRandom for u64 { + fn random_for_test(rng: &mut T) -> Self { + rng.next_u64() + } +} + +impl TestRandom for u32 { + fn random_for_test(rng: &mut T) -> Self { + rng.next_u32() + } +} + +impl TestRandom for usize { + fn random_for_test(rng: &mut T) -> Self { + rng.next_u32() as usize + } +} + +impl TestRandom for Vec +where + U: TestRandom, +{ + fn random_for_test(rng: &mut T) -> Self { + vec![ + ::random_for_test(rng), + ::random_for_test(rng), + ::random_for_test(rng), + ] + } +} + +macro_rules! impl_test_random_for_u8_array { + ($len: expr) => { + impl TestRandom for [u8; $len] { + fn random_for_test(rng: &mut T) -> Self { + let mut bytes = [0; $len]; + rng.fill_bytes(&mut bytes); + bytes + } + } + }; +} + +impl_test_random_for_u8_array!(4); diff --git a/eth2/types/src/test_utils/address.rs b/eth2/types/src/test_utils/test_random/address.rs similarity index 100% rename from eth2/types/src/test_utils/address.rs rename to eth2/types/src/test_utils/test_random/address.rs diff --git a/eth2/types/src/test_utils/aggregate_signature.rs b/eth2/types/src/test_utils/test_random/aggregate_signature.rs similarity index 100% rename from eth2/types/src/test_utils/aggregate_signature.rs rename to eth2/types/src/test_utils/test_random/aggregate_signature.rs diff --git a/eth2/types/src/test_utils/bitfield.rs b/eth2/types/src/test_utils/test_random/bitfield.rs similarity index 90% rename from eth2/types/src/test_utils/bitfield.rs rename to eth2/types/src/test_utils/test_random/bitfield.rs index 15011edd9..9748458f1 100644 --- a/eth2/types/src/test_utils/bitfield.rs +++ b/eth2/types/src/test_utils/test_random/bitfield.rs @@ -1,5 +1,5 @@ -use super::super::Bitfield; use super::TestRandom; +use crate::Bitfield; use rand::RngCore; impl TestRandom for Bitfield { diff --git a/eth2/types/src/test_utils/hash256.rs b/eth2/types/src/test_utils/test_random/hash256.rs similarity index 100% rename from eth2/types/src/test_utils/hash256.rs rename to eth2/types/src/test_utils/test_random/hash256.rs diff --git a/eth2/types/src/test_utils/public_key.rs b/eth2/types/src/test_utils/test_random/public_key.rs similarity index 100% rename from eth2/types/src/test_utils/public_key.rs rename to eth2/types/src/test_utils/test_random/public_key.rs diff --git a/eth2/types/src/test_utils/secret_key.rs b/eth2/types/src/test_utils/test_random/secret_key.rs similarity index 100% rename from eth2/types/src/test_utils/secret_key.rs rename to eth2/types/src/test_utils/test_random/secret_key.rs diff --git a/eth2/types/src/test_utils/signature.rs b/eth2/types/src/test_utils/test_random/signature.rs similarity index 100% rename from eth2/types/src/test_utils/signature.rs rename to eth2/types/src/test_utils/test_random/signature.rs diff --git a/eth2/types/src/test_utils/testing_attestation_builder.rs b/eth2/types/src/test_utils/testing_attestation_builder.rs new file mode 100644 index 000000000..60624b48d --- /dev/null +++ b/eth2/types/src/test_utils/testing_attestation_builder.rs @@ -0,0 +1,94 @@ +use crate::test_utils::TestingAttestationDataBuilder; +use crate::*; +use ssz::TreeHash; + +/// Builds an attestation to be used for testing purposes. +/// +/// This struct should **never be used for production purposes.** +pub struct TestingAttestationBuilder { + committee: Vec, + attestation: Attestation, +} + +impl TestingAttestationBuilder { + /// Create a new attestation builder. + pub fn new( + state: &BeaconState, + committee: &[usize], + slot: Slot, + shard: u64, + spec: &ChainSpec, + ) -> Self { + let data_builder = TestingAttestationDataBuilder::new(state, shard, slot, spec); + + let mut aggregation_bitfield = Bitfield::new(); + let mut custody_bitfield = Bitfield::new(); + + for (i, _) in committee.iter().enumerate() { + custody_bitfield.set(i, false); + aggregation_bitfield.set(i, false); + } + + let attestation = Attestation { + aggregation_bitfield, + data: data_builder.build(), + custody_bitfield, + aggregate_signature: AggregateSignature::new(), + }; + + Self { + attestation, + committee: committee.to_vec(), + } + } + + /// Signs the attestation with a subset (or all) committee members. + /// + /// `secret_keys` must be supplied in the same order as `signing_validators`. I.e., the first + /// keypair must be that of the first signing validator. + pub fn sign( + &mut self, + signing_validators: &[usize], + secret_keys: &[&SecretKey], + fork: &Fork, + spec: &ChainSpec, + ) { + assert_eq!( + signing_validators.len(), + secret_keys.len(), + "Must be a key for each validator" + ); + + for (key_index, validator_index) in signing_validators.iter().enumerate() { + let committee_index = self + .committee + .iter() + .position(|v| *v == *validator_index) + .expect("Signing validator not in attestation committee"); + + self.attestation + .aggregation_bitfield + .set(committee_index, true); + + let message = AttestationDataAndCustodyBit { + data: self.attestation.data.clone(), + custody_bit: false, + } + .hash_tree_root(); + + let domain = spec.get_domain( + self.attestation.data.slot.epoch(spec.slots_per_epoch), + Domain::Attestation, + fork, + ); + + let signature = Signature::new(&message, domain, secret_keys[key_index]); + self.attestation.aggregate_signature.add(&signature) + } + } + + /// Consume the builder and return the attestation. + pub fn build(self) -> Attestation { + self.attestation + } +} diff --git a/eth2/types/src/test_utils/testing_attestation_data_builder.rs b/eth2/types/src/test_utils/testing_attestation_data_builder.rs new file mode 100644 index 000000000..a270e3859 --- /dev/null +++ b/eth2/types/src/test_utils/testing_attestation_data_builder.rs @@ -0,0 +1,66 @@ +use crate::*; + +/// Builds an `AttestationData` to be used for testing purposes. +/// +/// This struct should **never be used for production purposes.** +pub struct TestingAttestationDataBuilder { + data: AttestationData, +} + +impl TestingAttestationDataBuilder { + /// Configures a new `AttestationData` which attests to all of the same parameters as the + /// state. + pub fn new(state: &BeaconState, shard: u64, slot: Slot, spec: &ChainSpec) -> Self { + let current_epoch = state.current_epoch(spec); + let previous_epoch = state.previous_epoch(spec); + + let is_previous_epoch = + state.slot.epoch(spec.slots_per_epoch) != slot.epoch(spec.slots_per_epoch); + + let source_epoch = if is_previous_epoch { + state.previous_justified_epoch + } else { + state.current_justified_epoch + }; + + let target_root = if is_previous_epoch { + *state + .get_block_root(previous_epoch.start_slot(spec.slots_per_epoch), spec) + .unwrap() + } else { + *state + .get_block_root(current_epoch.start_slot(spec.slots_per_epoch), spec) + .unwrap() + }; + + let source_root = *state + .get_block_root(source_epoch.start_slot(spec.slots_per_epoch), spec) + .unwrap(); + + let data = AttestationData { + // LMD GHOST vote + slot, + beacon_block_root: *state.get_block_root(slot, spec).unwrap(), + + // FFG Vote + source_epoch, + source_root, + target_root, + + // Crosslink vote + shard, + previous_crosslink: Crosslink { + epoch: slot.epoch(spec.slots_per_epoch), + crosslink_data_root: spec.zero_hash, + }, + crosslink_data_root: spec.zero_hash, + }; + + Self { data } + } + + /// Returns the `AttestationData`, consuming the builder. + pub fn build(self) -> AttestationData { + self.data + } +} diff --git a/eth2/types/src/attester_slashing/builder.rs b/eth2/types/src/test_utils/testing_attester_slashing_builder.rs similarity index 59% rename from eth2/types/src/attester_slashing/builder.rs rename to eth2/types/src/test_utils/testing_attester_slashing_builder.rs index 05301f30b..fcaa3285b 100644 --- a/eth2/types/src/attester_slashing/builder.rs +++ b/eth2/types/src/test_utils/testing_attester_slashing_builder.rs @@ -2,9 +2,11 @@ use crate::*; use ssz::TreeHash; /// Builds an `AttesterSlashing`. -pub struct AttesterSlashingBuilder(); +/// +/// This struct should **never be used for production purposes.** +pub struct TestingAttesterSlashingBuilder(); -impl AttesterSlashingBuilder { +impl TestingAttesterSlashingBuilder { /// Builds an `AttesterSlashing` that is a double vote. /// /// The `signer` function is used to sign the double-vote and accepts: @@ -21,56 +23,53 @@ impl AttesterSlashingBuilder { { let double_voted_slot = Slot::new(0); let shard = 0; - let justified_epoch = Epoch::new(0); let epoch = Epoch::new(0); let hash_1 = Hash256::from_low_u64_le(1); let hash_2 = Hash256::from_low_u64_le(2); + let data_1 = AttestationData { + slot: double_voted_slot, + beacon_block_root: hash_1, + source_epoch: epoch, + source_root: hash_1, + target_root: hash_1, + shard, + previous_crosslink: Crosslink { + epoch, + crosslink_data_root: hash_1, + }, + crosslink_data_root: hash_1, + }; + + let data_2 = AttestationData { + beacon_block_root: hash_2, + ..data_1.clone() + }; + let mut slashable_attestation_1 = SlashableAttestation { validator_indices: validator_indices.to_vec(), - data: AttestationData { - slot: double_voted_slot, - shard, - beacon_block_root: hash_1, - epoch_boundary_root: hash_1, - crosslink_data_root: hash_1, - latest_crosslink: Crosslink { - epoch, - crosslink_data_root: hash_1, - }, - justified_epoch, - justified_block_root: hash_1, - }, + data: data_1, custody_bitfield: Bitfield::new(), aggregate_signature: AggregateSignature::new(), }; let mut slashable_attestation_2 = SlashableAttestation { validator_indices: validator_indices.to_vec(), - data: AttestationData { - slot: double_voted_slot, - shard, - beacon_block_root: hash_2, - epoch_boundary_root: hash_2, - crosslink_data_root: hash_2, - latest_crosslink: Crosslink { - epoch, - crosslink_data_root: hash_2, - }, - justified_epoch, - justified_block_root: hash_2, - }, + data: data_2, custody_bitfield: Bitfield::new(), aggregate_signature: AggregateSignature::new(), }; let add_signatures = |attestation: &mut SlashableAttestation| { + // All validators sign with a `false` custody bit. + let attestation_data_and_custody_bit = AttestationDataAndCustodyBit { + data: attestation.data.clone(), + custody_bit: false, + }; + let message = attestation_data_and_custody_bit.hash_tree_root(); + for (i, validator_index) in validator_indices.iter().enumerate() { - let attestation_data_and_custody_bit = AttestationDataAndCustodyBit { - data: attestation.data.clone(), - custody_bit: attestation.custody_bitfield.get(i).unwrap(), - }; - let message = attestation_data_and_custody_bit.hash_tree_root(); + attestation.custody_bitfield.set(i, false); let signature = signer(*validator_index, &message[..], epoch, Domain::Attestation); attestation.aggregate_signature.add(&signature); } diff --git a/eth2/types/src/test_utils/testing_beacon_block_builder.rs b/eth2/types/src/test_utils/testing_beacon_block_builder.rs new file mode 100644 index 000000000..c5cd22ed4 --- /dev/null +++ b/eth2/types/src/test_utils/testing_beacon_block_builder.rs @@ -0,0 +1,280 @@ +use crate::{ + test_utils::{ + TestingAttestationBuilder, TestingAttesterSlashingBuilder, TestingDepositBuilder, + TestingProposerSlashingBuilder, TestingTransferBuilder, TestingVoluntaryExitBuilder, + }, + *, +}; +use rayon::prelude::*; +use ssz::{SignedRoot, TreeHash}; + +/// Builds a beacon block to be used for testing purposes. +/// +/// This struct should **never be used for production purposes.** +pub struct TestingBeaconBlockBuilder { + pub block: BeaconBlock, +} + +impl TestingBeaconBlockBuilder { + /// Create a new builder from genesis. + pub fn new(spec: &ChainSpec) -> Self { + Self { + block: BeaconBlock::empty(spec), + } + } + + /// Set the slot of the block. + pub fn set_slot(&mut self, slot: Slot) { + self.block.slot = slot; + } + + /// Signs the block. + /// + /// Modifying the block after signing may invalidate the signature. + pub fn sign(&mut self, sk: &SecretKey, fork: &Fork, spec: &ChainSpec) { + let message = self.block.signed_root(); + let epoch = self.block.slot.epoch(spec.slots_per_epoch); + let domain = spec.get_domain(epoch, Domain::BeaconBlock, fork); + self.block.signature = Signature::new(&message, domain, sk); + } + + /// Sets the randao to be a signature across the blocks epoch. + /// + /// Modifying the block's slot after signing may invalidate the signature. + pub fn set_randao_reveal(&mut self, sk: &SecretKey, fork: &Fork, spec: &ChainSpec) { + let epoch = self.block.slot.epoch(spec.slots_per_epoch); + let message = epoch.hash_tree_root(); + let domain = spec.get_domain(epoch, Domain::Randao, fork); + self.block.body.randao_reveal = Signature::new(&message, domain, sk); + } + + /// Inserts a signed, valid `ProposerSlashing` for the validator. + pub fn insert_proposer_slashing( + &mut self, + validator_index: u64, + secret_key: &SecretKey, + fork: &Fork, + spec: &ChainSpec, + ) { + let proposer_slashing = build_proposer_slashing(validator_index, secret_key, fork, spec); + self.block.body.proposer_slashings.push(proposer_slashing); + } + + /// Inserts a signed, valid `AttesterSlashing` for each validator index in `validator_indices`. + pub fn insert_attester_slashing( + &mut self, + validator_indices: &[u64], + secret_keys: &[&SecretKey], + fork: &Fork, + spec: &ChainSpec, + ) { + let attester_slashing = + build_double_vote_attester_slashing(validator_indices, secret_keys, fork, spec); + self.block.body.attester_slashings.push(attester_slashing); + } + + /// Fills the block with `num_attestations` attestations. + /// + /// It will first go and get each committee that is able to include an attestation in this + /// block. If there _are_ enough committees, it will produce an attestation for each. If there + /// _are not_ enough committees, it will start splitting the committees in half until it + /// achieves the target. It will then produce separate attestations for each split committee. + /// + /// Note: the signed messages of the split committees will be identical -- it would be possible + /// to aggregate these split attestations. + pub fn insert_attestations( + &mut self, + state: &BeaconState, + secret_keys: &[&SecretKey], + num_attestations: usize, + spec: &ChainSpec, + ) -> Result<(), BeaconStateError> { + let mut slot = self.block.slot - spec.min_attestation_inclusion_delay; + let mut attestations_added = 0; + + // Stores the following (in order): + // + // - The slot of the committee. + // - A list of all validators in the committee. + // - A list of all validators in the committee that should sign the attestation. + // - The shard of the committee. + let mut committees: Vec<(Slot, Vec, Vec, u64)> = vec![]; + + // Loop backwards through slots gathering each committee, until: + // + // - The slot is too old to be included in a block at this slot. + // - The `MAX_ATTESTATIONS`. + loop { + if state.slot >= slot + spec.slots_per_epoch { + break; + } + + for crosslink_committee in state.get_crosslink_committees_at_slot(slot, spec)? { + if attestations_added >= num_attestations { + break; + } + + committees.push(( + slot, + crosslink_committee.committee.clone(), + crosslink_committee.committee.clone(), + crosslink_committee.shard, + )); + + attestations_added += 1; + } + + slot -= 1; + } + + // Loop through all the committees, splitting each one in half until we have + // `MAX_ATTESTATIONS` committees. + loop { + if committees.len() >= num_attestations as usize { + break; + } + + for index in 0..committees.len() { + if committees.len() >= num_attestations as usize { + break; + } + + let (slot, committee, mut signing_validators, shard) = committees[index].clone(); + + let new_signing_validators = + signing_validators.split_off(signing_validators.len() / 2); + + committees[index] = (slot, committee.clone(), signing_validators, shard); + committees.push((slot, committee, new_signing_validators, shard)); + } + } + + let mut attestations: Vec = committees + .par_iter() + .map(|(slot, committee, signing_validators, shard)| { + let mut builder = + TestingAttestationBuilder::new(state, committee, *slot, *shard, spec); + + let signing_secret_keys: Vec<&SecretKey> = signing_validators + .iter() + .map(|validator_index| secret_keys[*validator_index]) + .collect(); + builder.sign(signing_validators, &signing_secret_keys, &state.fork, spec); + + builder.build() + }) + .collect(); + + self.block.body.attestations.append(&mut attestations); + + Ok(()) + } + + /// Insert a `Valid` deposit into the state. + pub fn insert_deposit( + &mut self, + amount: u64, + index: u64, + state: &BeaconState, + spec: &ChainSpec, + ) { + let keypair = Keypair::random(); + + let mut builder = TestingDepositBuilder::new(keypair.pk.clone(), amount); + builder.set_index(index); + builder.sign( + &keypair, + state.slot.epoch(spec.slots_per_epoch), + &state.fork, + spec, + ); + + self.block.body.deposits.push(builder.build()) + } + + /// Insert a `Valid` exit into the state. + pub fn insert_exit( + &mut self, + state: &BeaconState, + validator_index: u64, + secret_key: &SecretKey, + spec: &ChainSpec, + ) { + let mut builder = TestingVoluntaryExitBuilder::new( + state.slot.epoch(spec.slots_per_epoch), + validator_index, + ); + + builder.sign(secret_key, &state.fork, spec); + + self.block.body.voluntary_exits.push(builder.build()) + } + + /// Insert a `Valid` transfer into the state. + /// + /// Note: this will set the validator to be withdrawable by directly modifying the state + /// validator registry. This _may_ cause problems historic hashes, etc. + pub fn insert_transfer( + &mut self, + state: &BeaconState, + from: u64, + to: u64, + amount: u64, + keypair: Keypair, + spec: &ChainSpec, + ) { + let mut builder = TestingTransferBuilder::new(from, to, amount, state.slot); + builder.sign(keypair, &state.fork, spec); + + self.block.body.transfers.push(builder.build()) + } + + /// Signs and returns the block, consuming the builder. + pub fn build(mut self, sk: &SecretKey, fork: &Fork, spec: &ChainSpec) -> BeaconBlock { + self.sign(sk, fork, spec); + self.block + } + + /// Returns the block, consuming the builder. + pub fn build_without_signing(self) -> BeaconBlock { + self.block + } +} + +/// Builds an `ProposerSlashing` for some `validator_index`. +/// +/// Signs the message using a `BeaconChainHarness`. +fn build_proposer_slashing( + validator_index: u64, + secret_key: &SecretKey, + fork: &Fork, + spec: &ChainSpec, +) -> ProposerSlashing { + let signer = |_validator_index: u64, message: &[u8], epoch: Epoch, domain: Domain| { + let domain = spec.get_domain(epoch, domain, fork); + Signature::new(message, domain, secret_key) + }; + + TestingProposerSlashingBuilder::double_vote(validator_index, signer, spec) +} + +/// Builds an `AttesterSlashing` for some `validator_indices`. +/// +/// Signs the message using a `BeaconChainHarness`. +fn build_double_vote_attester_slashing( + validator_indices: &[u64], + secret_keys: &[&SecretKey], + fork: &Fork, + spec: &ChainSpec, +) -> AttesterSlashing { + let signer = |validator_index: u64, message: &[u8], epoch: Epoch, domain: Domain| { + let key_index = validator_indices + .iter() + .position(|&i| i == validator_index) + .expect("Unable to find attester slashing key"); + let domain = spec.get_domain(epoch, domain, fork); + Signature::new(message, domain, secret_keys[key_index]) + }; + + TestingAttesterSlashingBuilder::double_vote(validator_indices, signer) +} diff --git a/eth2/types/src/test_utils/testing_beacon_state_builder.rs b/eth2/types/src/test_utils/testing_beacon_state_builder.rs new file mode 100644 index 000000000..e76a01e49 --- /dev/null +++ b/eth2/types/src/test_utils/testing_beacon_state_builder.rs @@ -0,0 +1,245 @@ +use super::{generate_deterministic_keypairs, KeypairsFile}; +use crate::test_utils::TestingPendingAttestationBuilder; +use crate::*; +use bls::get_withdrawal_credentials; +use dirs; +use log::debug; +use rayon::prelude::*; +use std::path::{Path, PathBuf}; + +pub const KEYPAIRS_FILE: &str = "keypairs.raw_keypairs"; + +/// Returns the directory where the generated keypairs should be stored. +/// +/// It is either `$HOME/.lighthouse/keypairs.raw_keypairs` or, if `$HOME` is not available, +/// `./keypairs.raw_keypairs`. +pub fn keypairs_path() -> PathBuf { + let dir = dirs::home_dir() + .and_then(|home| Some(home.join(".lighthouse"))) + .unwrap_or_else(|| PathBuf::from("")); + dir.join(KEYPAIRS_FILE) +} + +/// Builds a beacon state to be used for testing purposes. +/// +/// This struct should **never be used for production purposes.** +pub struct TestingBeaconStateBuilder { + state: BeaconState, + keypairs: Vec, +} + +impl TestingBeaconStateBuilder { + /// Attempts to load validators from a file in `$HOME/.lighthouse/keypairs.raw_keypairs`. If + /// the file is unavailable, it generates the keys at runtime. + /// + /// If the `$HOME` environment variable is not set, the local directory is used. + /// + /// See the `Self::from_keypairs_file` method for more info. + /// + /// # Panics + /// + /// If the file does not contain enough keypairs or is invalid. + pub fn from_default_keypairs_file_if_exists(validator_count: usize, spec: &ChainSpec) -> Self { + let dir = dirs::home_dir() + .and_then(|home| Some(home.join(".lighthouse"))) + .unwrap_or_else(|| PathBuf::from("")); + let file = dir.join(KEYPAIRS_FILE); + + if file.exists() { + TestingBeaconStateBuilder::from_keypairs_file(validator_count, &file, spec) + } else { + TestingBeaconStateBuilder::from_deterministic_keypairs(validator_count, spec) + } + } + + /// Loads the initial validator keypairs from a file on disk. + /// + /// Loading keypairs from file is ~10x faster than generating them. Use the `gen_keys` command + /// on the `test_harness` binary to generate the keys. In the `test_harness` dir, run `cargo + /// run -- gen_keys -h` for help. + /// + /// # Panics + /// + /// If the file does not exist, is invalid or does not contain enough keypairs. + pub fn from_keypairs_file(validator_count: usize, path: &Path, spec: &ChainSpec) -> Self { + debug!("Loading {} keypairs from file...", validator_count); + let keypairs = Vec::from_raw_file(path, validator_count).unwrap(); + TestingBeaconStateBuilder::from_keypairs(keypairs, spec) + } + + /// Generates the validator keypairs deterministically. + pub fn from_deterministic_keypairs(validator_count: usize, spec: &ChainSpec) -> Self { + debug!("Generating {} deterministic keypairs...", validator_count); + let keypairs = generate_deterministic_keypairs(validator_count); + TestingBeaconStateBuilder::from_keypairs(keypairs, spec) + } + + /// Uses the given keypair for all validators. + pub fn from_single_keypair( + validator_count: usize, + keypair: &Keypair, + spec: &ChainSpec, + ) -> Self { + debug!("Generating {} cloned keypairs...", validator_count); + + let mut keypairs = Vec::with_capacity(validator_count); + for _ in 0..validator_count { + keypairs.push(keypair.clone()) + } + + TestingBeaconStateBuilder::from_keypairs(keypairs, spec) + } + + /// Creates the builder from an existing set of keypairs. + pub fn from_keypairs(keypairs: Vec, spec: &ChainSpec) -> Self { + let validator_count = keypairs.len(); + + debug!( + "Building {} Validator objects from keypairs...", + validator_count + ); + let validators = keypairs + .par_iter() + .map(|keypair| { + let withdrawal_credentials = Hash256::from_slice(&get_withdrawal_credentials( + &keypair.pk, + spec.bls_withdrawal_prefix_byte, + )); + + Validator { + pubkey: keypair.pk.clone(), + withdrawal_credentials, + // All validators start active. + activation_epoch: spec.genesis_epoch, + exit_epoch: spec.far_future_epoch, + withdrawable_epoch: spec.far_future_epoch, + initiated_exit: false, + slashed: false, + } + }) + .collect(); + + let mut state = BeaconState::genesis( + 0, + Eth1Data { + deposit_root: Hash256::zero(), + block_hash: Hash256::zero(), + }, + spec, + ); + + let balances = vec![32_000_000_000; validator_count]; + + debug!("Importing {} existing validators...", validator_count); + state.validator_registry = validators; + state.validator_balances = balances; + + debug!("BeaconState initialized."); + + Self { state, keypairs } + } + + /// Consume the builder and return the `BeaconState` and the keypairs for each validator. + pub fn build(self) -> (BeaconState, Vec) { + (self.state, self.keypairs) + } + + /// Ensures that the state returned from `Self::build(..)` has all caches pre-built. + /// + /// Note: this performs the build when called. Ensure that no changes are made that would + /// invalidate this cache. + pub fn build_caches(&mut self, spec: &ChainSpec) -> Result<(), BeaconStateError> { + let state = &mut self.state; + + state.build_epoch_cache(RelativeEpoch::Previous, &spec)?; + state.build_epoch_cache(RelativeEpoch::Current, &spec)?; + state.build_epoch_cache(RelativeEpoch::NextWithRegistryChange, &spec)?; + state.build_epoch_cache(RelativeEpoch::NextWithoutRegistryChange, &spec)?; + + state.update_pubkey_cache()?; + + Ok(()) + } + + /// Sets the `BeaconState` to be in a slot, calling `teleport_to_epoch` to update the epoch. + pub fn teleport_to_slot(&mut self, slot: Slot, spec: &ChainSpec) { + self.teleport_to_epoch(slot.epoch(spec.slots_per_epoch), spec); + self.state.slot = slot; + } + + /// Sets the `BeaconState` to be in the first slot of the given epoch. + /// + /// Sets all justification/finalization parameters to be be as "perfect" as possible (i.e., + /// highest justified and finalized slots, full justification bitfield, etc). + fn teleport_to_epoch(&mut self, epoch: Epoch, spec: &ChainSpec) { + let state = &mut self.state; + + let slot = epoch.start_slot(spec.slots_per_epoch); + + state.slot = slot; + + state.previous_shuffling_epoch = epoch - 1; + state.current_shuffling_epoch = epoch; + + state.previous_shuffling_seed = Hash256::from_low_u64_le(0); + state.current_shuffling_seed = Hash256::from_low_u64_le(1); + + state.previous_justified_epoch = epoch - 3; + state.current_justified_epoch = epoch - 2; + state.justification_bitfield = u64::max_value(); + + state.finalized_epoch = epoch - 3; + state.validator_registry_update_epoch = epoch - 3; + } + + /// Creates a full set of attestations for the `BeaconState`. Each attestation has full + /// participation from its committee and references the expected beacon_block hashes. + /// + /// These attestations should be fully conducive to justification and finalization. + pub fn insert_attestations(&mut self, spec: &ChainSpec) { + let state = &mut self.state; + + state + .build_epoch_cache(RelativeEpoch::Previous, spec) + .unwrap(); + state + .build_epoch_cache(RelativeEpoch::Current, spec) + .unwrap(); + + let current_epoch = state.current_epoch(spec); + let previous_epoch = state.previous_epoch(spec); + + let first_slot = previous_epoch.start_slot(spec.slots_per_epoch).as_u64(); + let last_slot = current_epoch.end_slot(spec.slots_per_epoch).as_u64() + - spec.min_attestation_inclusion_delay; + let last_slot = std::cmp::min(state.slot.as_u64(), last_slot); + + for slot in first_slot..last_slot + 1 { + let slot = Slot::from(slot); + + let committees = state + .get_crosslink_committees_at_slot(slot, spec) + .unwrap() + .clone(); + + for crosslink_committee in committees { + let mut builder = TestingPendingAttestationBuilder::new( + state, + crosslink_committee.shard, + slot, + spec, + ); + // The entire committee should have signed the pending attestation. + let signers = vec![true; crosslink_committee.committee.len()]; + builder.add_committee_participation(signers); + let attestation = builder.build(); + + if attestation.data.slot.epoch(spec.slots_per_epoch) < state.current_epoch(spec) { + state.previous_epoch_attestations.push(attestation) + } else { + state.current_epoch_attestations.push(attestation) + } + } + } + } +} diff --git a/eth2/types/src/test_utils/testing_deposit_builder.rs b/eth2/types/src/test_utils/testing_deposit_builder.rs new file mode 100644 index 000000000..ee258e7fe --- /dev/null +++ b/eth2/types/src/test_utils/testing_deposit_builder.rs @@ -0,0 +1,63 @@ +use crate::*; +use bls::get_withdrawal_credentials; + +/// Builds an deposit to be used for testing purposes. +/// +/// This struct should **never be used for production purposes.** +pub struct TestingDepositBuilder { + deposit: Deposit, +} + +impl TestingDepositBuilder { + /// Instantiates a new builder. + pub fn new(pubkey: PublicKey, amount: u64) -> Self { + let deposit = Deposit { + proof: vec![], + index: 0, + deposit_data: DepositData { + amount, + timestamp: 1, + deposit_input: DepositInput { + pubkey, + withdrawal_credentials: Hash256::zero(), + proof_of_possession: Signature::empty_signature(), + }, + }, + }; + + Self { deposit } + } + + /// Set the `deposit.index` value. + pub fn set_index(&mut self, index: u64) { + self.deposit.index = index; + } + + /// Signs the deposit, also setting the following values: + /// + /// - `pubkey` to the signing pubkey. + /// - `withdrawal_credentials` to the signing pubkey. + /// - `proof_of_possesssion` + pub fn sign(&mut self, keypair: &Keypair, epoch: Epoch, fork: &Fork, spec: &ChainSpec) { + let withdrawal_credentials = Hash256::from_slice( + &get_withdrawal_credentials(&keypair.pk, spec.bls_withdrawal_prefix_byte)[..], + ); + + self.deposit.deposit_data.deposit_input.pubkey = keypair.pk.clone(); + self.deposit + .deposit_data + .deposit_input + .withdrawal_credentials = withdrawal_credentials.clone(); + + self.deposit.deposit_data.deposit_input.proof_of_possession = self + .deposit + .deposit_data + .deposit_input + .create_proof_of_possession(&keypair.sk, epoch, fork, spec); + } + + /// Builds the deposit, consuming the builder. + pub fn build(self) -> Deposit { + self.deposit + } +} diff --git a/eth2/types/src/test_utils/testing_pending_attestation_builder.rs b/eth2/types/src/test_utils/testing_pending_attestation_builder.rs new file mode 100644 index 000000000..655b3d1e8 --- /dev/null +++ b/eth2/types/src/test_utils/testing_pending_attestation_builder.rs @@ -0,0 +1,55 @@ +use crate::test_utils::TestingAttestationDataBuilder; +use crate::*; + +/// Builds an `AttesterSlashing` to be used for testing purposes. +/// +/// This struct should **never be used for production purposes.** +pub struct TestingPendingAttestationBuilder { + pending_attestation: PendingAttestation, +} + +impl TestingPendingAttestationBuilder { + /// Create a new valid* `PendingAttestation` for the given parameters. + /// + /// The `inclusion_slot` will be set to be the earliest possible slot the `Attestation` could + /// have been included (`slot + MIN_ATTESTATION_INCLUSION_DELAY`). + /// + /// * The aggregation and custody bitfields will all be empty, they need to be set with + /// `Self::add_committee_participation`. + pub fn new(state: &BeaconState, shard: u64, slot: Slot, spec: &ChainSpec) -> Self { + let data_builder = TestingAttestationDataBuilder::new(state, shard, slot, spec); + + let pending_attestation = PendingAttestation { + aggregation_bitfield: Bitfield::new(), + data: data_builder.build(), + custody_bitfield: Bitfield::new(), + inclusion_slot: slot + spec.min_attestation_inclusion_delay, + }; + + Self { + pending_attestation, + } + } + + /// Sets the committee participation in the `PendingAttestation`. + /// + /// The `PendingAttestation` will appear to be signed by each committee member who's value in + /// `signers` is true. + pub fn add_committee_participation(&mut self, signers: Vec) { + let mut aggregation_bitfield = Bitfield::new(); + let mut custody_bitfield = Bitfield::new(); + + for (i, signed) in signers.iter().enumerate() { + aggregation_bitfield.set(i, *signed); + custody_bitfield.set(i, false); // Fixed to `false` for phase 0. + } + + self.pending_attestation.aggregation_bitfield = aggregation_bitfield; + self.pending_attestation.custody_bitfield = custody_bitfield; + } + + /// Returns the `PendingAttestation`, consuming the builder. + pub fn build(self) -> PendingAttestation { + self.pending_attestation + } +} diff --git a/eth2/types/src/proposer_slashing/builder.rs b/eth2/types/src/test_utils/testing_proposer_slashing_builder.rs similarity index 58% rename from eth2/types/src/proposer_slashing/builder.rs rename to eth2/types/src/test_utils/testing_proposer_slashing_builder.rs index 472a76ec1..2cfebd915 100644 --- a/eth2/types/src/proposer_slashing/builder.rs +++ b/eth2/types/src/test_utils/testing_proposer_slashing_builder.rs @@ -2,9 +2,11 @@ use crate::*; use ssz::SignedRoot; /// Builds a `ProposerSlashing`. -pub struct ProposerSlashingBuilder(); +/// +/// This struct should **never be used for production purposes.** +pub struct TestingProposerSlashingBuilder(); -impl ProposerSlashingBuilder { +impl TestingProposerSlashingBuilder { /// Builds a `ProposerSlashing` that is a double vote. /// /// The `signer` function is used to sign the double-vote and accepts: @@ -20,38 +22,38 @@ impl ProposerSlashingBuilder { F: Fn(u64, &[u8], Epoch, Domain) -> Signature, { let slot = Slot::new(0); - let shard = 0; + let hash_1 = Hash256::from([1; 32]); + let hash_2 = Hash256::from([2; 32]); - let mut proposal_1 = Proposal { + let mut header_1 = BeaconBlockHeader { slot, - shard, - block_root: Hash256::from_low_u64_le(1), + previous_block_root: hash_1, + state_root: hash_1, + block_body_root: hash_1, signature: Signature::empty_signature(), }; - let mut proposal_2 = Proposal { - slot, - shard, - block_root: Hash256::from_low_u64_le(2), - signature: Signature::empty_signature(), + let mut header_2 = BeaconBlockHeader { + previous_block_root: hash_2, + ..header_1.clone() }; - proposal_1.signature = { - let message = proposal_1.signed_root(); + header_1.signature = { + let message = header_1.signed_root(); let epoch = slot.epoch(spec.slots_per_epoch); - signer(proposer_index, &message[..], epoch, Domain::Proposal) + signer(proposer_index, &message[..], epoch, Domain::BeaconBlock) }; - proposal_2.signature = { - let message = proposal_2.signed_root(); + header_2.signature = { + let message = header_2.signed_root(); let epoch = slot.epoch(spec.slots_per_epoch); - signer(proposer_index, &message[..], epoch, Domain::Proposal) + signer(proposer_index, &message[..], epoch, Domain::BeaconBlock) }; ProposerSlashing { proposer_index, - proposal_1, - proposal_2, + header_1, + header_2, } } } diff --git a/eth2/types/src/test_utils/testing_transfer_builder.rs b/eth2/types/src/test_utils/testing_transfer_builder.rs new file mode 100644 index 000000000..354e29aa5 --- /dev/null +++ b/eth2/types/src/test_utils/testing_transfer_builder.rs @@ -0,0 +1,45 @@ +use crate::*; +use ssz::SignedRoot; + +/// Builds a transfer to be used for testing purposes. +/// +/// This struct should **never be used for production purposes.** +pub struct TestingTransferBuilder { + transfer: Transfer, +} + +impl TestingTransferBuilder { + /// Instantiates a new builder. + pub fn new(sender: u64, recipient: u64, amount: u64, slot: Slot) -> Self { + let keypair = Keypair::random(); + + let transfer = Transfer { + sender, + recipient, + amount, + fee: 0, + slot, + pubkey: keypair.pk, + signature: Signature::empty_signature(), + }; + + Self { transfer } + } + + /// Signs the transfer. + /// + /// The keypair must match that of the `from` validator index. + pub fn sign(&mut self, keypair: Keypair, fork: &Fork, spec: &ChainSpec) { + self.transfer.pubkey = keypair.pk; + let message = self.transfer.signed_root(); + let epoch = self.transfer.slot.epoch(spec.slots_per_epoch); + let domain = spec.get_domain(epoch, Domain::Transfer, fork); + + self.transfer.signature = Signature::new(&message, domain, &keypair.sk); + } + + /// Builds the transfer, consuming the builder. + pub fn build(self) -> Transfer { + self.transfer + } +} diff --git a/eth2/types/src/test_utils/testing_voluntary_exit_builder.rs b/eth2/types/src/test_utils/testing_voluntary_exit_builder.rs new file mode 100644 index 000000000..fe5c8325a --- /dev/null +++ b/eth2/types/src/test_utils/testing_voluntary_exit_builder.rs @@ -0,0 +1,37 @@ +use crate::*; +use ssz::SignedRoot; + +/// Builds an exit to be used for testing purposes. +/// +/// This struct should **never be used for production purposes.** +pub struct TestingVoluntaryExitBuilder { + exit: VoluntaryExit, +} + +impl TestingVoluntaryExitBuilder { + /// Instantiates a new builder. + pub fn new(epoch: Epoch, validator_index: u64) -> Self { + let exit = VoluntaryExit { + epoch, + validator_index, + signature: Signature::empty_signature(), + }; + + Self { exit } + } + + /// Signs the exit. + /// + /// The signing secret key must match that of the exiting validator. + pub fn sign(&mut self, secret_key: &SecretKey, fork: &Fork, spec: &ChainSpec) { + let message = self.exit.signed_root(); + let domain = spec.get_domain(self.exit.epoch, Domain::Exit, fork); + + self.exit.signature = Signature::new(&message, domain, secret_key); + } + + /// Builds the exit, consuming the builder. + pub fn build(self) -> VoluntaryExit { + self.exit + } +} diff --git a/eth2/types/src/transfer.rs b/eth2/types/src/transfer.rs index 0382dee11..1c9968702 100644 --- a/eth2/types/src/transfer.rs +++ b/eth2/types/src/transfer.rs @@ -2,7 +2,7 @@ use super::Slot; use crate::test_utils::TestRandom; use bls::{PublicKey, Signature}; use rand::RngCore; -use serde_derive::Serialize; +use serde_derive::{Deserialize, Serialize}; use ssz::TreeHash; use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; use test_random_derive::TestRandom; @@ -10,10 +10,21 @@ use test_random_derive::TestRandom; /// The data submitted to the deposit contract. /// /// Spec v0.4.0 -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)] +#[derive( + Debug, + PartialEq, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + TestRandom, + SignedRoot, +)] pub struct Transfer { - pub from: u64, - pub to: u64, + pub sender: u64, + pub recipient: u64, pub amount: u64, pub fee: u64, pub slot: Slot, @@ -24,29 +35,6 @@ pub struct Transfer { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Transfer::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Transfer::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(Transfer); } diff --git a/eth2/types/src/validator.rs b/eth2/types/src/validator.rs index 43701ca05..f57261175 100644 --- a/eth2/types/src/validator.rs +++ b/eth2/types/src/validator.rs @@ -1,13 +1,13 @@ use crate::{test_utils::TestRandom, Epoch, Hash256, PublicKey}; use rand::RngCore; -use serde_derive::Serialize; +use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; /// Information about a `BeaconChain` validator. /// -/// Spec v0.4.0 -#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, TestRandom, TreeHash)] +/// Spec v0.5.0 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TestRandom, TreeHash)] pub struct Validator { pub pubkey: PublicKey, pub withdrawal_credentials: Hash256, @@ -53,52 +53,61 @@ impl Default for Validator { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Validator::random_for_test(&mut rng); + fn default() { + let v = Validator::default(); - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); + let epoch = Epoch::new(0); - assert_eq!(original, decoded); + assert_eq!(v.is_active_at(epoch), false); + assert_eq!(v.is_exited_at(epoch), false); + assert_eq!(v.is_withdrawable_at(epoch), false); + assert_eq!(v.initiated_exit, false); + assert_eq!(v.slashed, false); } #[test] - fn test_validator_can_be_active() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let mut validator = Validator::random_for_test(&mut rng); + fn is_active_at() { + let epoch = Epoch::new(10); - let activation_epoch = u64::random_for_test(&mut rng); - let exit_epoch = activation_epoch + 234; + let v = Validator { + activation_epoch: epoch, + ..Validator::default() + }; - validator.activation_epoch = Epoch::from(activation_epoch); - validator.exit_epoch = Epoch::from(exit_epoch); - - for slot in (activation_epoch - 100)..(exit_epoch + 100) { - let slot = Epoch::from(slot); - if slot < activation_epoch { - assert!(!validator.is_active_at(slot)); - } else if slot >= exit_epoch { - assert!(!validator.is_active_at(slot)); - } else { - assert!(validator.is_active_at(slot)); - } - } + assert_eq!(v.is_active_at(epoch - 1), false); + assert_eq!(v.is_active_at(epoch), true); + assert_eq!(v.is_active_at(epoch + 1), true); } #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Validator::random_for_test(&mut rng); + fn is_exited_at() { + let epoch = Epoch::new(10); - let result = original.hash_tree_root_internal(); + let v = Validator { + exit_epoch: epoch, + ..Validator::default() + }; - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 + assert_eq!(v.is_exited_at(epoch - 1), false); + assert_eq!(v.is_exited_at(epoch), true); + assert_eq!(v.is_exited_at(epoch + 1), true); } + + #[test] + fn is_withdrawable_at() { + let epoch = Epoch::new(10); + + let v = Validator { + withdrawable_epoch: epoch, + ..Validator::default() + }; + + assert_eq!(v.is_withdrawable_at(epoch - 1), false); + assert_eq!(v.is_withdrawable_at(epoch), true); + assert_eq!(v.is_withdrawable_at(epoch + 1), true); + } + + ssz_tests!(Validator); } diff --git a/eth2/types/src/validator_registry.rs b/eth2/types/src/validator_registry.rs index 7b55e78cb..db35ae993 100644 --- a/eth2/types/src/validator_registry.rs +++ b/eth2/types/src/validator_registry.rs @@ -7,17 +7,17 @@ use crate::Epoch; /// /// Spec v0.4.0 pub fn get_active_validator_indices(validators: &[Validator], epoch: Epoch) -> Vec { - validators - .iter() - .enumerate() - .filter_map(|(index, validator)| { - if validator.is_active_at(epoch) { - Some(index) - } else { - None - } - }) - .collect::>() + let mut active = Vec::with_capacity(validators.len()); + + for (index, validator) in validators.iter().enumerate() { + if validator.is_active_at(epoch) { + active.push(index) + } + } + + active.shrink_to_fit(); + + active } #[cfg(test)] diff --git a/eth2/types/src/validator_registry_delta_block.rs b/eth2/types/src/validator_registry_delta_block.rs deleted file mode 100644 index e9a075052..000000000 --- a/eth2/types/src/validator_registry_delta_block.rs +++ /dev/null @@ -1,36 +0,0 @@ -use crate::{test_utils::TestRandom, Hash256, Slot}; -use bls::PublicKey; -use rand::RngCore; -use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, TreeHash}; -use test_random_derive::TestRandom; - -// The information gathered from the PoW chain validator registration function. -#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, TreeHash, TestRandom)] -pub struct ValidatorRegistryDeltaBlock { - pub latest_registry_delta_root: Hash256, - pub validator_index: u32, - pub pubkey: PublicKey, - pub slot: Slot, - pub flag: u64, -} - -impl Default for ValidatorRegistryDeltaBlock { - /// Yields a "default" `Validator`. Primarily used for testing. - fn default() -> Self { - Self { - latest_registry_delta_root: Hash256::zero(), - validator_index: std::u32::MAX, - pubkey: PublicKey::default(), - slot: Slot::from(std::u64::MAX), - flag: std::u64::MAX, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - ssz_tests!(ValidatorRegistryDeltaBlock); -} diff --git a/eth2/types/src/voluntary_exit.rs b/eth2/types/src/voluntary_exit.rs index 58c3ae4c2..f64f950cb 100644 --- a/eth2/types/src/voluntary_exit.rs +++ b/eth2/types/src/voluntary_exit.rs @@ -1,15 +1,26 @@ use crate::{test_utils::TestRandom, Epoch}; use bls::Signature; use rand::RngCore; -use serde_derive::Serialize; +use serde_derive::{Deserialize, Serialize}; use ssz::TreeHash; use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; use test_random_derive::TestRandom; /// An exit voluntarily submitted a validator who wishes to withdraw. /// -/// Spec v0.4.0 -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)] +/// Spec v0.5.0 +#[derive( + Debug, + PartialEq, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + TestRandom, + SignedRoot, +)] pub struct VoluntaryExit { pub epoch: Epoch, pub validator_index: u64, @@ -19,29 +30,6 @@ pub struct VoluntaryExit { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = VoluntaryExit::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = VoluntaryExit::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(VoluntaryExit); } diff --git a/eth2/utils/bls/Cargo.toml b/eth2/utils/bls/Cargo.toml index 7a436307b..4230a06ea 100644 --- a/eth2/utils/bls/Cargo.toml +++ b/eth2/utils/bls/Cargo.toml @@ -5,8 +5,10 @@ authors = ["Paul Hauner "] edition = "2018" [dependencies] -bls-aggregates = { git = "https://github.com/sigp/signature-schemes", tag = "0.5.2" } +bls-aggregates = { git = "https://github.com/sigp/signature-schemes", tag = "0.6.1" } hashing = { path = "../hashing" } hex = "0.3" serde = "1.0" +serde_derive = "1.0" +serde_hex = { path = "../serde_hex" } ssz = { path = "../ssz" } diff --git a/eth2/utils/bls/src/aggregate_signature.rs b/eth2/utils/bls/src/aggregate_signature.rs index 2d8776353..7b80d3bbf 100644 --- a/eth2/utils/bls/src/aggregate_signature.rs +++ b/eth2/utils/bls/src/aggregate_signature.rs @@ -2,7 +2,9 @@ use super::{AggregatePublicKey, Signature}; use bls_aggregates::{ AggregatePublicKey as RawAggregatePublicKey, AggregateSignature as RawAggregateSignature, }; +use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; +use serde_hex::{encode as hex_encode, PrefixedHexVisitor}; use ssz::{ decode_ssz_list, hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash, }; @@ -48,16 +50,8 @@ impl AggregateSignature { domain: u64, aggregate_public_keys: &[&AggregatePublicKey], ) -> bool { - // TODO: the API for `RawAggregatePublicKey` shoudn't need to take an owned - // `AggregatePublicKey`. There is an issue to fix this, but in the meantime we need to - // clone. - // - // https://github.com/sigp/signature-schemes/issues/10 - let aggregate_public_keys: Vec = aggregate_public_keys - .iter() - .map(|pk| pk.as_raw()) - .cloned() - .collect(); + let aggregate_public_keys: Vec<&RawAggregatePublicKey> = + aggregate_public_keys.iter().map(|pk| pk.as_raw()).collect(); // Messages are concatenated into one long message. let mut msg: Vec = vec![]; @@ -90,12 +84,24 @@ impl Serialize for AggregateSignature { where S: Serializer, { - serializer.serialize_bytes(&ssz_encode(self)) + serializer.serialize_str(&hex_encode(ssz_encode(self))) + } +} + +impl<'de> Deserialize<'de> for AggregateSignature { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?; + let (obj, _) = <_>::ssz_decode(&bytes[..], 0) + .map_err(|e| serde::de::Error::custom(format!("invalid ssz ({:?})", e)))?; + Ok(obj) } } impl TreeHash for AggregateSignature { - fn hash_tree_root_internal(&self) -> Vec { + fn hash_tree_root(&self) -> Vec { hash(&self.0.as_bytes()) } } diff --git a/eth2/utils/bls/src/keypair.rs b/eth2/utils/bls/src/keypair.rs index 1cce9c10e..d60a2fc25 100644 --- a/eth2/utils/bls/src/keypair.rs +++ b/eth2/utils/bls/src/keypair.rs @@ -1,6 +1,7 @@ use super::{PublicKey, SecretKey}; +use serde_derive::{Deserialize, Serialize}; -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Keypair { pub sk: SecretKey, pub pk: PublicKey, diff --git a/eth2/utils/bls/src/lib.rs b/eth2/utils/bls/src/lib.rs index bb109b0a1..38a129908 100644 --- a/eth2/utils/bls/src/lib.rs +++ b/eth2/utils/bls/src/lib.rs @@ -6,6 +6,7 @@ mod aggregate_signature; mod keypair; mod public_key; mod secret_key; +mod serde_vistors; mod signature; pub use crate::aggregate_public_key::AggregatePublicKey; @@ -20,20 +21,6 @@ pub const BLS_AGG_SIG_BYTE_SIZE: usize = 96; use hashing::hash; use ssz::ssz_encode; -/// For some signature and public key, ensure that the signature message was the public key and it -/// was signed by the secret key that corresponds to that public key. -pub fn verify_proof_of_possession(sig: &Signature, pubkey: &PublicKey) -> bool { - // TODO: replace this function with state.validate_proof_of_possession - // https://github.com/sigp/lighthouse/issues/239 - sig.verify(&ssz_encode(pubkey), 0, &pubkey) -} - -// TODO: Update this method -// https://github.com/sigp/lighthouse/issues/239 -pub fn create_proof_of_possession(keypair: &Keypair) -> Signature { - Signature::new(&ssz_encode(&keypair.pk), 0, &keypair.sk) -} - /// Returns the withdrawal credentials for a given public key. pub fn get_withdrawal_credentials(pubkey: &PublicKey, prefix_byte: u8) -> Vec { let hashed = hash(&ssz_encode(pubkey)); diff --git a/eth2/utils/bls/src/public_key.rs b/eth2/utils/bls/src/public_key.rs index c7fd526a0..5a348f530 100644 --- a/eth2/utils/bls/src/public_key.rs +++ b/eth2/utils/bls/src/public_key.rs @@ -1,7 +1,8 @@ use super::SecretKey; use bls_aggregates::PublicKey as RawPublicKey; -use hex::encode as hex_encode; +use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; +use serde_hex::{encode as hex_encode, PrefixedHexVisitor}; use ssz::{ decode_ssz_list, hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash, }; @@ -25,6 +26,24 @@ impl PublicKey { &self.0 } + /// Converts compressed bytes to PublicKey + pub fn from_bytes(bytes: &[u8]) -> Result { + let pubkey = RawPublicKey::from_bytes(&bytes).map_err(|_| DecodeError::Invalid)?; + Ok(PublicKey(pubkey)) + } + + /// Returns the PublicKey as (x, y) bytes + pub fn as_uncompressed_bytes(&self) -> Vec { + RawPublicKey::as_uncompressed_bytes(&mut self.0.clone()) + } + + /// Converts (x, y) bytes to PublicKey + pub fn from_uncompressed_bytes(bytes: &[u8]) -> Result { + let pubkey = + RawPublicKey::from_uncompressed_bytes(&bytes).map_err(|_| DecodeError::Invalid)?; + Ok(PublicKey(pubkey)) + } + /// Returns the last 6 bytes of the SSZ encoding of the public key, as a hex string. /// /// Useful for providing a short identifier to the user. @@ -61,12 +80,24 @@ impl Serialize for PublicKey { where S: Serializer, { - serializer.serialize_bytes(&ssz_encode(self)) + serializer.serialize_str(&hex_encode(self.as_raw().as_bytes())) + } +} + +impl<'de> Deserialize<'de> for PublicKey { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?; + let obj = PublicKey::from_bytes(&bytes[..]) + .map_err(|e| serde::de::Error::custom(format!("invalid pubkey ({:?})", e)))?; + Ok(obj) } } impl TreeHash for PublicKey { - fn hash_tree_root_internal(&self) -> Vec { + fn hash_tree_root(&self) -> Vec { hash(&self.0.as_bytes()) } } @@ -78,8 +109,14 @@ impl PartialEq for PublicKey { } impl Hash for PublicKey { + /// Note: this is distinct from consensus serialization, it will produce a different hash. + /// + /// This method uses the uncompressed bytes, which are much faster to obtain than the + /// compressed bytes required for consensus serialization. + /// + /// Use `ssz::Encode` to obtain the bytes required for consensus hashing. fn hash(&self, state: &mut H) { - ssz_encode(self).hash(state) + self.as_uncompressed_bytes().hash(state) } } diff --git a/eth2/utils/bls/src/secret_key.rs b/eth2/utils/bls/src/secret_key.rs index f2d54f4ac..a3914310a 100644 --- a/eth2/utils/bls/src/secret_key.rs +++ b/eth2/utils/bls/src/secret_key.rs @@ -1,5 +1,9 @@ +use super::serde_vistors::HexVisitor; use bls_aggregates::{DecodeError as BlsDecodeError, SecretKey as RawSecretKey}; -use ssz::{decode_ssz_list, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use hex::encode as hex_encode; +use serde::de::{Deserialize, Deserializer}; +use serde::ser::{Serialize, Serializer}; +use ssz::{decode_ssz_list, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash}; /// A single BLS signature. /// @@ -40,8 +44,29 @@ impl Decodable for SecretKey { } } +impl Serialize for SecretKey { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&hex_encode(ssz_encode(self))) + } +} + +impl<'de> Deserialize<'de> for SecretKey { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let bytes = deserializer.deserialize_str(HexVisitor)?; + let (pubkey, _) = <_>::ssz_decode(&bytes[..], 0) + .map_err(|e| serde::de::Error::custom(format!("invalid ssz ({:?})", e)))?; + Ok(pubkey) + } +} + impl TreeHash for SecretKey { - fn hash_tree_root_internal(&self) -> Vec { + fn hash_tree_root(&self) -> Vec { self.0.as_bytes().clone() } } diff --git a/eth2/utils/bls/src/serde_vistors.rs b/eth2/utils/bls/src/serde_vistors.rs new file mode 100644 index 000000000..55eadb883 --- /dev/null +++ b/eth2/utils/bls/src/serde_vistors.rs @@ -0,0 +1,20 @@ +use hex; +use serde::de::{self, Visitor}; +use std::fmt; + +pub struct HexVisitor; + +impl<'de> Visitor<'de> for HexVisitor { + type Value = Vec; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a hex string (without 0x prefix)") + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + Ok(hex::decode(value).map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e)))?) + } +} diff --git a/eth2/utils/bls/src/signature.rs b/eth2/utils/bls/src/signature.rs index c0c31ef27..47598bc66 100644 --- a/eth2/utils/bls/src/signature.rs +++ b/eth2/utils/bls/src/signature.rs @@ -1,5 +1,8 @@ +use super::serde_vistors::HexVisitor; use super::{PublicKey, SecretKey}; use bls_aggregates::Signature as RawSignature; +use hex::encode as hex_encode; +use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; use ssz::{ decode_ssz_list, hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash, @@ -51,9 +54,12 @@ impl Signature { /// Returns a new empty signature. pub fn empty_signature() -> Self { + // Empty Signature is currently being represented as BLS::Signature.point_at_infinity() + // However it should be represented as vec![0; 96] but this + // would require all signatures to be represented in byte form as opposed to Signature let mut empty: Vec = vec![0; 96]; - // TODO: Modify the way flags are used (b_flag should not be used for empty_signature in the future) - empty[0] += u8::pow(2, 6); + // Sets C_flag and B_flag to 1 and all else to 0 + empty[0] += u8::pow(2, 6) + u8::pow(2, 7); Signature(RawSignature::from_bytes(&empty).unwrap()) } } @@ -73,7 +79,7 @@ impl Decodable for Signature { } impl TreeHash for Signature { - fn hash_tree_root_internal(&self) -> Vec { + fn hash_tree_root(&self) -> Vec { hash(&self.0.as_bytes()) } } @@ -83,7 +89,19 @@ impl Serialize for Signature { where S: Serializer, { - serializer.serialize_bytes(&ssz_encode(self)) + serializer.serialize_str(&hex_encode(ssz_encode(self))) + } +} + +impl<'de> Deserialize<'de> for Signature { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let bytes = deserializer.deserialize_str(HexVisitor)?; + let (pubkey, _) = <_>::ssz_decode(&bytes[..], 0) + .map_err(|e| serde::de::Error::custom(format!("invalid ssz ({:?})", e)))?; + Ok(pubkey) } } @@ -114,7 +132,7 @@ mod tests { assert_eq!(sig_as_bytes.len(), 96); for (i, one_byte) in sig_as_bytes.iter().enumerate() { if i == 0 { - assert_eq!(*one_byte, u8::pow(2, 6)); + assert_eq!(*one_byte, u8::pow(2, 6) + u8::pow(2, 7)); } else { assert_eq!(*one_byte, 0); } diff --git a/eth2/utils/boolean-bitfield/Cargo.toml b/eth2/utils/boolean-bitfield/Cargo.toml index d94b9f7b1..cf037c5d7 100644 --- a/eth2/utils/boolean-bitfield/Cargo.toml +++ b/eth2/utils/boolean-bitfield/Cargo.toml @@ -5,6 +5,7 @@ authors = ["Paul Hauner "] edition = "2018" [dependencies] +serde_hex = { path = "../serde_hex" } ssz = { path = "../ssz" } bit-vec = "0.5.0" serde = "1.0" diff --git a/eth2/utils/boolean-bitfield/fuzz/.gitignore b/eth2/utils/boolean-bitfield/fuzz/.gitignore new file mode 100644 index 000000000..572e03bdf --- /dev/null +++ b/eth2/utils/boolean-bitfield/fuzz/.gitignore @@ -0,0 +1,4 @@ + +target +corpus +artifacts diff --git a/eth2/utils/boolean-bitfield/fuzz/Cargo.toml b/eth2/utils/boolean-bitfield/fuzz/Cargo.toml new file mode 100644 index 000000000..9769fc50e --- /dev/null +++ b/eth2/utils/boolean-bitfield/fuzz/Cargo.toml @@ -0,0 +1,33 @@ + +[package] +name = "boolean-bitfield-fuzz" +version = "0.0.1" +authors = ["Automatically generated"] +publish = false + +[package.metadata] +cargo-fuzz = true + +[dependencies] +ssz = { path = "../../ssz" } + +[dependencies.boolean-bitfield] +path = ".." +[dependencies.libfuzzer-sys] +git = "https://github.com/rust-fuzz/libfuzzer-sys.git" + +# Prevent this from interfering with workspaces +[workspace] +members = ["."] + +[[bin]] +name = "fuzz_target_from_bytes" +path = "fuzz_targets/fuzz_target_from_bytes.rs" + +[[bin]] +name = "fuzz_target_ssz_decode" +path = "fuzz_targets/fuzz_target_ssz_decode.rs" + +[[bin]] +name = "fuzz_target_ssz_encode" +path = "fuzz_targets/fuzz_target_ssz_encode.rs" diff --git a/eth2/utils/boolean-bitfield/fuzz/fuzz_targets/fuzz_target_from_bytes.rs b/eth2/utils/boolean-bitfield/fuzz/fuzz_targets/fuzz_target_from_bytes.rs new file mode 100644 index 000000000..0c71c6d68 --- /dev/null +++ b/eth2/utils/boolean-bitfield/fuzz/fuzz_targets/fuzz_target_from_bytes.rs @@ -0,0 +1,9 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate boolean_bitfield; + +use boolean_bitfield::BooleanBitfield; + +fuzz_target!(|data: &[u8]| { + let _result = BooleanBitfield::from_bytes(data); +}); diff --git a/eth2/utils/boolean-bitfield/fuzz/fuzz_targets/fuzz_target_ssz_decode.rs b/eth2/utils/boolean-bitfield/fuzz/fuzz_targets/fuzz_target_ssz_decode.rs new file mode 100644 index 000000000..14ddbb0a9 --- /dev/null +++ b/eth2/utils/boolean-bitfield/fuzz/fuzz_targets/fuzz_target_ssz_decode.rs @@ -0,0 +1,11 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate boolean_bitfield; +extern crate ssz; + +use boolean_bitfield::BooleanBitfield; +use ssz::{Decodable, DecodeError}; + +fuzz_target!(|data: &[u8]| { + let result: Result<(BooleanBitfield, usize), DecodeError> = <_>::ssz_decode(data, 0); +}); diff --git a/eth2/utils/boolean-bitfield/fuzz/fuzz_targets/fuzz_target_ssz_encode.rs b/eth2/utils/boolean-bitfield/fuzz/fuzz_targets/fuzz_target_ssz_encode.rs new file mode 100644 index 000000000..0626e5db7 --- /dev/null +++ b/eth2/utils/boolean-bitfield/fuzz/fuzz_targets/fuzz_target_ssz_encode.rs @@ -0,0 +1,13 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate boolean_bitfield; +extern crate ssz; + +use boolean_bitfield::BooleanBitfield; +use ssz::SszStream; + +fuzz_target!(|data: &[u8]| { + let bitfield = BooleanBitfield::from_bytes(data); + let mut ssz = SszStream::new(); + ssz.append(&bitfield); +}); diff --git a/eth2/utils/boolean-bitfield/src/lib.rs b/eth2/utils/boolean-bitfield/src/lib.rs index fb3a78e7a..443cd06da 100644 --- a/eth2/utils/boolean-bitfield/src/lib.rs +++ b/eth2/utils/boolean-bitfield/src/lib.rs @@ -3,7 +3,10 @@ extern crate ssz; use bit_vec::BitVec; +use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; +use serde_hex::{encode, PrefixedHexVisitor}; +use ssz::Decodable; use std::cmp; use std::default; @@ -178,17 +181,31 @@ impl ssz::Decodable for BooleanBitfield { } impl Serialize for BooleanBitfield { + /// Serde serialization is compliant the Ethereum YAML test format. fn serialize(&self, serializer: S) -> Result where S: Serializer, { - serializer.serialize_bytes(&ssz::ssz_encode(self)) + serializer.serialize_str(&encode(&ssz::ssz_encode(self))) + } +} + +impl<'de> Deserialize<'de> for BooleanBitfield { + /// Serde serialization is compliant the Ethereum YAML test format. + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?; + let (bitfield, _) = <_>::ssz_decode(&bytes[..], 0) + .map_err(|e| serde::de::Error::custom(format!("invalid ssz ({:?})", e)))?; + Ok(bitfield) } } impl ssz::TreeHash for BooleanBitfield { - fn hash_tree_root_internal(&self) -> Vec { - self.to_bytes().hash_tree_root_internal() + fn hash_tree_root(&self) -> Vec { + self.to_bytes().hash_tree_root() } } diff --git a/eth2/utils/hashing/fuzz/.gitignore b/eth2/utils/hashing/fuzz/.gitignore new file mode 100644 index 000000000..572e03bdf --- /dev/null +++ b/eth2/utils/hashing/fuzz/.gitignore @@ -0,0 +1,4 @@ + +target +corpus +artifacts diff --git a/eth2/utils/hashing/fuzz/Cargo.toml b/eth2/utils/hashing/fuzz/Cargo.toml new file mode 100644 index 000000000..57e0172eb --- /dev/null +++ b/eth2/utils/hashing/fuzz/Cargo.toml @@ -0,0 +1,22 @@ + +[package] +name = "hashing-fuzz" +version = "0.0.1" +authors = ["Automatically generated"] +publish = false + +[package.metadata] +cargo-fuzz = true + +[dependencies.hashing] +path = ".." +[dependencies.libfuzzer-sys] +git = "https://github.com/rust-fuzz/libfuzzer-sys.git" + +# Prevent this from interfering with workspaces +[workspace] +members = ["."] + +[[bin]] +name = "fuzz_target_hash" +path = "fuzz_targets/fuzz_target_hash.rs" diff --git a/eth2/utils/hashing/fuzz/fuzz_targets/fuzz_target_hash.rs b/eth2/utils/hashing/fuzz/fuzz_targets/fuzz_target_hash.rs new file mode 100644 index 000000000..dd78d1ac8 --- /dev/null +++ b/eth2/utils/hashing/fuzz/fuzz_targets/fuzz_target_hash.rs @@ -0,0 +1,9 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate hashing; + +use hashing::hash; + +fuzz_target!(|data: &[u8]| { + let _result = hash(data); +}); diff --git a/eth2/utils/serde_hex/Cargo.toml b/eth2/utils/serde_hex/Cargo.toml new file mode 100644 index 000000000..b28194dd6 --- /dev/null +++ b/eth2/utils/serde_hex/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "serde_hex" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" + +[dependencies] +serde = "1.0" +hex = "0.3" diff --git a/eth2/utils/serde_hex/src/lib.rs b/eth2/utils/serde_hex/src/lib.rs new file mode 100644 index 000000000..3be20d93f --- /dev/null +++ b/eth2/utils/serde_hex/src/lib.rs @@ -0,0 +1,59 @@ +use hex; +use hex::ToHex; +use serde::de::{self, Visitor}; +use std::fmt; + +pub fn encode>(data: T) -> String { + let mut hex = String::with_capacity(data.as_ref().len() * 2); + + // Writing to a string never errors, so we can unwrap here. + data.write_hex(&mut hex).unwrap(); + + let mut s = "0x".to_string(); + + s.push_str(hex.as_str()); + + s +} + +pub struct PrefixedHexVisitor; + +impl<'de> Visitor<'de> for PrefixedHexVisitor { + type Value = Vec; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a hex string with 0x prefix") + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + if value.starts_with("0x") { + Ok(hex::decode(&value[2..]) + .map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e)))?) + } else { + Err(de::Error::custom("missing 0x prefix")) + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn encoding() { + let bytes = vec![0, 255]; + let hex = encode(&bytes); + assert_eq!(hex.as_str(), "0x00ff"); + + let bytes = vec![]; + let hex = encode(&bytes); + assert_eq!(hex.as_str(), "0x"); + + let bytes = vec![1, 2, 3]; + let hex = encode(&bytes); + assert_eq!(hex.as_str(), "0x010203"); + } +} diff --git a/eth2/utils/ssz/fuzz/.gitignore b/eth2/utils/ssz/fuzz/.gitignore new file mode 100644 index 000000000..572e03bdf --- /dev/null +++ b/eth2/utils/ssz/fuzz/.gitignore @@ -0,0 +1,4 @@ + +target +corpus +artifacts diff --git a/eth2/utils/ssz/fuzz/Cargo.toml b/eth2/utils/ssz/fuzz/Cargo.toml new file mode 100644 index 000000000..055d031a0 --- /dev/null +++ b/eth2/utils/ssz/fuzz/Cargo.toml @@ -0,0 +1,105 @@ + +[package] +name = "ssz-fuzz" +version = "0.0.1" +authors = ["Automatically generated"] +publish = false + +[package.metadata] +cargo-fuzz = true + +[dependencies] +ethereum-types = "0.4.0" + +[dependencies.ssz] +path = ".." +[dependencies.libfuzzer-sys] +git = "https://github.com/rust-fuzz/libfuzzer-sys.git" + +# Prevent this from interfering with workspaces +[workspace] +members = ["."] + +[[bin]] +name = "fuzz_target_bool_decode" +path = "fuzz_targets/fuzz_target_bool_decode.rs" + +[[bin]] +name = "fuzz_target_bool_encode" +path = "fuzz_targets/fuzz_target_bool_encode.rs" + +[[bin]] +name = "fuzz_target_u8_decode" +path = "fuzz_targets/fuzz_target_u8_decode.rs" + +[[bin]] +name = "fuzz_target_u8_encode" +path = "fuzz_targets/fuzz_target_u8_encode.rs" + +[[bin]] +name = "fuzz_target_u16_decode" +path = "fuzz_targets/fuzz_target_u16_decode.rs" + +[[bin]] +name = "fuzz_target_u16_encode" +path = "fuzz_targets/fuzz_target_u16_encode.rs" + +[[bin]] +name = "fuzz_target_u32_decode" +path = "fuzz_targets/fuzz_target_u32_decode.rs" + +[[bin]] +name = "fuzz_target_u32_encode" +path = "fuzz_targets/fuzz_target_u32_encode.rs" + +[[bin]] +name = "fuzz_target_u64_decode" +path = "fuzz_targets/fuzz_target_u64_decode.rs" + +[[bin]] +name = "fuzz_target_u64_encode" +path = "fuzz_targets/fuzz_target_u64_encode.rs" + +[[bin]] +name = "fuzz_target_usize_decode" +path = "fuzz_targets/fuzz_target_usize_decode.rs" + +[[bin]] +name = "fuzz_target_usize_encode" +path = "fuzz_targets/fuzz_target_usize_encode.rs" + +[[bin]] +name = "fuzz_target_hash256_decode" +path = "fuzz_targets/fuzz_target_hash256_decode.rs" + +[[bin]] +name = "fuzz_target_hash256_encode" +path = "fuzz_targets/fuzz_target_hash256_encode.rs" + +[[bin]] +name = "fuzz_target_address_decode" +path = "fuzz_targets/fuzz_target_address_decode.rs" + +[[bin]] +name = "fuzz_target_address_encode" +path = "fuzz_targets/fuzz_target_address_encode.rs" + +[[bin]] +name = "fuzz_target_vec_decode" +path = "fuzz_targets/fuzz_target_vec_decode.rs" + +[[bin]] +name = "fuzz_target_vec_address_decode" +path = "fuzz_targets/fuzz_target_vec_address_decode.rs" + +[[bin]] +name = "fuzz_target_vec_u64_decode" +path = "fuzz_targets/fuzz_target_vec_u64_decode.rs" + +[[bin]] +name = "fuzz_target_vec_bool_decode" +path = "fuzz_targets/fuzz_target_vec_bool_decode.rs" + +[[bin]] +name = "fuzz_target_vec_encode" +path = "fuzz_targets/fuzz_target_vec_encode.rs" diff --git a/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_address_decode.rs b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_address_decode.rs new file mode 100644 index 000000000..c49be500a --- /dev/null +++ b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_address_decode.rs @@ -0,0 +1,21 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate ethereum_types; +extern crate ssz; + +use ethereum_types::Address; +use ssz::{DecodeError, Decodable}; + +// Fuzz ssz_decode() +fuzz_target!(|data: &[u8]| { + let result: Result<(Address, usize), DecodeError> = Decodable::ssz_decode(data, 0); + if data.len() >= 20 { + // Should have valid result + let (address, index) = result.unwrap(); + assert_eq!(index, 20); + assert_eq!(address, Address::from_slice(&data[..20])); + } else { + // Length of less than 32 should return error + assert_eq!(result, Err(DecodeError::TooShort)); + } +}); diff --git a/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_address_encode.rs b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_address_encode.rs new file mode 100644 index 000000000..0e51e00ac --- /dev/null +++ b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_address_encode.rs @@ -0,0 +1,20 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate ethereum_types; +extern crate ssz; + +use ethereum_types::Address; +use ssz::SszStream; + +// Fuzz ssz_encode (via ssz_append) +fuzz_target!(|data: &[u8]| { + let mut ssz = SszStream::new(); + if data.len() >= 20 { + let hash = Address::from_slice(&data[..20]); + ssz.append(&hash); + let ssz = ssz.drain(); + + assert_eq!(data[..20], ssz[..20]); + assert_eq!(ssz.len(), 20); + } +}); diff --git a/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_bool_decode.rs b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_bool_decode.rs new file mode 100644 index 000000000..4fb1052b1 --- /dev/null +++ b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_bool_decode.rs @@ -0,0 +1,28 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate ssz; + +use ssz::{DecodeError, Decodable}; + +// Fuzz ssz_decode() +fuzz_target!(|data: &[u8]| { + let result: Result<(bool, usize), DecodeError> = Decodable::ssz_decode(data, 0); + if data.len() >= 1 { + // TODO: change to little endian bytes + // https://github.com/sigp/lighthouse/issues/215 + if data[0] == u8::pow(2,7) { + let (val_bool, index) = result.unwrap(); + assert!(val_bool); + assert_eq!(index, 1); + } else if data[0] == 0 { + let (val_bool, index) = result.unwrap(); + assert!(!val_bool); + assert_eq!(index, 1); + } else { + assert_eq!(result, Err(DecodeError::Invalid)); + } + } else { + // Length of 0 should return error + assert_eq!(result, Err(DecodeError::TooShort)); + } +}); diff --git a/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_bool_encode.rs b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_bool_encode.rs new file mode 100644 index 000000000..4f344cb7d --- /dev/null +++ b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_bool_encode.rs @@ -0,0 +1,22 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate ssz; + +use ssz::SszStream; + +// Fuzz ssz_encode (via ssz_append) +fuzz_target!(|data: &[u8]| { + let mut ssz = SszStream::new(); + let mut val_bool = 0; + if data.len() >= 1 { + val_bool = data[0] % u8::pow(2, 6); + } + + ssz.append(&val_bool); + let ssz = ssz.drain(); + + // TODO: change to little endian bytes + // https://github.com/sigp/lighthouse/issues/215 + assert_eq!(val_bool, ssz[0] % u8::pow(2, 6)); + assert_eq!(ssz.len(), 1); +}); diff --git a/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_hash256_decode.rs b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_hash256_decode.rs new file mode 100644 index 000000000..e4ccc56a4 --- /dev/null +++ b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_hash256_decode.rs @@ -0,0 +1,21 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate ethereum_types; +extern crate ssz; + +use ethereum_types::H256; +use ssz::{DecodeError, Decodable}; + +// Fuzz ssz_decode() +fuzz_target!(|data: &[u8]| { + let result: Result<(H256, usize), DecodeError> = Decodable::ssz_decode(data, 0); + if data.len() >= 32 { + // Should have valid result + let (hash, index) = result.unwrap(); + assert_eq!(index, 32); + assert_eq!(hash, H256::from_slice(&data[..32])); + } else { + // Length of less than 32 should return error + assert_eq!(result, Err(DecodeError::TooShort)); + } +}); diff --git a/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_hash256_encode.rs b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_hash256_encode.rs new file mode 100644 index 000000000..537d9cdf9 --- /dev/null +++ b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_hash256_encode.rs @@ -0,0 +1,20 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate ethereum_types; +extern crate ssz; + +use ethereum_types::H256; +use ssz::SszStream; + +// Fuzz ssz_encode (via ssz_append) +fuzz_target!(|data: &[u8]| { + let mut ssz = SszStream::new(); + if data.len() >= 32 { + let hash = H256::from_slice(&data[..32]); + ssz.append(&hash); + let ssz = ssz.drain(); + + assert_eq!(data[..32], ssz[..32]); + assert_eq!(ssz.len(), 32); + } +}); diff --git a/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u16_decode.rs b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u16_decode.rs new file mode 100644 index 000000000..73395f3af --- /dev/null +++ b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u16_decode.rs @@ -0,0 +1,22 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate ssz; + +use ssz::{DecodeError, Decodable}; + +// Fuzz ssz_decode() +fuzz_target!(|data: &[u8]| { + let result: Result<(u16, usize), DecodeError> = Decodable::ssz_decode(data, 0); + if data.len() >= 2 { + // Valid result + let (number_u16, index) = result.unwrap(); + assert_eq!(index, 2); + // TODO: change to little endian bytes + // https://github.com/sigp/lighthouse/issues/215 + let val = u16::from_be_bytes([data[0], data[1]]); + assert_eq!(number_u16, val); + } else { + // Length of 0 or 1 should return error + assert_eq!(result, Err(DecodeError::TooShort)); + } +}); diff --git a/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u16_encode.rs b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u16_encode.rs new file mode 100644 index 000000000..ce8a51845 --- /dev/null +++ b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u16_encode.rs @@ -0,0 +1,22 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate ssz; + +use ssz::SszStream; + +// Fuzz ssz_encode (via ssz_append) +fuzz_target!(|data: &[u8]| { + let mut ssz = SszStream::new(); + let mut number_u16 = 0; + if data.len() >= 2 { + number_u16 = u16::from_be_bytes([data[0], data[1]]); + } + + ssz.append(&number_u16); + let ssz = ssz.drain(); + + // TODO: change to little endian bytes + // https://github.com/sigp/lighthouse/issues/215 + assert_eq!(ssz.len(), 2); + assert_eq!(number_u16, u16::from_be_bytes([ssz[0], ssz[1]])); +}); diff --git a/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u32_decode.rs b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u32_decode.rs new file mode 100644 index 000000000..e99bf2fad --- /dev/null +++ b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u32_decode.rs @@ -0,0 +1,22 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate ssz; + +use ssz::{DecodeError, Decodable}; + +// Fuzz ssz_decode() +fuzz_target!(|data: &[u8]| { + let result: Result<(u32, usize), DecodeError> = Decodable::ssz_decode(data, 0); + if data.len() >= 4 { + // Valid result + let (number_u32, index) = result.unwrap(); + assert_eq!(index, 4); + // TODO: change to little endian bytes + // https://github.com/sigp/lighthouse/issues/215 + let val = u32::from_be_bytes([data[0], data[1], data[2], data[3]]); + assert_eq!(number_u32, val); + } else { + // Length less then 4 should return error + assert_eq!(result, Err(DecodeError::TooShort)); + } +}); diff --git a/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u32_encode.rs b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u32_encode.rs new file mode 100644 index 000000000..c71bcecaf --- /dev/null +++ b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u32_encode.rs @@ -0,0 +1,22 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate ssz; + +use ssz::SszStream; + +// Fuzz ssz_encode (via ssz_append) +fuzz_target!(|data: &[u8]| { + let mut ssz = SszStream::new(); + let mut number_u32 = 0; + if data.len() >= 4 { + number_u32 = u32::from_be_bytes([data[0], data[1], data[2], data[3]]); + } + + ssz.append(&number_u32); + let ssz = ssz.drain(); + + // TODO: change to little endian bytes + // https://github.com/sigp/lighthouse/issues/215 + assert_eq!(ssz.len(), 4); + assert_eq!(number_u32, u32::from_be_bytes([ssz[0], ssz[1], ssz[2], ssz[3]])); +}); diff --git a/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u64_decode.rs b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u64_decode.rs new file mode 100644 index 000000000..63eb60f55 --- /dev/null +++ b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u64_decode.rs @@ -0,0 +1,31 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate ssz; + +use ssz::{DecodeError, Decodable}; + +// Fuzz ssz_decode() +fuzz_target!(|data: &[u8]| { + let result: Result<(u64, usize), DecodeError> = Decodable::ssz_decode(data, 0); + if data.len() >= 8 { + // Valid result + let (number_u64, index) = result.unwrap(); + assert_eq!(index, 8); + // TODO: change to little endian bytes + // https://github.com/sigp/lighthouse/issues/215 + let val = u64::from_be_bytes([ + data[0], + data[1], + data[2], + data[3], + data[4], + data[5], + data[6], + data[7], + ]); + assert_eq!(number_u64, val); + } else { + // Length less then 8 should return error + assert_eq!(result, Err(DecodeError::TooShort)); + } +}); diff --git a/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u64_encode.rs b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u64_encode.rs new file mode 100644 index 000000000..68616e0da --- /dev/null +++ b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u64_encode.rs @@ -0,0 +1,40 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate ssz; + +use ssz::SszStream; + +// Fuzz ssz_encode (via ssz_append) +fuzz_target!(|data: &[u8]| { + let mut ssz = SszStream::new(); + let mut number_u64 = 0; + if data.len() >= 8 { + number_u64 = u64::from_be_bytes([ + data[0], + data[1], + data[2], + data[3], + data[4], + data[5], + data[6], + data[7], + ]); + } + + ssz.append(&number_u64); + let ssz = ssz.drain(); + + // TODO: change to little endian bytes + // https://github.com/sigp/lighthouse/issues/215 + assert_eq!(ssz.len(), 8); + assert_eq!(number_u64, u64::from_be_bytes([ + ssz[0], + ssz[1], + ssz[2], + ssz[3], + ssz[4], + ssz[5], + ssz[6], + ssz[7], + ])); +}); diff --git a/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u8_decode.rs b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u8_decode.rs new file mode 100644 index 000000000..6f17a4c85 --- /dev/null +++ b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u8_decode.rs @@ -0,0 +1,21 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate ssz; + +use ssz::{DecodeError, Decodable}; + +// Fuzz ssz_decode() +fuzz_target!(|data: &[u8]| { + let result: Result<(u8, usize), DecodeError> = Decodable::ssz_decode(data, 0); + if data.len() >= 1 { + // Should have valid result + let (number_u8, index) = result.unwrap(); + // TODO: change to little endian bytes + // https://github.com/sigp/lighthouse/issues/215 + assert_eq!(index, 1); + assert_eq!(number_u8, data[0]); + } else { + // Length of 0 should return error + assert_eq!(result, Err(DecodeError::TooShort)); + } +}); diff --git a/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u8_encode.rs b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u8_encode.rs new file mode 100644 index 000000000..a135f2cd5 --- /dev/null +++ b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_u8_encode.rs @@ -0,0 +1,22 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate ssz; + +use ssz::SszStream; + +// Fuzz ssz_encode (via ssz_append) +fuzz_target!(|data: &[u8]| { + let mut ssz = SszStream::new(); + let mut number_u8 = 0; + if data.len() >= 1 { + number_u8 = data[0]; + } + + ssz.append(&number_u8); + let ssz = ssz.drain(); + + // TODO: change to little endian bytes + // https://github.com/sigp/lighthouse/issues/215 + assert_eq!(number_u8, ssz[0]); + assert_eq!(ssz.len(), 1); +}); diff --git a/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_usize_decode.rs b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_usize_decode.rs new file mode 100644 index 000000000..1458bfae9 --- /dev/null +++ b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_usize_decode.rs @@ -0,0 +1,32 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate ssz; + +use ssz::{DecodeError, Decodable}; + +// Fuzz ssz_decode() +fuzz_target!(|data: &[u8]| { + // Note: we assume architecture is 64 bit -> usize == 64 bits + let result: Result<(usize, usize), DecodeError> = Decodable::ssz_decode(data, 0); + if data.len() >= 8 { + // Valid result + let (number_usize, index) = result.unwrap(); + assert_eq!(index, 8); + // TODO: change to little endian bytes + // https://github.com/sigp/lighthouse/issues/215 + let val = u64::from_be_bytes([ + data[0], + data[1], + data[2], + data[3], + data[4], + data[5], + data[6], + data[7], + ]); + assert_eq!(number_usize, val as usize); + } else { + // Length less then 8 should return error + assert_eq!(result, Err(DecodeError::TooShort)); + } +}); diff --git a/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_usize_encode.rs b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_usize_encode.rs new file mode 100644 index 000000000..d5aa9751f --- /dev/null +++ b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_usize_encode.rs @@ -0,0 +1,40 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate ssz; + +use ssz::SszStream; + +// Fuzz ssz_encode (via ssz_append) +fuzz_target!(|data: &[u8]| { + let mut ssz = SszStream::new(); + let mut number_usize = 0; + if data.len() >= 8 { + number_usize = u64::from_be_bytes([ + data[0], + data[1], + data[2], + data[3], + data[4], + data[5], + data[6], + data[7], + ]) as usize; + } + + ssz.append(&number_usize); + let ssz = ssz.drain(); + + // TODO: change to little endian bytes + // https://github.com/sigp/lighthouse/issues/215 + assert_eq!(ssz.len(), 8); + assert_eq!(number_usize, u64::from_be_bytes([ + ssz[0], + ssz[1], + ssz[2], + ssz[3], + ssz[4], + ssz[5], + ssz[6], + ssz[7], + ]) as usize); +}); diff --git a/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_vec_address_decode.rs b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_vec_address_decode.rs new file mode 100644 index 000000000..6c686df1a --- /dev/null +++ b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_vec_address_decode.rs @@ -0,0 +1,12 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate ethereum_types; +extern crate ssz; + +use ethereum_types::{Address}; +use ssz::{DecodeError, Decodable}; + +// Fuzz ssz_decode() +fuzz_target!(|data: &[u8]| { + let _result: Result<(Vec
, usize), DecodeError> = Decodable::ssz_decode(data, 0); +}); diff --git a/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_vec_bool_decode.rs b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_vec_bool_decode.rs new file mode 100644 index 000000000..25017ef25 --- /dev/null +++ b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_vec_bool_decode.rs @@ -0,0 +1,10 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate ssz; + +use ssz::{DecodeError, Decodable}; + +// Fuzz ssz_decode() +fuzz_target!(|data: &[u8]| { + let _result: Result<(Vec, usize), DecodeError> = Decodable::ssz_decode(data, 0); +}); diff --git a/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_vec_decode.rs b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_vec_decode.rs new file mode 100644 index 000000000..cc1dc09f5 --- /dev/null +++ b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_vec_decode.rs @@ -0,0 +1,12 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate ethereum_types; +extern crate ssz; + +use ethereum_types::{Address, H256}; +use ssz::{DecodeError, Decodable}; + +// Fuzz ssz_decode() +fuzz_target!(|data: &[u8]| { + let _result: Result<(Vec, usize), DecodeError> = Decodable::ssz_decode(data, 0); +}); diff --git a/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_vec_encode.rs b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_vec_encode.rs new file mode 100644 index 000000000..39500b782 --- /dev/null +++ b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_vec_encode.rs @@ -0,0 +1,15 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate ethereum_types; +extern crate ssz; + +use ethereum_types::{Address, H256}; +use ssz::SszStream; + +// Fuzz ssz_encode() +fuzz_target!(|data: &[u8]| { + + let mut ssz = SszStream::new(); + let data_vec = data.to_vec(); + ssz.append(&data_vec); +}); diff --git a/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_vec_u64_decode.rs b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_vec_u64_decode.rs new file mode 100644 index 000000000..ee25a6378 --- /dev/null +++ b/eth2/utils/ssz/fuzz/fuzz_targets/fuzz_target_vec_u64_decode.rs @@ -0,0 +1,10 @@ +#![no_main] +#[macro_use] extern crate libfuzzer_sys; +extern crate ssz; + +use ssz::{DecodeError, Decodable}; + +// Fuzz ssz_decode() +fuzz_target!(|data: &[u8]| { + let _result: Result<(Vec, usize), DecodeError> = Decodable::ssz_decode(data, 0); +}); diff --git a/eth2/utils/ssz/src/impl_decode.rs b/eth2/utils/ssz/src/impl_decode.rs index b13cbeb5d..152e36760 100644 --- a/eth2/utils/ssz/src/impl_decode.rs +++ b/eth2/utils/ssz/src/impl_decode.rs @@ -24,11 +24,30 @@ macro_rules! impl_decodable_for_uint { }; } +macro_rules! impl_decodable_for_u8_array { + ($len: expr) => { + impl Decodable for [u8; $len] { + fn ssz_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), DecodeError> { + if index + $len > bytes.len() { + Err(DecodeError::TooShort) + } else { + let mut array: [u8; $len] = [0; $len]; + array.copy_from_slice(&bytes[index..index + $len]); + + Ok((array, index + $len)) + } + } + } + }; +} + impl_decodable_for_uint!(u16, 16); impl_decodable_for_uint!(u32, 32); impl_decodable_for_uint!(u64, 64); impl_decodable_for_uint!(usize, 64); +impl_decodable_for_u8_array!(4); + impl Decodable for u8 { fn ssz_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), DecodeError> { if index >= bytes.len() { @@ -246,4 +265,12 @@ mod tests { let result: Result<(bool, usize), DecodeError> = decode_ssz(&ssz, 0); assert_eq!(result, Err(DecodeError::Invalid)); } + + #[test] + fn test_decode_u8_array() { + let ssz = vec![0, 1, 2, 3]; + let (result, index): ([u8; 4], usize) = decode_ssz(&ssz, 0).unwrap(); + assert_eq!(index, 4); + assert_eq!(result, [0, 1, 2, 3]); + } } diff --git a/eth2/utils/ssz/src/impl_encode.rs b/eth2/utils/ssz/src/impl_encode.rs index bb1ec42d5..b7d008ccf 100644 --- a/eth2/utils/ssz/src/impl_encode.rs +++ b/eth2/utils/ssz/src/impl_encode.rs @@ -40,12 +40,25 @@ macro_rules! impl_encodable_for_uint { }; } +macro_rules! impl_encodable_for_u8_array { + ($len: expr) => { + impl Encodable for [u8; $len] { + fn ssz_append(&self, s: &mut SszStream) { + let bytes: Vec = self.iter().cloned().collect(); + s.append_encoded_raw(&bytes); + } + } + }; +} + impl_encodable_for_uint!(u8, 8); impl_encodable_for_uint!(u16, 16); impl_encodable_for_uint!(u32, 32); impl_encodable_for_uint!(u64, 64); impl_encodable_for_uint!(usize, 64); +impl_encodable_for_u8_array!(4); + impl Encodable for bool { fn ssz_append(&self, s: &mut SszStream) { let byte = if *self { 0b1000_0000 } else { 0b0000_0000 }; @@ -77,6 +90,7 @@ where #[cfg(test)] mod tests { use super::*; + use crate::ssz_encode; #[test] fn test_ssz_encode_h256() { @@ -226,4 +240,15 @@ mod tests { ssz.append(&x); assert_eq!(ssz.drain(), vec![0b1000_0000]); } + + #[test] + fn test_ssz_encode_u8_array() { + let x: [u8; 4] = [0, 1, 7, 8]; + let ssz = ssz_encode(&x); + assert_eq!(ssz, vec![0, 1, 7, 8]); + + let x: [u8; 4] = [255, 255, 255, 255]; + let ssz = ssz_encode(&x); + assert_eq!(ssz, vec![255, 255, 255, 255]); + } } diff --git a/eth2/utils/ssz/src/impl_tree_hash.rs b/eth2/utils/ssz/src/impl_tree_hash.rs index 54bd7c139..03976f637 100644 --- a/eth2/utils/ssz/src/impl_tree_hash.rs +++ b/eth2/utils/ssz/src/impl_tree_hash.rs @@ -3,55 +3,55 @@ use super::{merkle_hash, ssz_encode, TreeHash}; use hashing::hash; impl TreeHash for u8 { - fn hash_tree_root_internal(&self) -> Vec { + fn hash_tree_root(&self) -> Vec { ssz_encode(self) } } impl TreeHash for u16 { - fn hash_tree_root_internal(&self) -> Vec { + fn hash_tree_root(&self) -> Vec { ssz_encode(self) } } impl TreeHash for u32 { - fn hash_tree_root_internal(&self) -> Vec { + fn hash_tree_root(&self) -> Vec { ssz_encode(self) } } impl TreeHash for u64 { - fn hash_tree_root_internal(&self) -> Vec { + fn hash_tree_root(&self) -> Vec { ssz_encode(self) } } impl TreeHash for usize { - fn hash_tree_root_internal(&self) -> Vec { + fn hash_tree_root(&self) -> Vec { ssz_encode(self) } } impl TreeHash for bool { - fn hash_tree_root_internal(&self) -> Vec { + fn hash_tree_root(&self) -> Vec { ssz_encode(self) } } impl TreeHash for Address { - fn hash_tree_root_internal(&self) -> Vec { + fn hash_tree_root(&self) -> Vec { ssz_encode(self) } } impl TreeHash for H256 { - fn hash_tree_root_internal(&self) -> Vec { + fn hash_tree_root(&self) -> Vec { ssz_encode(self) } } impl TreeHash for [u8] { - fn hash_tree_root_internal(&self) -> Vec { + fn hash_tree_root(&self) -> Vec { if self.len() > 32 { return hash(&self); } @@ -63,12 +63,12 @@ impl TreeHash for Vec where T: TreeHash, { - /// Returns the merkle_hash of a list of hash_tree_root_internal values created + /// Returns the merkle_hash of a list of hash_tree_root values created /// from the given list. /// Note: A byte vector, Vec, must be converted to a slice (as_slice()) /// to be handled properly (i.e. hashed) as byte array. - fn hash_tree_root_internal(&self) -> Vec { - let mut tree_hashes = self.iter().map(|x| x.hash_tree_root_internal()).collect(); + fn hash_tree_root(&self) -> Vec { + let mut tree_hashes = self.iter().map(|x| x.hash_tree_root()).collect(); merkle_hash(&mut tree_hashes) } } @@ -79,7 +79,7 @@ mod tests { #[test] fn test_impl_tree_hash_vec() { - let result = vec![1u32, 2, 3, 4, 5, 6, 7].hash_tree_root_internal(); + let result = vec![1u32, 2, 3, 4, 5, 6, 7].hash_tree_root(); assert_eq!(result.len(), 32); } } diff --git a/eth2/utils/ssz/src/tree_hash.rs b/eth2/utils/ssz/src/tree_hash.rs index 7c1ab35e9..85e56924c 100644 --- a/eth2/utils/ssz/src/tree_hash.rs +++ b/eth2/utils/ssz/src/tree_hash.rs @@ -1,44 +1,31 @@ use hashing::hash; -const SSZ_CHUNK_SIZE: usize = 128; +const BYTES_PER_CHUNK: usize = 32; const HASHSIZE: usize = 32; pub trait TreeHash { - fn hash_tree_root_internal(&self) -> Vec; - fn hash_tree_root(&self) -> Vec { - let mut result = self.hash_tree_root_internal(); - zpad(&mut result, HASHSIZE); - result - } + fn hash_tree_root(&self) -> Vec; } /// Returns a 32 byte hash of 'list' - a vector of byte vectors. /// Note that this will consume 'list'. pub fn merkle_hash(list: &mut Vec>) -> Vec { // flatten list - let (mut chunk_size, mut chunkz) = list_to_blob(list); + let mut chunkz = list_to_blob(list); // get data_len as bytes. It will hashed will the merkle root let mut datalen = list.len().to_le_bytes().to_vec(); zpad(&mut datalen, 32); - // Tree-hash + // merklelize while chunkz.len() > HASHSIZE { let mut new_chunkz: Vec = Vec::new(); - for two_chunks in chunkz.chunks(chunk_size * 2) { - if two_chunks.len() == chunk_size { - // Odd number of chunks - let mut c = two_chunks.to_vec(); - c.append(&mut vec![0; SSZ_CHUNK_SIZE]); - new_chunkz.append(&mut hash(&c)); - } else { - // Hash two chuncks together - new_chunkz.append(&mut hash(two_chunks)); - } + for two_chunks in chunkz.chunks(BYTES_PER_CHUNK * 2) { + // Hash two chuncks together + new_chunkz.append(&mut hash(two_chunks)); } - chunk_size = HASHSIZE; chunkz = new_chunkz; } @@ -46,17 +33,13 @@ pub fn merkle_hash(list: &mut Vec>) -> Vec { hash(&chunkz) } -fn list_to_blob(list: &mut Vec>) -> (usize, Vec) { - let chunk_size = if list.is_empty() || list[0].len() < SSZ_CHUNK_SIZE { - SSZ_CHUNK_SIZE - } else { - list[0].len() - }; - +fn list_to_blob(list: &mut Vec>) -> Vec { + // pack - fit as many many items per chunk as we can and then + // right pad to BYTES_PER_CHUNCK let (items_per_chunk, chunk_count) = if list.is_empty() { (1, 1) } else { - let items_per_chunk = SSZ_CHUNK_SIZE / list[0].len(); + let items_per_chunk = BYTES_PER_CHUNK / list[0].len(); let chunk_count = list.len() / items_per_chunk; (items_per_chunk, chunk_count) }; @@ -64,20 +47,20 @@ fn list_to_blob(list: &mut Vec>) -> (usize, Vec) { let mut chunkz = Vec::new(); if list.is_empty() { // handle and empty list - chunkz.append(&mut vec![0; SSZ_CHUNK_SIZE]); - } else if list[0].len() <= SSZ_CHUNK_SIZE { + chunkz.append(&mut vec![0; BYTES_PER_CHUNK * 2]); + } else if list[0].len() <= BYTES_PER_CHUNK { // just create a blob here; we'll divide into // chunked slices when we merklize - let mut chunk = Vec::with_capacity(chunk_size); + let mut chunk = Vec::with_capacity(BYTES_PER_CHUNK); let mut item_count_in_chunk = 0; - chunkz.reserve(chunk_count * chunk_size); + chunkz.reserve(chunk_count * BYTES_PER_CHUNK); for item in list.iter_mut() { item_count_in_chunk += 1; chunk.append(item); // completed chunk? if item_count_in_chunk == items_per_chunk { - zpad(&mut chunk, chunk_size); + zpad(&mut chunk, BYTES_PER_CHUNK); chunkz.append(&mut chunk); item_count_in_chunk = 0; } @@ -85,18 +68,18 @@ fn list_to_blob(list: &mut Vec>) -> (usize, Vec) { // left-over uncompleted chunk? if item_count_in_chunk != 0 { - zpad(&mut chunk, chunk_size); + zpad(&mut chunk, BYTES_PER_CHUNK); chunkz.append(&mut chunk); } - } else { - // chunks larger than SSZ_CHUNK_SIZE - chunkz.reserve(chunk_count * chunk_size); - for item in list.iter_mut() { - chunkz.append(item); - } } - (chunk_size, chunkz) + // extend the number of chunks to a power of two if necessary + if !chunk_count.is_power_of_two() { + let zero_chunks_count = chunk_count.next_power_of_two() - chunk_count; + chunkz.append(&mut vec![0; zero_chunks_count * BYTES_PER_CHUNK]); + } + + chunkz } /// right pads with zeros making 'bytes' 'size' in length @@ -112,9 +95,9 @@ mod tests { #[test] fn test_merkle_hash() { - let data1 = vec![1; 100]; - let data2 = vec![2; 100]; - let data3 = vec![3; 100]; + let data1 = vec![1; 32]; + let data2 = vec![2; 32]; + let data3 = vec![3; 32]; let mut list = vec![data1, data2, data3]; let result = merkle_hash(&mut list); diff --git a/eth2/utils/ssz_derive/src/lib.rs b/eth2/utils/ssz_derive/src/lib.rs index 0d2e17f76..9ba1de416 100644 --- a/eth2/utils/ssz_derive/src/lib.rs +++ b/eth2/utils/ssz_derive/src/lib.rs @@ -56,10 +56,46 @@ fn get_named_field_idents<'a>(struct_data: &'a syn::DataStruct) -> Vec<&'a syn:: .collect() } +/// Returns a Vec of `syn::Ident` for each named field in the struct, whilst filtering out fields +/// that should not be serialized. +/// +/// # Panics +/// Any unnamed struct field (like in a tuple struct) will raise a panic at compile time. +fn get_serializable_named_field_idents<'a>( + struct_data: &'a syn::DataStruct, +) -> Vec<&'a syn::Ident> { + struct_data + .fields + .iter() + .filter_map(|f| { + if should_skip_serializing(&f) { + None + } else { + Some(match &f.ident { + Some(ref ident) => ident, + _ => panic!("ssz_derive only supports named struct fields."), + }) + } + }) + .collect() +} + +/// Returns true if some field has an attribute declaring it should not be serialized. +/// +/// The field attribute is: `#[ssz(skip_serializing)]` +fn should_skip_serializing(field: &syn::Field) -> bool { + for attr in &field.attrs { + if attr.tts.to_string() == "( skip_serializing )" { + return true; + } + } + false +} + /// Implements `ssz::Encodable` for some `struct`. /// /// Fields are encoded in the order they are defined. -#[proc_macro_derive(Encode)] +#[proc_macro_derive(Encode, attributes(ssz))] pub fn ssz_encode_derive(input: TokenStream) -> TokenStream { let item = parse_macro_input!(input as DeriveInput); @@ -70,7 +106,7 @@ pub fn ssz_encode_derive(input: TokenStream) -> TokenStream { _ => panic!("ssz_derive only supports structs."), }; - let field_idents = get_named_field_idents(&struct_data); + let field_idents = get_serializable_named_field_idents(&struct_data); let output = quote! { impl ssz::Encodable for #name { @@ -84,6 +120,18 @@ pub fn ssz_encode_derive(input: TokenStream) -> TokenStream { output.into() } +/// Returns true if some field has an attribute declaring it should not be deserialized. +/// +/// The field attribute is: `#[ssz(skip_deserializing)]` +fn should_skip_deserializing(field: &syn::Field) -> bool { + for attr in &field.attrs { + if attr.tts.to_string() == "( skip_deserializing )" { + return true; + } + } + false +} + /// Implements `ssz::Decodable` for some `struct`. /// /// Fields are decoded in the order they are defined. @@ -98,26 +146,39 @@ pub fn ssz_decode_derive(input: TokenStream) -> TokenStream { _ => panic!("ssz_derive only supports structs."), }; - let field_idents = get_named_field_idents(&struct_data); + let all_idents = get_named_field_idents(&struct_data); - // Using a var in an iteration always consumes the var, therefore we must make a `fields_a` and - // a `fields_b` in order to perform two loops. - // - // https://github.com/dtolnay/quote/issues/8 - let field_idents_a = &field_idents; - let field_idents_b = &field_idents; + // Build quotes for fields that should be deserialized and those that should be built from + // `Default`. + let mut quotes = vec![]; + for field in &struct_data.fields { + match &field.ident { + Some(ref ident) => { + if should_skip_deserializing(field) { + quotes.push(quote! { + let #ident = <_>::default(); + }); + } else { + quotes.push(quote! { + let (#ident, i) = <_>::ssz_decode(bytes, i)?; + }); + } + } + _ => panic!("ssz_derive only supports named struct fields."), + }; + } let output = quote! { impl ssz::Decodable for #name { fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), ssz::DecodeError> { #( - let (#field_idents_a, i) = <_>::ssz_decode(bytes, i)?; + #quotes )* Ok(( Self { #( - #field_idents_b, + #all_idents, )* }, i @@ -128,10 +189,46 @@ pub fn ssz_decode_derive(input: TokenStream) -> TokenStream { output.into() } +/// Returns a Vec of `syn::Ident` for each named field in the struct, whilst filtering out fields +/// that should not be tree hashed. +/// +/// # Panics +/// Any unnamed struct field (like in a tuple struct) will raise a panic at compile time. +fn get_tree_hashable_named_field_idents<'a>( + struct_data: &'a syn::DataStruct, +) -> Vec<&'a syn::Ident> { + struct_data + .fields + .iter() + .filter_map(|f| { + if should_skip_tree_hash(&f) { + None + } else { + Some(match &f.ident { + Some(ref ident) => ident, + _ => panic!("ssz_derive only supports named struct fields."), + }) + } + }) + .collect() +} + +/// Returns true if some field has an attribute declaring it should not be tree-hashed. +/// +/// The field attribute is: `#[tree_hash(skip_hashing)]` +fn should_skip_tree_hash(field: &syn::Field) -> bool { + for attr in &field.attrs { + if attr.tts.to_string() == "( skip_hashing )" { + return true; + } + } + false +} + /// Implements `ssz::TreeHash` for some `struct`. /// /// Fields are processed in the order they are defined. -#[proc_macro_derive(TreeHash)] +#[proc_macro_derive(TreeHash, attributes(tree_hash))] pub fn ssz_tree_hash_derive(input: TokenStream) -> TokenStream { let item = parse_macro_input!(input as DeriveInput); @@ -142,14 +239,14 @@ pub fn ssz_tree_hash_derive(input: TokenStream) -> TokenStream { _ => panic!("ssz_derive only supports structs."), }; - let field_idents = get_named_field_idents(&struct_data); + let field_idents = get_tree_hashable_named_field_idents(&struct_data); let output = quote! { impl ssz::TreeHash for #name { - fn hash_tree_root_internal(&self) -> Vec { + fn hash_tree_root(&self) -> Vec { let mut list: Vec> = Vec::new(); #( - list.push(self.#field_idents.hash_tree_root_internal()); + list.push(self.#field_idents.hash_tree_root()); )* ssz::merkle_hash(&mut list) @@ -224,7 +321,7 @@ pub fn ssz_signed_root_derive(input: TokenStream) -> TokenStream { fn signed_root(&self) -> Vec { let mut list: Vec> = Vec::new(); #( - list.push(self.#field_idents.hash_tree_root_internal()); + list.push(self.#field_idents.hash_tree_root()); )* ssz::merkle_hash(&mut list) diff --git a/eth2/utils/test_random_derive/src/lib.rs b/eth2/utils/test_random_derive/src/lib.rs index 9a456606c..7920ea695 100644 --- a/eth2/utils/test_random_derive/src/lib.rs +++ b/eth2/utils/test_random_derive/src/lib.rs @@ -4,7 +4,20 @@ use crate::proc_macro::TokenStream; use quote::quote; use syn::{parse_macro_input, DeriveInput}; -#[proc_macro_derive(TestRandom)] +/// Returns true if some field has an attribute declaring it should be generated from default (not +/// randomized). +/// +/// The field attribute is: `#[test_random(default)]` +fn should_use_default(field: &syn::Field) -> bool { + for attr in &field.attrs { + if attr.tts.to_string() == "( default )" { + return true; + } + } + false +} + +#[proc_macro_derive(TestRandom, attributes(test_random))] pub fn test_random_derive(input: TokenStream) -> TokenStream { let derived_input = parse_macro_input!(input as DeriveInput); let name = &derived_input.ident; @@ -14,14 +27,32 @@ pub fn test_random_derive(input: TokenStream) -> TokenStream { _ => panic!("test_random_derive only supports structs."), }; - let field_names = get_named_field_idents(&struct_data); + // Build quotes for fields that should be generated and those that should be built from + // `Default`. + let mut quotes = vec![]; + for field in &struct_data.fields { + match &field.ident { + Some(ref ident) => { + if should_use_default(field) { + quotes.push(quote! { + #ident: <_>::default(), + }); + } else { + quotes.push(quote! { + #ident: <_>::random_for_test(rng), + }); + } + } + _ => panic!("test_random_derive only supports named struct fields."), + }; + } let output = quote! { impl TestRandom for #name { fn random_for_test(rng: &mut T) -> Self { Self { #( - #field_names: <_>::random_for_test(rng), + #quotes )* } } @@ -30,14 +61,3 @@ pub fn test_random_derive(input: TokenStream) -> TokenStream { output.into() } - -fn get_named_field_idents(struct_data: &syn::DataStruct) -> Vec<(&syn::Ident)> { - struct_data - .fields - .iter() - .map(|f| match &f.ident { - Some(ref ident) => ident, - _ => panic!("test_random_derive only supports named struct fields."), - }) - .collect() -} diff --git a/validator_client/src/block_producer_service/beacon_block_grpc_client.rs b/validator_client/src/block_producer_service/beacon_block_grpc_client.rs index 6ce5c0fa0..04a02a221 100644 --- a/validator_client/src/block_producer_service/beacon_block_grpc_client.rs +++ b/validator_client/src/block_producer_service/beacon_block_grpc_client.rs @@ -50,15 +50,15 @@ impl BeaconNode for BeaconBlockGrpcClient { // TODO: this conversion is incomplete; fix it. Ok(Some(BeaconBlock { slot: Slot::new(block.get_slot()), - parent_root: Hash256::zero(), + previous_block_root: Hash256::zero(), state_root: Hash256::zero(), - randao_reveal, - eth1_data: Eth1Data { - deposit_root: Hash256::zero(), - block_hash: Hash256::zero(), - }, signature, body: BeaconBlockBody { + randao_reveal, + eth1_data: Eth1Data { + deposit_root: Hash256::zero(), + block_hash: Hash256::zero(), + }, proposer_slashings: vec![], attester_slashings: vec![], attestations: vec![], @@ -83,7 +83,7 @@ impl BeaconNode for BeaconBlockGrpcClient { let mut grpc_block = GrpcBeaconBlock::new(); grpc_block.set_slot(block.slot.as_u64()); grpc_block.set_block_root(vec![0]); - grpc_block.set_randao_reveal(ssz_encode(&block.randao_reveal)); + grpc_block.set_randao_reveal(ssz_encode(&block.body.randao_reveal)); grpc_block.set_signature(ssz_encode(&block.signature)); req.set_block(grpc_block); diff --git a/validator_client/src/duties/epoch_duties.rs b/validator_client/src/duties/epoch_duties.rs index 35668b4a9..71f5f26ab 100644 --- a/validator_client/src/duties/epoch_duties.rs +++ b/validator_client/src/duties/epoch_duties.rs @@ -81,8 +81,8 @@ impl DutiesReader for EpochDutiesMap { // // It will almost certainly cause signatures to fail verification. Ok(Fork { - previous_version: 0, - current_version: 0, + previous_version: [0; 4], + current_version: [0; 4], epoch: Epoch::new(0), }) }