From 977f3edfb688a67ad098d45a52834a4bc4294c77 Mon Sep 17 00:00:00 2001 From: Kirk Baird Date: Fri, 15 Feb 2019 13:58:14 +1100 Subject: [PATCH 001/132] Add domain to all signature funcitons, modify validate_proof_of_possession() --- .../src/attestation_aggregator.rs | 2 +- .../src/validator_harness/local_signer.rs | 16 +++++----- eth2/attester/src/lib.rs | 3 +- eth2/attester/src/test_utils/local_signer.rs | 4 +-- eth2/attester/src/traits.rs | 2 +- eth2/block_producer/src/lib.rs | 4 +-- .../src/test_utils/local_signer.rs | 8 ++--- eth2/block_producer/src/traits.rs | 4 +-- eth2/fork_choice/src/optimised_lmd_ghost.rs | 11 ++++--- eth2/fork_choice/src/slow_lmd_ghost.rs | 2 +- .../state_processing/src/block_processable.rs | 10 +++--- eth2/types/src/attestation.rs | 5 ++- eth2/types/src/beacon_state.rs | 32 ++++++++++++++++--- eth2/types/src/fork.rs | 16 ++++++++++ eth2/types/src/test_utils/signature.rs | 2 +- eth2/utils/bls/Cargo.toml | 2 +- eth2/utils/bls/src/aggregate_signature.rs | 6 ++-- eth2/utils/bls/src/lib.rs | 14 +++----- eth2/utils/bls/src/signature.rs | 20 ++++++------ 19 files changed, 98 insertions(+), 65 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_aggregator.rs b/beacon_node/beacon_chain/src/attestation_aggregator.rs index 149f0d60d..e8576276c 100644 --- a/beacon_node/beacon_chain/src/attestation_aggregator.rs +++ b/beacon_node/beacon_chain/src/attestation_aggregator.rs @@ -112,7 +112,7 @@ impl AttestationAggregator { if !free_attestation .signature - .verify(&signable_message, &validator_record.pubkey) + .verify(&signable_message, spec.domain_attestation, &validator_record.pubkey) { return Ok(Outcome { valid: false, diff --git a/beacon_node/beacon_chain/test_harness/src/validator_harness/local_signer.rs b/beacon_node/beacon_chain/test_harness/src/validator_harness/local_signer.rs index 8e901b057..f176a6889 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator_harness/local_signer.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator_harness/local_signer.rs @@ -25,23 +25,23 @@ impl LocalSigner { } /// Sign some message. - fn bls_sign(&self, message: &[u8]) -> Option { - Some(Signature::new(message, &self.keypair.sk)) + fn bls_sign(&self, message: &[u8], domain: u64) -> Option { + Some(Signature::new(message, domain, &self.keypair.sk)) } } impl BlockProposerSigner for LocalSigner { - fn sign_block_proposal(&self, message: &[u8]) -> Option { - self.bls_sign(message) + fn sign_block_proposal(&self, message: &[u8], domain: u64) -> Option { + self.bls_sign(message, domain) } - fn sign_randao_reveal(&self, message: &[u8]) -> Option { - self.bls_sign(message) + fn sign_randao_reveal(&self, message: &[u8], domain: u64) -> Option { + self.bls_sign(message, domain) } } impl AttesterSigner for LocalSigner { - fn sign_attestation_message(&self, message: &[u8]) -> Option { - self.bls_sign(message) + fn sign_attestation_message(&self, message: &[u8], domain: u64) -> Option { + self.bls_sign(message, domain) } } diff --git a/eth2/attester/src/lib.rs b/eth2/attester/src/lib.rs index 7352dd2ea..f2bbd6db3 100644 --- a/eth2/attester/src/lib.rs +++ b/eth2/attester/src/lib.rs @@ -10,6 +10,7 @@ pub use self::traits::{ }; const PHASE_0_CUSTODY_BIT: bool = false; +const DOMAIN_ATTESTATION: u64 = 1; #[derive(Debug, PartialEq)] pub enum PollOutcome { @@ -137,7 +138,7 @@ impl Attester Option { - Some(Signature::new(message, &self.keypair.sk)) + fn sign_attestation_message(&self, message: &[u8], domain: u64) -> Option { + Some(Signature::new(message, domain, &self.keypair.sk)) } } diff --git a/eth2/attester/src/traits.rs b/eth2/attester/src/traits.rs index 53bce3aaa..6062460cb 100644 --- a/eth2/attester/src/traits.rs +++ b/eth2/attester/src/traits.rs @@ -45,5 +45,5 @@ pub trait DutiesReader: Send + Sync { /// Signs message using an internally-maintained private key. pub trait Signer { - fn sign_attestation_message(&self, message: &[u8]) -> Option; + fn sign_attestation_message(&self, message: &[u8], domain: u64) -> Option; } diff --git a/eth2/block_producer/src/lib.rs b/eth2/block_producer/src/lib.rs index f6a0fd6df..e5651780a 100644 --- a/eth2/block_producer/src/lib.rs +++ b/eth2/block_producer/src/lib.rs @@ -134,7 +134,7 @@ impl BlockProducer return Ok(PollOutcome::SignerRejection(slot)), Some(signature) => signature, } @@ -168,7 +168,7 @@ impl BlockProducer None, Some(signature) => { diff --git a/eth2/block_producer/src/test_utils/local_signer.rs b/eth2/block_producer/src/test_utils/local_signer.rs index 0ebefa29d..d7f490c30 100644 --- a/eth2/block_producer/src/test_utils/local_signer.rs +++ b/eth2/block_producer/src/test_utils/local_signer.rs @@ -25,11 +25,11 @@ impl LocalSigner { } impl Signer for LocalSigner { - fn sign_block_proposal(&self, message: &[u8]) -> Option { - Some(Signature::new(message, &self.keypair.sk)) + fn sign_block_proposal(&self, message: &[u8], domain: u64) -> Option { + Some(Signature::new(message, domain, &self.keypair.sk)) } - fn sign_randao_reveal(&self, message: &[u8]) -> Option { - Some(Signature::new(message, &self.keypair.sk)) + fn sign_randao_reveal(&self, message: &[u8], domain: u64) -> Option { + Some(Signature::new(message, domain, &self.keypair.sk)) } } diff --git a/eth2/block_producer/src/traits.rs b/eth2/block_producer/src/traits.rs index 5eb27bce7..c6e57d833 100644 --- a/eth2/block_producer/src/traits.rs +++ b/eth2/block_producer/src/traits.rs @@ -44,6 +44,6 @@ pub trait DutiesReader: Send + Sync { /// Signs message using an internally-maintained private key. pub trait Signer { - fn sign_block_proposal(&self, message: &[u8]) -> Option; - fn sign_randao_reveal(&self, message: &[u8]) -> Option; + fn sign_block_proposal(&self, message: &[u8], domain: u64) -> Option; + fn sign_randao_reveal(&self, message: &[u8], domain: u64) -> Option; } diff --git a/eth2/fork_choice/src/optimised_lmd_ghost.rs b/eth2/fork_choice/src/optimised_lmd_ghost.rs index 7104834cb..6b73c2a8f 100644 --- a/eth2/fork_choice/src/optimised_lmd_ghost.rs +++ b/eth2/fork_choice/src/optimised_lmd_ghost.rs @@ -31,7 +31,8 @@ use std::collections::HashMap; use std::sync::Arc; use types::{ readers::BeaconBlockReader, - slot_epoch_height::{Height, Slot}, + slot_epoch::Slot, + slot_height::SlotHeight, validator_registry::get_active_validator_indices, BeaconBlock, Hash256, }; @@ -77,7 +78,7 @@ pub struct OptimisedLMDGhost { block_store: Arc>, /// State storage access. state_store: Arc>, - max_known_height: Height, + max_known_height: SlotHeight, } impl OptimisedLMDGhost @@ -93,7 +94,7 @@ where ancestors: vec![HashMap::new(); 16], latest_attestation_targets: HashMap::new(), children: HashMap::new(), - max_known_height: Height::new(0), + max_known_height: SlotHeight::new(0), block_store, state_store, } @@ -137,7 +138,7 @@ where } /// Gets the ancestor at a given height `at_height` of a block specified by `block_hash`. - fn get_ancestor(&mut self, block_hash: Hash256, at_height: Height) -> Option { + fn get_ancestor(&mut self, block_hash: Hash256, at_height: SlotHeight) -> Option { // return None if we can't get the block from the db. let block_height = { let block_slot = self @@ -186,7 +187,7 @@ where fn get_clear_winner( &mut self, latest_votes: &HashMap, - block_height: Height, + block_height: SlotHeight, ) -> Option { // map of vote counts for every hash at this height let mut current_votes: HashMap = HashMap::new(); diff --git a/eth2/fork_choice/src/slow_lmd_ghost.rs b/eth2/fork_choice/src/slow_lmd_ghost.rs index e0e347cef..44b429fa8 100644 --- a/eth2/fork_choice/src/slow_lmd_ghost.rs +++ b/eth2/fork_choice/src/slow_lmd_ghost.rs @@ -29,7 +29,7 @@ use std::collections::HashMap; use std::sync::Arc; use types::{ readers::{BeaconBlockReader, BeaconStateReader}, - slot_epoch_height::Slot, + slot_epoch::Slot, validator_registry::get_active_validator_indices, BeaconBlock, Hash256, }; diff --git a/eth2/state_processing/src/block_processable.rs b/eth2/state_processing/src/block_processable.rs index f043a723d..1bf6022ec 100644 --- a/eth2/state_processing/src/block_processable.rs +++ b/eth2/state_processing/src/block_processable.rs @@ -374,14 +374,12 @@ fn validate_attestation_signature_optional( Ok(()) } -fn get_domain(_fork: &Fork, _epoch: Epoch, _domain_type: u64) -> u64 { - // TODO: stubbed out. - 0 +fn get_domain(fork: &Fork, epoch: Epoch, domain_type: u64) -> u64 { + fork.get_domain(epoch, domain_type) } -fn bls_verify(pubkey: &PublicKey, message: &[u8], signature: &Signature, _domain: u64) -> bool { - // TODO: add domain - signature.verify(message, pubkey) +fn bls_verify(pubkey: &PublicKey, message: &[u8], signature: &Signature, domain: u64) -> bool { + signature.verify(message, domain, pubkey) } impl From for Error { diff --git a/eth2/types/src/attestation.rs b/eth2/types/src/attestation.rs index eb375d490..be0b12d9e 100644 --- a/eth2/types/src/attestation.rs +++ b/eth2/types/src/attestation.rs @@ -25,11 +25,10 @@ impl Attestation { &self, group_public_key: &AggregatePublicKey, custody_bit: bool, - // TODO: use domain. - _domain: u64, + domain: u64, ) -> bool { self.aggregate_signature - .verify(&self.signable_message(custody_bit), group_public_key) + .verify(&self.signable_message(custody_bit), domain, group_public_key) } } diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index ed53bfea9..278569609 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -1,10 +1,9 @@ use crate::test_utils::TestRandom; use crate::{ validator::StatusFlags, validator_registry::get_active_validator_indices, AttestationData, - Bitfield, ChainSpec, Crosslink, Deposit, Epoch, Eth1Data, Eth1DataVote, Fork, Hash256, + Bitfield, ChainSpec, Crosslink, Deposit, DepositInput, Epoch, Eth1Data, Eth1DataVote, Fork, Hash256, PendingAttestation, PublicKey, Signature, Slot, Validator, }; -use bls::verify_proof_of_possession; use honey_badger_split::SplitExt; use rand::RngCore; use serde_derive::Serialize; @@ -587,6 +586,32 @@ impl BeaconState { self.validator_registry_update_epoch = current_epoch; } + + /// Confirm validator owns PublicKey + pub fn validate_proof_of_possession( + &self, + pubkey: PublicKey, + proof_of_possession: Signature, + withdrawal_credentials: Hash256, + spec: &ChainSpec + ) -> bool { + let proof_of_possession_data = DepositInput { + pubkey: pubkey.clone(), + withdrawal_credentials, + proof_of_possession: proof_of_possession.clone(), + }; + + proof_of_possession.verify( + &proof_of_possession_data.hash_tree_root(), + self.fork.get_domain( + self.slot.epoch(spec.epoch_length), + spec.domain_deposit, + ), + &pubkey, + ) + } + + /// Process a validator deposit, returning the validator index if the deposit is valid. /// /// Spec v0.2.0 @@ -598,8 +623,7 @@ impl BeaconState { withdrawal_credentials: Hash256, spec: &ChainSpec, ) -> Result { - // TODO: ensure verify proof-of-possession represents the spec accurately. - if !verify_proof_of_possession(&proof_of_possession, &pubkey) { + if !self.validate_proof_of_possession(pubkey.clone(), proof_of_possession, withdrawal_credentials, &spec) { return Err(()); } diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs index 1c96a34ac..c103a2653 100644 --- a/eth2/types/src/fork.rs +++ b/eth2/types/src/fork.rs @@ -10,6 +10,22 @@ pub struct Fork { pub epoch: Epoch, } +impl Fork { + /// Return the fork version of the given ``epoch``. + pub fn get_fork_version(&self, epoch: Epoch) -> u64 { + if epoch < self.epoch { + return self.previous_version; + } + self.current_version + } + + /// Get the domain number that represents the fork meta and signature domain. + pub fn get_domain(&self, epoch: Epoch, domain_type: u64) -> u64 { + let fork_version = self.get_fork_version(epoch); + fork_version * u64::pow(2,32) + domain_type + } +} + impl Encodable for Fork { fn ssz_append(&self, s: &mut SszStream) { s.append(&self.previous_version); diff --git a/eth2/types/src/test_utils/signature.rs b/eth2/types/src/test_utils/signature.rs index 9ec7aec60..d9995835a 100644 --- a/eth2/types/src/test_utils/signature.rs +++ b/eth2/types/src/test_utils/signature.rs @@ -8,6 +8,6 @@ impl TestRandom for Signature { let mut message = vec![0; 32]; rng.fill_bytes(&mut message); - Signature::new(&message, &secret_key) + Signature::new(&message, 0, &secret_key) } } diff --git a/eth2/utils/bls/Cargo.toml b/eth2/utils/bls/Cargo.toml index 465510c59..c8204ca7a 100644 --- a/eth2/utils/bls/Cargo.toml +++ b/eth2/utils/bls/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Paul Hauner "] edition = "2018" [dependencies] -bls-aggregates = { git = "https://github.com/sigp/signature-schemes", tag = "v0.3.0" } +bls-aggregates = { git = "https://github.com/sigp/signature-schemes", tag = "0.4.1" } hashing = { path = "../hashing" } hex = "0.3" serde = "1.0" diff --git a/eth2/utils/bls/src/aggregate_signature.rs b/eth2/utils/bls/src/aggregate_signature.rs index 6fed183f0..b684c2b5b 100644 --- a/eth2/utils/bls/src/aggregate_signature.rs +++ b/eth2/utils/bls/src/aggregate_signature.rs @@ -27,8 +27,8 @@ impl AggregateSignature { /// /// Only returns `true` if the set of keys in the `AggregatePublicKey` match the set of keys /// that signed the `AggregateSignature`. - pub fn verify(&self, msg: &[u8], aggregate_public_key: &AggregatePublicKey) -> bool { - self.0.verify(msg, aggregate_public_key) + pub fn verify(&self, msg: &[u8], domain: u64, aggregate_public_key: &AggregatePublicKey) -> bool { + self.0.verify(msg, domain, aggregate_public_key) } } @@ -73,7 +73,7 @@ mod tests { let keypair = Keypair::random(); let mut original = AggregateSignature::new(); - original.add(&Signature::new(&[42, 42], &keypair.sk)); + original.add(&Signature::new(&[42, 42], 0, &keypair.sk)); let bytes = ssz_encode(&original); let (decoded, _) = AggregateSignature::ssz_decode(&bytes, 0).unwrap(); diff --git a/eth2/utils/bls/src/lib.rs b/eth2/utils/bls/src/lib.rs index 646047d18..39d4a95f2 100644 --- a/eth2/utils/bls/src/lib.rs +++ b/eth2/utils/bls/src/lib.rs @@ -29,24 +29,18 @@ fn extend_if_needed(hash: &mut Vec) { /// For some signature and public key, ensure that the signature message was the public key and it /// was signed by the secret key that corresponds to that public key. -pub fn verify_proof_of_possession(sig: &Signature, pubkey: &PublicKey) -> bool { - let mut hash = hash(&ssz_encode(pubkey)); - extend_if_needed(&mut hash); - sig.verify_hashed(&hash, &pubkey) -} + pub fn create_proof_of_possession(keypair: &Keypair) -> Signature { - let mut hash = hash(&ssz_encode(&keypair.pk)); - extend_if_needed(&mut hash); - Signature::new_hashed(&hash, &keypair.sk) + Signature::new(&ssz_encode(&keypair.pk), 0, &keypair.sk) } pub fn bls_verify_aggregate( pubkey: &AggregatePublicKey, message: &[u8], signature: &AggregateSignature, - _domain: u64, + domain: u64, ) -> bool { // TODO: add domain - signature.verify(message, pubkey) + signature.verify(message, domain, pubkey) } diff --git a/eth2/utils/bls/src/signature.rs b/eth2/utils/bls/src/signature.rs index 396e4eab7..61440498e 100644 --- a/eth2/utils/bls/src/signature.rs +++ b/eth2/utils/bls/src/signature.rs @@ -14,24 +14,24 @@ pub struct Signature(RawSignature); impl Signature { /// Instantiate a new Signature from a message and a SecretKey. - pub fn new(msg: &[u8], sk: &SecretKey) -> Self { - Signature(RawSignature::new(msg, sk.as_raw())) + pub fn new(msg: &[u8], domain: u64, sk: &SecretKey) -> Self { + Signature(RawSignature::new(msg, domain, sk.as_raw())) } /// Instantiate a new Signature from a message and a SecretKey, where the message has already /// been hashed. - pub fn new_hashed(msg_hashed: &[u8], sk: &SecretKey) -> Self { - Signature(RawSignature::new_hashed(msg_hashed, sk.as_raw())) + pub fn new_hashed(x_real_hashed: &[u8], x_imaginary_hashed: &[u8], sk: &SecretKey) -> Self { + Signature(RawSignature::new_hashed(x_real_hashed, x_imaginary_hashed, sk.as_raw())) } /// Verify the Signature against a PublicKey. - pub fn verify(&self, msg: &[u8], pk: &PublicKey) -> bool { - self.0.verify(msg, pk.as_raw()) + pub fn verify(&self, msg: &[u8], domain: u64, pk: &PublicKey) -> bool { + self.0.verify(msg, domain, pk.as_raw()) } /// Verify the Signature against a PublicKey, where the message has already been hashed. - pub fn verify_hashed(&self, msg_hash: &[u8], pk: &PublicKey) -> bool { - self.0.verify_hashed(msg_hash, pk.as_raw()) + pub fn verify_hashed(&self, x_real_hashed: &[u8], x_imaginary_hashed: &[u8], pk: &PublicKey) -> bool { + self.0.verify_hashed(x_real_hashed, x_imaginary_hashed, pk.as_raw()) } /// Returns the underlying signature. @@ -41,7 +41,7 @@ impl Signature { /// Returns a new empty signature. pub fn empty_signature() -> Self { - let empty: Vec = vec![0; 97]; + let empty: Vec = vec![0; 96]; Signature(RawSignature::from_bytes(&empty).unwrap()) } } @@ -85,7 +85,7 @@ mod tests { pub fn test_ssz_round_trip() { let keypair = Keypair::random(); - let original = Signature::new(&[42, 42], &keypair.sk); + let original = Signature::new(&[42, 42], 0, &keypair.sk); let bytes = ssz_encode(&original); let (decoded, _) = Signature::ssz_decode(&bytes, 0).unwrap(); From 9c4a1f1d1f6814dee9f1761ec2ae4cbb815c8ca7 Mon Sep 17 00:00:00 2001 From: Kirk Baird Date: Mon, 18 Feb 2019 10:50:40 +1100 Subject: [PATCH 002/132] Update to signature-scheme 0.5.2 --- .../src/attestation_aggregator.rs | 9 +++--- eth2/attester/src/lib.rs | 6 ++-- eth2/block_producer/src/lib.rs | 13 ++++---- eth2/fork_choice/src/optimised_lmd_ghost.rs | 7 ++--- eth2/fork_choice/src/protolambda_lmd_ghost.rs | 1 + eth2/types/src/attestation.rs | 7 +++-- eth2/types/src/beacon_state.rs | 20 +++++++------ eth2/types/src/fork.rs | 2 +- eth2/utils/bls/Cargo.toml | 2 +- eth2/utils/bls/src/aggregate_signature.rs | 7 ++++- eth2/utils/bls/src/lib.rs | 1 - eth2/utils/bls/src/signature.rs | 30 ++++++++++++++----- 12 files changed, 67 insertions(+), 38 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_aggregator.rs b/beacon_node/beacon_chain/src/attestation_aggregator.rs index e8576276c..abedf62f6 100644 --- a/beacon_node/beacon_chain/src/attestation_aggregator.rs +++ b/beacon_node/beacon_chain/src/attestation_aggregator.rs @@ -110,10 +110,11 @@ impl AttestationAggregator { Message::BadValidatorIndex ); - if !free_attestation - .signature - .verify(&signable_message, spec.domain_attestation, &validator_record.pubkey) - { + if !free_attestation.signature.verify( + &signable_message, + spec.domain_attestation, + &validator_record.pubkey, + ) { return Ok(Outcome { valid: false, message: Message::BadSignature, diff --git a/eth2/attester/src/lib.rs b/eth2/attester/src/lib.rs index f2bbd6db3..13a1d72bb 100644 --- a/eth2/attester/src/lib.rs +++ b/eth2/attester/src/lib.rs @@ -137,8 +137,10 @@ impl Attester Option { self.store_produce(attestation_data); - self.signer - .sign_attestation_message(&attestation_data.signable_message(PHASE_0_CUSTODY_BIT)[..], DOMAIN_ATTESTATION) + self.signer.sign_attestation_message( + &attestation_data.signable_message(PHASE_0_CUSTODY_BIT)[..], + DOMAIN_ATTESTATION, + ) } /// Returns `true` if signing some attestation_data is safe (non-slashable). diff --git a/eth2/block_producer/src/lib.rs b/eth2/block_producer/src/lib.rs index e5651780a..fefaa7c04 100644 --- a/eth2/block_producer/src/lib.rs +++ b/eth2/block_producer/src/lib.rs @@ -134,7 +134,10 @@ impl BlockProducer return Ok(PollOutcome::SignerRejection(slot)), Some(signature) => signature, } @@ -166,10 +169,10 @@ impl BlockProducer Option { self.store_produce(&block); - match self - .signer - .sign_block_proposal(&block.proposal_root(&self.spec)[..], self.spec.domain_proposal) - { + match self.signer.sign_block_proposal( + &block.proposal_root(&self.spec)[..], + self.spec.domain_proposal, + ) { None => None, Some(signature) => { block.signature = signature; diff --git a/eth2/fork_choice/src/optimised_lmd_ghost.rs b/eth2/fork_choice/src/optimised_lmd_ghost.rs index 6b73c2a8f..dcf9c8380 100644 --- a/eth2/fork_choice/src/optimised_lmd_ghost.rs +++ b/eth2/fork_choice/src/optimised_lmd_ghost.rs @@ -30,11 +30,8 @@ use fast_math::log2_raw; use std::collections::HashMap; use std::sync::Arc; use types::{ - readers::BeaconBlockReader, - slot_epoch::Slot, - slot_height::SlotHeight, - validator_registry::get_active_validator_indices, - BeaconBlock, Hash256, + readers::BeaconBlockReader, slot_epoch::Slot, slot_height::SlotHeight, + validator_registry::get_active_validator_indices, BeaconBlock, Hash256, }; //TODO: Pruning - Children diff --git a/eth2/fork_choice/src/protolambda_lmd_ghost.rs b/eth2/fork_choice/src/protolambda_lmd_ghost.rs index e69de29bb..8b1378917 100644 --- a/eth2/fork_choice/src/protolambda_lmd_ghost.rs +++ b/eth2/fork_choice/src/protolambda_lmd_ghost.rs @@ -0,0 +1 @@ + diff --git a/eth2/types/src/attestation.rs b/eth2/types/src/attestation.rs index be0b12d9e..2c4281fff 100644 --- a/eth2/types/src/attestation.rs +++ b/eth2/types/src/attestation.rs @@ -27,8 +27,11 @@ impl Attestation { custody_bit: bool, domain: u64, ) -> bool { - self.aggregate_signature - .verify(&self.signable_message(custody_bit), domain, group_public_key) + self.aggregate_signature.verify( + &self.signable_message(custody_bit), + domain, + group_public_key, + ) } } diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 278569609..34d0a5a1f 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -1,8 +1,8 @@ use crate::test_utils::TestRandom; use crate::{ validator::StatusFlags, validator_registry::get_active_validator_indices, AttestationData, - Bitfield, ChainSpec, Crosslink, Deposit, DepositInput, Epoch, Eth1Data, Eth1DataVote, Fork, Hash256, - PendingAttestation, PublicKey, Signature, Slot, Validator, + Bitfield, ChainSpec, Crosslink, Deposit, DepositInput, Epoch, Eth1Data, Eth1DataVote, Fork, + Hash256, PendingAttestation, PublicKey, Signature, Slot, Validator, }; use honey_badger_split::SplitExt; use rand::RngCore; @@ -593,7 +593,7 @@ impl BeaconState { pubkey: PublicKey, proof_of_possession: Signature, withdrawal_credentials: Hash256, - spec: &ChainSpec + spec: &ChainSpec, ) -> bool { let proof_of_possession_data = DepositInput { pubkey: pubkey.clone(), @@ -603,15 +603,12 @@ impl BeaconState { proof_of_possession.verify( &proof_of_possession_data.hash_tree_root(), - self.fork.get_domain( - self.slot.epoch(spec.epoch_length), - spec.domain_deposit, - ), + self.fork + .get_domain(self.slot.epoch(spec.epoch_length), spec.domain_deposit), &pubkey, ) } - /// Process a validator deposit, returning the validator index if the deposit is valid. /// /// Spec v0.2.0 @@ -623,7 +620,12 @@ impl BeaconState { withdrawal_credentials: Hash256, spec: &ChainSpec, ) -> Result { - if !self.validate_proof_of_possession(pubkey.clone(), proof_of_possession, withdrawal_credentials, &spec) { + if !self.validate_proof_of_possession( + pubkey.clone(), + proof_of_possession, + withdrawal_credentials, + &spec, + ) { return Err(()); } diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs index c103a2653..67a8c90eb 100644 --- a/eth2/types/src/fork.rs +++ b/eth2/types/src/fork.rs @@ -22,7 +22,7 @@ impl Fork { /// Get the domain number that represents the fork meta and signature domain. pub fn get_domain(&self, epoch: Epoch, domain_type: u64) -> u64 { let fork_version = self.get_fork_version(epoch); - fork_version * u64::pow(2,32) + domain_type + fork_version * u64::pow(2, 32) + domain_type } } diff --git a/eth2/utils/bls/Cargo.toml b/eth2/utils/bls/Cargo.toml index c8204ca7a..7a436307b 100644 --- a/eth2/utils/bls/Cargo.toml +++ b/eth2/utils/bls/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Paul Hauner "] edition = "2018" [dependencies] -bls-aggregates = { git = "https://github.com/sigp/signature-schemes", tag = "0.4.1" } +bls-aggregates = { git = "https://github.com/sigp/signature-schemes", tag = "0.5.2" } hashing = { path = "../hashing" } hex = "0.3" serde = "1.0" diff --git a/eth2/utils/bls/src/aggregate_signature.rs b/eth2/utils/bls/src/aggregate_signature.rs index b684c2b5b..8463b26b3 100644 --- a/eth2/utils/bls/src/aggregate_signature.rs +++ b/eth2/utils/bls/src/aggregate_signature.rs @@ -27,7 +27,12 @@ impl AggregateSignature { /// /// Only returns `true` if the set of keys in the `AggregatePublicKey` match the set of keys /// that signed the `AggregateSignature`. - pub fn verify(&self, msg: &[u8], domain: u64, aggregate_public_key: &AggregatePublicKey) -> bool { + pub fn verify( + &self, + msg: &[u8], + domain: u64, + aggregate_public_key: &AggregatePublicKey, + ) -> bool { self.0.verify(msg, domain, aggregate_public_key) } } diff --git a/eth2/utils/bls/src/lib.rs b/eth2/utils/bls/src/lib.rs index 39d4a95f2..074929b32 100644 --- a/eth2/utils/bls/src/lib.rs +++ b/eth2/utils/bls/src/lib.rs @@ -30,7 +30,6 @@ fn extend_if_needed(hash: &mut Vec) { /// For some signature and public key, ensure that the signature message was the public key and it /// was signed by the secret key that corresponds to that public key. - pub fn create_proof_of_possession(keypair: &Keypair) -> Signature { Signature::new(&ssz_encode(&keypair.pk), 0, &keypair.sk) } diff --git a/eth2/utils/bls/src/signature.rs b/eth2/utils/bls/src/signature.rs index 61440498e..23b0c0834 100644 --- a/eth2/utils/bls/src/signature.rs +++ b/eth2/utils/bls/src/signature.rs @@ -21,7 +21,11 @@ impl Signature { /// Instantiate a new Signature from a message and a SecretKey, where the message has already /// been hashed. pub fn new_hashed(x_real_hashed: &[u8], x_imaginary_hashed: &[u8], sk: &SecretKey) -> Self { - Signature(RawSignature::new_hashed(x_real_hashed, x_imaginary_hashed, sk.as_raw())) + Signature(RawSignature::new_hashed( + x_real_hashed, + x_imaginary_hashed, + sk.as_raw(), + )) } /// Verify the Signature against a PublicKey. @@ -30,8 +34,14 @@ impl Signature { } /// Verify the Signature against a PublicKey, where the message has already been hashed. - pub fn verify_hashed(&self, x_real_hashed: &[u8], x_imaginary_hashed: &[u8], pk: &PublicKey) -> bool { - self.0.verify_hashed(x_real_hashed, x_imaginary_hashed, pk.as_raw()) + pub fn verify_hashed( + &self, + x_real_hashed: &[u8], + x_imaginary_hashed: &[u8], + pk: &PublicKey, + ) -> bool { + self.0 + .verify_hashed(x_real_hashed, x_imaginary_hashed, pk.as_raw()) } /// Returns the underlying signature. @@ -41,7 +51,9 @@ impl Signature { /// Returns a new empty signature. pub fn empty_signature() -> Self { - let empty: Vec = vec![0; 96]; + let mut empty: Vec = vec![0; 96]; + // TODO: Modify the way flags are used (b_flag should not be used for empty_signature in the future) + empty[0] += u8::pow(2, 6); Signature(RawSignature::from_bytes(&empty).unwrap()) } } @@ -99,9 +111,13 @@ mod tests { let sig_as_bytes: Vec = sig.as_raw().as_bytes(); - assert_eq!(sig_as_bytes.len(), 97); - for one_byte in sig_as_bytes.iter() { - assert_eq!(*one_byte, 0); + assert_eq!(sig_as_bytes.len(), 96); + for (i, one_byte) in sig_as_bytes.iter().enumerate() { + if i == 0 { + assert_eq!(*one_byte, u8::pow(2, 6)); + } else { + assert_eq!(*one_byte, 0); + } } } } From 21d75f18536aec031dc772a86cf57bae20f8abd2 Mon Sep 17 00:00:00 2001 From: Kirk Baird Date: Mon, 18 Feb 2019 12:06:47 +1100 Subject: [PATCH 003/132] Use verify_proof_of_possession --- eth2/types/src/beacon_state.rs | 19 +++++++++++++------ eth2/utils/bls/src/lib.rs | 10 ++++++++-- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 18b5fc989..2bdf85fcc 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -4,6 +4,7 @@ use crate::{ Bitfield, ChainSpec, Crosslink, Deposit, DepositInput, Epoch, Eth1Data, Eth1DataVote, Fork, Hash256, PendingAttestation, PublicKey, Signature, Slot, Validator, }; +use bls::verify_proof_of_possession; use honey_badger_split::SplitExt; use log::trace; use rand::RngCore; @@ -389,6 +390,7 @@ impl BeaconState { &self, slot: Slot, registry_change: bool, + spec: &ChainSpec, ) -> Result, u64)>, BeaconStateError> { let epoch = slot.epoch(spec.epoch_length); @@ -668,12 +670,17 @@ impl BeaconState { withdrawal_credentials: Hash256, spec: &ChainSpec, ) -> Result { - if !self.validate_proof_of_possession( - pubkey.clone(), - proof_of_possession, - withdrawal_credentials, - &spec, - ) { + // TODO: update proof of possession to function written above ( + // requires bls::create_proof_of_possession to be updated + // https://github.com/sigp/lighthouse/issues/239 + if !verify_proof_of_possession(&proof_of_possession, &pubkey) + //if !self.validate_proof_of_possession( + // pubkey.clone(), + // proof_of_possession, + // withdrawal_credentials, + // &spec, + // ) + { return Err(()); } diff --git a/eth2/utils/bls/src/lib.rs b/eth2/utils/bls/src/lib.rs index 074929b32..4d0864a90 100644 --- a/eth2/utils/bls/src/lib.rs +++ b/eth2/utils/bls/src/lib.rs @@ -16,7 +16,7 @@ pub use crate::signature::Signature; pub use self::bls_aggregates::AggregatePublicKey; -pub const BLS_AGG_SIG_BYTE_SIZE: usize = 97; +pub const BLS_AGG_SIG_BYTE_SIZE: usize = 96; use hashing::hash; use ssz::ssz_encode; @@ -29,7 +29,14 @@ fn extend_if_needed(hash: &mut Vec) { /// For some signature and public key, ensure that the signature message was the public key and it /// was signed by the secret key that corresponds to that public key. +pub fn verify_proof_of_possession(sig: &Signature, pubkey: &PublicKey) -> bool { + // TODO: replace this function with state.validate_proof_of_possession + // https://github.com/sigp/lighthouse/issues/239 + sig.verify(&ssz_encode(pubkey), 0, &pubkey) +} +// TODO: Update this method +// https://github.com/sigp/lighthouse/issues/239 pub fn create_proof_of_possession(keypair: &Keypair) -> Signature { Signature::new(&ssz_encode(&keypair.pk), 0, &keypair.sk) } @@ -40,6 +47,5 @@ pub fn bls_verify_aggregate( signature: &AggregateSignature, domain: u64, ) -> bool { - // TODO: add domain signature.verify(message, domain, pubkey) } From b211e39331de307fc9f15a908475de1eeebe4d2d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 21 Feb 2019 19:00:55 +1300 Subject: [PATCH 004/132] Add progress on FastBeaconState --- eth2/types/src/fast_beacon_state.rs | 1136 +++++++++++++++++ .../src/fast_beacon_state/committees_cache.rs | 128 ++ eth2/types/src/lib.rs | 2 + 3 files changed, 1266 insertions(+) create mode 100644 eth2/types/src/fast_beacon_state.rs create mode 100644 eth2/types/src/fast_beacon_state/committees_cache.rs diff --git a/eth2/types/src/fast_beacon_state.rs b/eth2/types/src/fast_beacon_state.rs new file mode 100644 index 000000000..7e56df8ae --- /dev/null +++ b/eth2/types/src/fast_beacon_state.rs @@ -0,0 +1,1136 @@ +use crate::test_utils::TestRandom; +use crate::{ + validator::StatusFlags, validator_registry::get_active_validator_indices, AttestationData, + Bitfield, ChainSpec, Crosslink, Deposit, Epoch, Eth1Data, Eth1DataVote, Fork, Hash256, + PendingAttestation, PublicKey, Signature, Slot, Validator, +}; +use bls::verify_proof_of_possession; +use committees_cache::CommitteesCache; +use honey_badger_split::SplitExt; +use log::trace; +use rand::RngCore; +use serde_derive::Serialize; +use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use swap_or_not_shuffle::get_permutated_index; + +mod committees_cache; + +#[derive(Debug, PartialEq)] +pub enum FastBeaconStateError { + EpochOutOfBounds, + UnableToShuffle, + InsufficientRandaoMixes, + InsufficientValidators, + InsufficientBlockRoots, + InsufficientIndexRoots, + InsufficientAttestations, + InsufficientCommittees, +} + +#[derive(Debug, PartialEq)] +pub enum InclusionError { + /// The validator did not participate in an attestation in this period. + NoAttestationsForValidator, + AttestationParticipantsError(AttestationParticipantsError), +} + +#[derive(Debug, PartialEq)] +pub enum AttestationParticipantsError { + /// There is no committee for the given shard in the given epoch. + NoCommitteeForShard, + FastBeaconStateError(FastBeaconStateError), +} + +macro_rules! safe_add_assign { + ($a: expr, $b: expr) => { + $a = $a.saturating_add($b); + }; +} +macro_rules! safe_sub_assign { + ($a: expr, $b: expr) => { + $a = $a.saturating_sub($b); + }; +} + +#[derive(Debug, PartialEq, Clone, Default, Serialize)] +pub struct FastBeaconState { + // Misc + pub slot: Slot, + pub genesis_time: u64, + pub fork: Fork, + + // Validator registry + pub validator_registry: Vec, + pub validator_balances: Vec, + pub validator_registry_update_epoch: Epoch, + + // Randomness and committees + pub latest_randao_mixes: Vec, + pub previous_epoch_start_shard: u64, + pub current_epoch_start_shard: u64, + pub previous_calculation_epoch: Epoch, + pub current_calculation_epoch: Epoch, + pub previous_epoch_seed: Hash256, + pub current_epoch_seed: Hash256, + + // Finality + pub previous_justified_epoch: Epoch, + pub justified_epoch: Epoch, + pub justification_bitfield: u64, + pub finalized_epoch: Epoch, + + // Recent state + pub latest_crosslinks: Vec, + pub latest_block_roots: Vec, + pub latest_index_roots: Vec, + pub latest_penalized_balances: Vec, + pub latest_attestations: Vec, + pub batched_block_roots: Vec, + + // Ethereum 1.0 chain data + pub latest_eth1_data: Eth1Data, + pub eth1_data_votes: Vec, + + // Cache + committees_cache: CommitteesCache, +} + +impl FastBeaconState { + /// Produce the first state of the Beacon Chain. + pub fn genesis( + genesis_time: u64, + initial_validator_deposits: Vec, + latest_eth1_data: Eth1Data, + spec: &ChainSpec, + ) -> Result { + let initial_crosslink = Crosslink { + epoch: spec.genesis_epoch, + shard_block_root: spec.zero_hash, + }; + + let mut genesis_state = FastBeaconState { + /* + * Misc + */ + slot: spec.genesis_slot, + genesis_time, + fork: Fork { + previous_version: spec.genesis_fork_version, + current_version: spec.genesis_fork_version, + epoch: spec.genesis_epoch, + }, + + /* + * Validator registry + */ + validator_registry: vec![], // Set later in the function. + validator_balances: vec![], // Set later in the function. + validator_registry_update_epoch: spec.genesis_epoch, + + /* + * Randomness and committees + */ + latest_randao_mixes: vec![spec.zero_hash; spec.latest_randao_mixes_length as usize], + previous_epoch_start_shard: spec.genesis_start_shard, + current_epoch_start_shard: spec.genesis_start_shard, + previous_calculation_epoch: spec.genesis_epoch, + current_calculation_epoch: spec.genesis_epoch, + previous_epoch_seed: spec.zero_hash, + current_epoch_seed: spec.zero_hash, + + /* + * Finality + */ + previous_justified_epoch: spec.genesis_epoch, + justified_epoch: spec.genesis_epoch, + justification_bitfield: 0, + finalized_epoch: spec.genesis_epoch, + + /* + * Recent state + */ + latest_crosslinks: vec![initial_crosslink; spec.shard_count as usize], + latest_block_roots: vec![spec.zero_hash; spec.latest_block_roots_length as usize], + latest_index_roots: vec![spec.zero_hash; spec.latest_index_roots_length as usize], + latest_penalized_balances: vec![0; spec.latest_penalized_exit_length as usize], + latest_attestations: vec![], + batched_block_roots: vec![], + + /* + * PoW receipt root + */ + latest_eth1_data, + eth1_data_votes: vec![], + + /* + * Caches (not in spec) + */ + committees_cache: CommitteesCache::new(spec.genesis_epoch, spec), + }; + + for deposit in initial_validator_deposits { + let _index = genesis_state.process_deposit( + deposit.deposit_data.deposit_input.pubkey, + deposit.deposit_data.amount, + deposit.deposit_data.deposit_input.proof_of_possession, + deposit.deposit_data.deposit_input.withdrawal_credentials, + spec, + ); + } + + for validator_index in 0..genesis_state.validator_registry.len() { + if genesis_state.get_effective_balance(validator_index, spec) >= spec.max_deposit_amount + { + genesis_state.activate_validator(validator_index, true, spec); + } + } + + let genesis_active_index_root = hash_tree_root(get_active_validator_indices( + &genesis_state.validator_registry, + spec.genesis_epoch, + )); + genesis_state.latest_index_roots = + vec![genesis_active_index_root; spec.latest_index_roots_length]; + genesis_state.current_epoch_seed = genesis_state.generate_seed(spec.genesis_epoch, spec)?; + + Ok(genesis_state) + } + + /// Return the tree hash root for this `FastBeaconState`. + /// + /// Spec v0.2.0 + pub fn canonical_root(&self) -> Hash256 { + Hash256::from(&self.hash_tree_root()[..]) + } + + /// The epoch corresponding to `self.slot`. + /// + /// Spec v0.2.0 + pub fn current_epoch(&self, spec: &ChainSpec) -> Epoch { + self.slot.epoch(spec.epoch_length) + } + + /// The epoch prior to `self.current_epoch()`. + /// + /// Spec v0.2.0 + pub fn previous_epoch(&self, spec: &ChainSpec) -> Epoch { + let current_epoch = self.current_epoch(&spec); + if current_epoch == spec.genesis_epoch { + current_epoch + } else { + current_epoch - 1 + } + } + + /// The epoch following `self.current_epoch()`. + /// + /// Spec v0.2.0 + pub fn next_epoch(&self, spec: &ChainSpec) -> Epoch { + self.current_epoch(spec).saturating_add(1_u64) + } + + /// The first slot of the epoch corresponding to `self.slot`. + /// + /// Spec v0.2.0 + pub fn current_epoch_start_slot(&self, spec: &ChainSpec) -> Slot { + self.current_epoch(spec).start_slot(spec.epoch_length) + } + + /// The first slot of the epoch preceeding the one corresponding to `self.slot`. + /// + /// Spec v0.2.0 + pub fn previous_epoch_start_slot(&self, spec: &ChainSpec) -> Slot { + self.previous_epoch(spec).start_slot(spec.epoch_length) + } + + /// Return the number of committees in one epoch. + /// + /// TODO: this should probably be a method on `ChainSpec`. + /// + /// Spec v0.2.0 + pub fn get_epoch_committee_count( + &self, + active_validator_count: usize, + spec: &ChainSpec, + ) -> u64 { + std::cmp::max( + 1, + std::cmp::min( + spec.shard_count / spec.epoch_length, + active_validator_count as u64 / spec.epoch_length / spec.target_committee_size, + ), + ) * spec.epoch_length + } + + /// Shuffle ``validators`` into crosslink committees seeded by ``seed`` and ``epoch``. + /// Return a list of ``committees_per_epoch`` committees where each + /// committee is itself a list of validator indices. + /// + /// Spec v0.1 + pub fn get_shuffling( + &self, + seed: Hash256, + epoch: Epoch, + spec: &ChainSpec, + ) -> Option>> { + let active_validator_indices = + get_active_validator_indices(&self.validator_registry, epoch); + + if active_validator_indices.is_empty() { + return None; + } + + trace!( + "get_shuffling: active_validator_indices.len() == {}", + active_validator_indices.len() + ); + + let committees_per_epoch = + self.get_epoch_committee_count(active_validator_indices.len(), spec); + + trace!( + "get_shuffling: active_validator_indices.len() == {}, committees_per_epoch: {}", + active_validator_indices.len(), + committees_per_epoch + ); + + let mut shuffled_active_validator_indices = vec![0; active_validator_indices.len()]; + for &i in &active_validator_indices { + let shuffled_i = get_permutated_index( + i, + active_validator_indices.len(), + &seed[..], + spec.shuffle_round_count, + )?; + shuffled_active_validator_indices[i] = active_validator_indices[shuffled_i] + } + + Some( + shuffled_active_validator_indices + .honey_badger_split(committees_per_epoch as usize) + .map(|slice: &[usize]| slice.to_vec()) + .collect(), + ) + } + + /// Return the number of committees in the previous epoch. + /// + /// Spec v0.2.0 + fn get_previous_epoch_committee_count(&self, spec: &ChainSpec) -> u64 { + let previous_active_validators = + get_active_validator_indices(&self.validator_registry, self.previous_calculation_epoch); + self.get_epoch_committee_count(previous_active_validators.len(), spec) + } + + /// Return the number of committees in the current epoch. + /// + /// Spec v0.2.0 + pub fn get_current_epoch_committee_count(&self, spec: &ChainSpec) -> u64 { + let current_active_validators = + get_active_validator_indices(&self.validator_registry, self.current_calculation_epoch); + self.get_epoch_committee_count(current_active_validators.len(), spec) + } + + /// Return the number of committees in the next epoch. + /// + /// Spec v0.2.0 + pub fn get_next_epoch_committee_count(&self, spec: &ChainSpec) -> u64 { + let current_active_validators = + get_active_validator_indices(&self.validator_registry, self.next_epoch(spec)); + self.get_epoch_committee_count(current_active_validators.len(), spec) + } + + pub fn get_active_index_root(&self, epoch: Epoch, spec: &ChainSpec) -> Option { + let current_epoch = self.current_epoch(spec); + + let earliest_index_root = current_epoch - Epoch::from(spec.latest_index_roots_length) + + Epoch::from(spec.entry_exit_delay) + + 1; + let latest_index_root = current_epoch + spec.entry_exit_delay; + + trace!( + "get_active_index_root: epoch: {}, earliest: {}, latest: {}", + epoch, + earliest_index_root, + latest_index_root + ); + + if (epoch >= earliest_index_root) & (epoch <= latest_index_root) { + Some(self.latest_index_roots[epoch.as_usize() % spec.latest_index_roots_length]) + } else { + trace!("get_active_index_root: epoch out of range."); + None + } + } + + /// Generate a seed for the given ``epoch``. + /// + /// Spec v0.2.0 + pub fn generate_seed( + &self, + epoch: Epoch, + spec: &ChainSpec, + ) -> Result { + let mut input = self + .get_randao_mix(epoch, spec) + .ok_or_else(|| FastBeaconStateError::InsufficientRandaoMixes)? + .to_vec(); + + input.append( + &mut self + .get_active_index_root(epoch, spec) + .ok_or_else(|| FastBeaconStateError::InsufficientIndexRoots)? + .to_vec(), + ); + + // TODO: ensure `Hash256::from(u64)` == `int_to_bytes32`. + input.append(&mut Hash256::from(epoch.as_u64()).to_vec()); + + Ok(Hash256::from(&hash(&input[..])[..])) + } + + /// Return the list of ``(committee, shard)`` tuples for the ``slot``. + /// + /// Note: There are two possible shufflings for crosslink committees for a + /// `slot` in the next epoch: with and without a `registry_change` + /// + /// Spec v0.2.0 + pub fn get_crosslink_committees_at_slot( + &self, + slot: Slot, + registry_change: bool, + spec: &ChainSpec, + ) -> Result, u64)>, FastBeaconStateError> { + let epoch = slot.epoch(spec.epoch_length); + let current_epoch = self.current_epoch(spec); + let previous_epoch = self.previous_epoch(spec); + let next_epoch = self.next_epoch(spec); + + let (committees_per_epoch, seed, shuffling_epoch, shuffling_start_shard) = + if epoch == current_epoch { + trace!("get_crosslink_committees_at_slot: current_epoch"); + ( + self.get_current_epoch_committee_count(spec), + self.current_epoch_seed, + self.current_calculation_epoch, + self.current_epoch_start_shard, + ) + } else if epoch == previous_epoch { + trace!("get_crosslink_committees_at_slot: previous_epoch"); + ( + self.get_previous_epoch_committee_count(spec), + self.previous_epoch_seed, + self.previous_calculation_epoch, + self.previous_epoch_start_shard, + ) + } else if epoch == next_epoch { + trace!("get_crosslink_committees_at_slot: next_epoch"); + let current_committees_per_epoch = self.get_current_epoch_committee_count(spec); + let epochs_since_last_registry_update = + current_epoch - self.validator_registry_update_epoch; + let (seed, shuffling_start_shard) = if registry_change { + let next_seed = self.generate_seed(next_epoch, spec)?; + ( + next_seed, + (self.current_epoch_start_shard + current_committees_per_epoch) + % spec.shard_count, + ) + } else if (epochs_since_last_registry_update > 1) + & epochs_since_last_registry_update.is_power_of_two() + { + let next_seed = self.generate_seed(next_epoch, spec)?; + (next_seed, self.current_epoch_start_shard) + } else { + (self.current_epoch_seed, self.current_epoch_start_shard) + }; + ( + self.get_next_epoch_committee_count(spec), + seed, + next_epoch, + shuffling_start_shard, + ) + } else { + return Err(FastBeaconStateError::EpochOutOfBounds); + }; + + let shuffling = self + .get_shuffling(seed, shuffling_epoch, spec) + .ok_or_else(|| FastBeaconStateError::UnableToShuffle)?; + let offset = slot.as_u64() % spec.epoch_length; + let committees_per_slot = committees_per_epoch / spec.epoch_length; + let slot_start_shard = + (shuffling_start_shard + committees_per_slot * offset) % spec.shard_count; + + trace!( + "get_crosslink_committees_at_slot: committees_per_slot: {}, slot_start_shard: {}, seed: {}", + committees_per_slot, + slot_start_shard, + seed + ); + + let mut crosslinks_at_slot = vec![]; + for i in 0..committees_per_slot { + let tuple = ( + shuffling[(committees_per_slot * offset + i) as usize].clone(), + (slot_start_shard + i) % spec.shard_count, + ); + crosslinks_at_slot.push(tuple) + } + Ok(crosslinks_at_slot) + } + + /// Returns the `slot`, `shard` and `committee_index` for which a validator must produce an + /// attestation. + /// + /// Spec v0.2.0 + pub fn attestation_slot_and_shard_for_validator( + &self, + validator_index: usize, + spec: &ChainSpec, + ) -> Result, FastBeaconStateError> { + let mut result = None; + for slot in self.current_epoch(spec).slot_iter(spec.epoch_length) { + for (committee, shard) in self.get_crosslink_committees_at_slot(slot, false, spec)? { + if let Some(committee_index) = committee.iter().position(|&i| i == validator_index) + { + result = Some((slot, shard, committee_index as u64)); + } + } + } + Ok(result) + } + + /// An entry or exit triggered in the ``epoch`` given by the input takes effect at + /// the epoch given by the output. + /// + /// Spec v0.2.0 + pub fn get_entry_exit_effect_epoch(&self, epoch: Epoch, spec: &ChainSpec) -> Epoch { + epoch + 1 + spec.entry_exit_delay + } + + /// Returns the beacon proposer index for the `slot`. + /// + /// If the state does not contain an index for a beacon proposer at the requested `slot`, then `None` is returned. + /// + /// Spec v0.2.0 + pub fn get_beacon_proposer_index( + &self, + slot: Slot, + spec: &ChainSpec, + ) -> Result { + let committees = self.get_crosslink_committees_at_slot(slot, false, spec)?; + trace!( + "get_beacon_proposer_index: slot: {}, committees_count: {}", + slot, + committees.len() + ); + committees + .first() + .ok_or(FastBeaconStateError::InsufficientValidators) + .and_then(|(first_committee, _)| { + let index = (slot.as_usize()) + .checked_rem(first_committee.len()) + .ok_or(FastBeaconStateError::InsufficientValidators)?; + Ok(first_committee[index]) + }) + } + + /// Process the penalties and prepare the validators who are eligible to withdrawal. + /// + /// Spec v0.2.0 + pub fn process_penalties_and_exits(&mut self, spec: &ChainSpec) { + let current_epoch = self.current_epoch(spec); + let active_validator_indices = + get_active_validator_indices(&self.validator_registry, current_epoch); + let total_balance = self.get_total_balance(&active_validator_indices[..], spec); + + for index in 0..self.validator_balances.len() { + let validator = &self.validator_registry[index]; + + if current_epoch + == validator.penalized_epoch + Epoch::from(spec.latest_penalized_exit_length / 2) + { + let epoch_index: usize = + current_epoch.as_usize() % spec.latest_penalized_exit_length; + + let total_at_start = self.latest_penalized_balances + [(epoch_index + 1) % spec.latest_penalized_exit_length]; + let total_at_end = self.latest_penalized_balances[epoch_index]; + let total_penalities = total_at_end.saturating_sub(total_at_start); + let penalty = self.get_effective_balance(index, spec) + * std::cmp::min(total_penalities * 3, total_balance) + / total_balance; + safe_sub_assign!(self.validator_balances[index], penalty); + } + } + + let eligible = |index: usize| { + let validator = &self.validator_registry[index]; + + if validator.penalized_epoch <= current_epoch { + let penalized_withdrawal_epochs = spec.latest_penalized_exit_length / 2; + current_epoch >= validator.penalized_epoch + penalized_withdrawal_epochs as u64 + } else { + current_epoch >= validator.exit_epoch + spec.min_validator_withdrawal_epochs + } + }; + + let mut eligable_indices: Vec = (0..self.validator_registry.len()) + .filter(|i| eligible(*i)) + .collect(); + eligable_indices.sort_by_key(|i| self.validator_registry[*i].exit_epoch); + for (withdrawn_so_far, index) in eligable_indices.iter().enumerate() { + self.prepare_validator_for_withdrawal(*index); + if withdrawn_so_far as u64 >= spec.max_withdrawals_per_epoch { + break; + } + } + } + + /// Return the randao mix at a recent ``epoch``. + /// + /// Returns `None` if the epoch is out-of-bounds of `self.latest_randao_mixes`. + /// + /// Spec v0.2.0 + pub fn get_randao_mix(&self, epoch: Epoch, spec: &ChainSpec) -> Option<&Hash256> { + self.latest_randao_mixes + .get(epoch.as_usize() % spec.latest_randao_mixes_length) + } + + /// Update validator registry, activating/exiting validators if possible. + /// + /// Spec v0.2.0 + pub fn update_validator_registry(&mut self, spec: &ChainSpec) { + let current_epoch = self.current_epoch(spec); + let active_validator_indices = + get_active_validator_indices(&self.validator_registry, current_epoch); + let total_balance = self.get_total_balance(&active_validator_indices[..], spec); + + let max_balance_churn = std::cmp::max( + spec.max_deposit_amount, + total_balance / (2 * spec.max_balance_churn_quotient), + ); + + let mut balance_churn = 0; + for index in 0..self.validator_registry.len() { + let validator = &self.validator_registry[index]; + + if (validator.activation_epoch > self.get_entry_exit_effect_epoch(current_epoch, spec)) + && self.validator_balances[index] >= spec.max_deposit_amount + { + balance_churn += self.get_effective_balance(index, spec); + if balance_churn > max_balance_churn { + break; + } + self.activate_validator(index, false, spec); + } + } + + let mut balance_churn = 0; + for index in 0..self.validator_registry.len() { + let validator = &self.validator_registry[index]; + + if (validator.exit_epoch > self.get_entry_exit_effect_epoch(current_epoch, spec)) + && validator.status_flags == Some(StatusFlags::InitiatedExit) + { + balance_churn += self.get_effective_balance(index, spec); + if balance_churn > max_balance_churn { + break; + } + + self.exit_validator(index, spec); + } + } + + self.validator_registry_update_epoch = current_epoch; + } + /// Process a validator deposit, returning the validator index if the deposit is valid. + /// + /// Spec v0.2.0 + pub fn process_deposit( + &mut self, + pubkey: PublicKey, + amount: u64, + proof_of_possession: Signature, + withdrawal_credentials: Hash256, + spec: &ChainSpec, + ) -> Result { + // TODO: ensure verify proof-of-possession represents the spec accurately. + if !verify_proof_of_possession(&proof_of_possession, &pubkey) { + return Err(()); + } + + if let Some(index) = self + .validator_registry + .iter() + .position(|v| v.pubkey == pubkey) + { + if self.validator_registry[index].withdrawal_credentials == withdrawal_credentials { + safe_add_assign!(self.validator_balances[index], amount); + Ok(index) + } else { + Err(()) + } + } else { + let validator = Validator { + pubkey, + withdrawal_credentials, + activation_epoch: spec.far_future_epoch, + exit_epoch: spec.far_future_epoch, + withdrawal_epoch: spec.far_future_epoch, + penalized_epoch: spec.far_future_epoch, + status_flags: None, + }; + self.validator_registry.push(validator); + self.validator_balances.push(amount); + Ok(self.validator_registry.len() - 1) + } + } + + /// Activate the validator of the given ``index``. + /// + /// Spec v0.2.0 + pub fn activate_validator( + &mut self, + validator_index: usize, + is_genesis: bool, + spec: &ChainSpec, + ) { + let current_epoch = self.current_epoch(spec); + + self.validator_registry[validator_index].activation_epoch = if is_genesis { + spec.genesis_epoch + } else { + self.get_entry_exit_effect_epoch(current_epoch, spec) + } + } + + /// Initiate an exit for the validator of the given `index`. + /// + /// Spec v0.2.0 + pub fn initiate_validator_exit(&mut self, validator_index: usize) { + // TODO: the spec does an `|=` here, ensure this isn't buggy. + self.validator_registry[validator_index].status_flags = Some(StatusFlags::InitiatedExit); + } + + /// Exit the validator of the given `index`. + /// + /// Spec v0.2.0 + fn exit_validator(&mut self, validator_index: usize, spec: &ChainSpec) { + let current_epoch = self.current_epoch(spec); + + if self.validator_registry[validator_index].exit_epoch + <= self.get_entry_exit_effect_epoch(current_epoch, spec) + { + return; + } + + self.validator_registry[validator_index].exit_epoch = + self.get_entry_exit_effect_epoch(current_epoch, spec); + } + + /// Penalize the validator of the given ``index``. + /// + /// Exits the validator and assigns its effective balance to the block producer for this + /// state. + /// + /// Spec v0.2.0 + pub fn penalize_validator( + &mut self, + validator_index: usize, + spec: &ChainSpec, + ) -> Result<(), FastBeaconStateError> { + self.exit_validator(validator_index, spec); + let current_epoch = self.current_epoch(spec); + + self.latest_penalized_balances + [current_epoch.as_usize() % spec.latest_penalized_exit_length] += + self.get_effective_balance(validator_index, spec); + + let whistleblower_index = self.get_beacon_proposer_index(self.slot, spec)?; + let whistleblower_reward = self.get_effective_balance(validator_index, spec); + safe_add_assign!( + self.validator_balances[whistleblower_index as usize], + whistleblower_reward + ); + safe_sub_assign!( + self.validator_balances[validator_index], + whistleblower_reward + ); + self.validator_registry[validator_index].penalized_epoch = current_epoch; + Ok(()) + } + + /// Initiate an exit for the validator of the given `index`. + /// + /// Spec v0.2.0 + pub fn prepare_validator_for_withdrawal(&mut self, validator_index: usize) { + //TODO: we're not ANDing here, we're setting. Potentially wrong. + self.validator_registry[validator_index].status_flags = Some(StatusFlags::Withdrawable); + } + + /// Iterate through the validator registry and eject active validators with balance below + /// ``EJECTION_BALANCE``. + /// + /// Spec v0.2.0 + pub fn process_ejections(&mut self, spec: &ChainSpec) { + for validator_index in + get_active_validator_indices(&self.validator_registry, self.current_epoch(spec)) + { + if self.validator_balances[validator_index] < spec.ejection_balance { + self.exit_validator(validator_index, spec) + } + } + } + + /// Returns the penality that should be applied to some validator for inactivity. + /// + /// Note: this is defined "inline" in the spec, not as a helper function. + /// + /// Spec v0.2.0 + pub fn inactivity_penalty( + &self, + validator_index: usize, + epochs_since_finality: Epoch, + base_reward_quotient: u64, + spec: &ChainSpec, + ) -> u64 { + let effective_balance = self.get_effective_balance(validator_index, spec); + self.base_reward(validator_index, base_reward_quotient, spec) + + effective_balance * epochs_since_finality.as_u64() + / spec.inactivity_penalty_quotient + / 2 + } + + /// Returns the distance between the first included attestation for some validator and this + /// slot. + /// + /// Note: In the spec this is defined "inline", not as a helper function. + /// + /// Spec v0.2.0 + pub fn inclusion_distance( + &self, + attestations: &[&PendingAttestation], + validator_index: usize, + spec: &ChainSpec, + ) -> Result { + let attestation = + self.earliest_included_attestation(attestations, validator_index, spec)?; + Ok((attestation.inclusion_slot - attestation.data.slot).as_u64()) + } + + /// Returns the slot of the earliest included attestation for some validator. + /// + /// Note: In the spec this is defined "inline", not as a helper function. + /// + /// Spec v0.2.0 + pub fn inclusion_slot( + &self, + attestations: &[&PendingAttestation], + validator_index: usize, + spec: &ChainSpec, + ) -> Result { + let attestation = + self.earliest_included_attestation(attestations, validator_index, spec)?; + Ok(attestation.inclusion_slot) + } + + /// Finds the earliest included attestation for some validator. + /// + /// Note: In the spec this is defined "inline", not as a helper function. + /// + /// Spec v0.2.0 + fn earliest_included_attestation( + &self, + attestations: &[&PendingAttestation], + validator_index: usize, + spec: &ChainSpec, + ) -> Result { + let mut included_attestations = vec![]; + + for (i, a) in attestations.iter().enumerate() { + let participants = + self.get_attestation_participants(&a.data, &a.aggregation_bitfield, spec)?; + if participants.iter().any(|i| *i == validator_index) { + included_attestations.push(i); + } + } + + let earliest_attestation_index = included_attestations + .iter() + .min_by_key(|i| attestations[**i].inclusion_slot) + .ok_or_else(|| InclusionError::NoAttestationsForValidator)?; + Ok(attestations[*earliest_attestation_index].clone()) + } + + /// Returns the base reward for some validator. + /// + /// Note: In the spec this is defined "inline", not as a helper function. + /// + /// Spec v0.2.0 + pub fn base_reward( + &self, + validator_index: usize, + base_reward_quotient: u64, + spec: &ChainSpec, + ) -> u64 { + self.get_effective_balance(validator_index, spec) / base_reward_quotient / 5 + } + + /// Return the combined effective balance of an array of validators. + /// + /// Spec v0.2.0 + pub fn get_total_balance(&self, validator_indices: &[usize], spec: &ChainSpec) -> u64 { + validator_indices + .iter() + .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)) + } + + /// Return the effective balance (also known as "balance at stake") for a validator with the given ``index``. + /// + /// Spec v0.2.0 + pub fn get_effective_balance(&self, validator_index: usize, spec: &ChainSpec) -> u64 { + std::cmp::min( + self.validator_balances[validator_index], + spec.max_deposit_amount, + ) + } + + /// Return the block root at a recent `slot`. + /// + /// Spec v0.2.0 + pub fn get_block_root(&self, slot: Slot, spec: &ChainSpec) -> Option<&Hash256> { + self.latest_block_roots + .get(slot.as_usize() % spec.latest_block_roots_length) + } + + pub fn get_attestation_participants_union( + &self, + attestations: &[&PendingAttestation], + spec: &ChainSpec, + ) -> Result, AttestationParticipantsError> { + let mut all_participants = attestations.iter().try_fold::<_, _, Result< + Vec, + AttestationParticipantsError, + >>(vec![], |mut acc, a| { + acc.append(&mut self.get_attestation_participants( + &a.data, + &a.aggregation_bitfield, + spec, + )?); + Ok(acc) + })?; + all_participants.sort_unstable(); + all_participants.dedup(); + Ok(all_participants) + } + + /// Return the participant indices at for the ``attestation_data`` and ``bitfield``. + /// + /// In effect, this converts the "committee indices" on the bitfield into "validator indices" + /// for self.validator_registy. + /// + /// Spec v0.2.0 + pub fn get_attestation_participants( + &self, + attestation_data: &AttestationData, + bitfield: &Bitfield, + spec: &ChainSpec, + ) -> Result, AttestationParticipantsError> { + let crosslink_committees = + self.get_crosslink_committees_at_slot(attestation_data.slot, false, spec)?; + + let committee_index: usize = crosslink_committees + .iter() + .position(|(_committee, shard)| *shard == attestation_data.shard) + .ok_or_else(|| AttestationParticipantsError::NoCommitteeForShard)?; + let (crosslink_committee, _shard) = &crosslink_committees[committee_index]; + + /* + * TODO: verify bitfield length is valid. + */ + + let mut participants = vec![]; + for (i, validator_index) in crosslink_committee.iter().enumerate() { + if bitfield.get(i).unwrap() { + participants.push(*validator_index); + } + } + Ok(participants) + } +} + +fn hash_tree_root(input: Vec) -> Hash256 { + Hash256::from(&input.hash_tree_root()[..]) +} + +impl From for AttestationParticipantsError { + fn from(e: FastBeaconStateError) -> AttestationParticipantsError { + AttestationParticipantsError::FastBeaconStateError(e) + } +} + +impl From for InclusionError { + fn from(e: AttestationParticipantsError) -> InclusionError { + InclusionError::AttestationParticipantsError(e) + } +} + +impl Encodable for FastBeaconState { + fn ssz_append(&self, s: &mut SszStream) { + s.append(&self.slot); + s.append(&self.genesis_time); + s.append(&self.fork); + s.append(&self.validator_registry); + s.append(&self.validator_balances); + s.append(&self.validator_registry_update_epoch); + s.append(&self.latest_randao_mixes); + s.append(&self.previous_epoch_start_shard); + s.append(&self.current_epoch_start_shard); + s.append(&self.previous_calculation_epoch); + s.append(&self.current_calculation_epoch); + s.append(&self.previous_epoch_seed); + s.append(&self.current_epoch_seed); + s.append(&self.previous_justified_epoch); + s.append(&self.justified_epoch); + s.append(&self.justification_bitfield); + s.append(&self.finalized_epoch); + s.append(&self.latest_crosslinks); + s.append(&self.latest_block_roots); + s.append(&self.latest_index_roots); + s.append(&self.latest_penalized_balances); + s.append(&self.latest_attestations); + s.append(&self.batched_block_roots); + s.append(&self.latest_eth1_data); + s.append(&self.eth1_data_votes); + } +} + +impl Decodable for FastBeaconState { + fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { + let (slot, i) = <_>::ssz_decode(bytes, i)?; + let (genesis_time, i) = <_>::ssz_decode(bytes, i)?; + let (fork, i) = <_>::ssz_decode(bytes, i)?; + let (validator_registry, i) = <_>::ssz_decode(bytes, i)?; + let (validator_balances, i) = <_>::ssz_decode(bytes, i)?; + let (validator_registry_update_epoch, i) = <_>::ssz_decode(bytes, i)?; + let (latest_randao_mixes, i) = <_>::ssz_decode(bytes, i)?; + let (previous_epoch_start_shard, i) = <_>::ssz_decode(bytes, i)?; + let (current_epoch_start_shard, i) = <_>::ssz_decode(bytes, i)?; + let (previous_calculation_epoch, i) = <_>::ssz_decode(bytes, i)?; + let (current_calculation_epoch, i) = <_>::ssz_decode(bytes, i)?; + let (previous_epoch_seed, i) = <_>::ssz_decode(bytes, i)?; + let (current_epoch_seed, i) = <_>::ssz_decode(bytes, i)?; + let (previous_justified_epoch, i) = <_>::ssz_decode(bytes, i)?; + let (justified_epoch, i) = <_>::ssz_decode(bytes, i)?; + let (justification_bitfield, i) = <_>::ssz_decode(bytes, i)?; + let (finalized_epoch, i) = <_>::ssz_decode(bytes, i)?; + let (latest_crosslinks, i) = <_>::ssz_decode(bytes, i)?; + let (latest_block_roots, i) = <_>::ssz_decode(bytes, i)?; + let (latest_index_roots, i) = <_>::ssz_decode(bytes, i)?; + let (latest_penalized_balances, i) = <_>::ssz_decode(bytes, i)?; + let (latest_attestations, i) = <_>::ssz_decode(bytes, i)?; + let (batched_block_roots, i) = <_>::ssz_decode(bytes, i)?; + let (latest_eth1_data, i) = <_>::ssz_decode(bytes, i)?; + let (eth1_data_votes, i) = <_>::ssz_decode(bytes, i)?; + + Ok(( + Self { + slot, + genesis_time, + fork, + validator_registry, + validator_balances, + validator_registry_update_epoch, + latest_randao_mixes, + previous_epoch_start_shard, + current_epoch_start_shard, + previous_calculation_epoch, + current_calculation_epoch, + previous_epoch_seed, + current_epoch_seed, + previous_justified_epoch, + justified_epoch, + justification_bitfield, + finalized_epoch, + latest_crosslinks, + latest_block_roots, + latest_index_roots, + latest_penalized_balances, + latest_attestations, + batched_block_roots, + latest_eth1_data, + eth1_data_votes, + }, + i, + )) + } +} + +impl TreeHash for FastBeaconState { + fn hash_tree_root_internal(&self) -> Vec { + let mut result: Vec = vec![]; + result.append(&mut self.slot.hash_tree_root_internal()); + result.append(&mut self.genesis_time.hash_tree_root_internal()); + result.append(&mut self.fork.hash_tree_root_internal()); + result.append(&mut self.validator_registry.hash_tree_root_internal()); + result.append(&mut self.validator_balances.hash_tree_root_internal()); + result.append( + &mut self + .validator_registry_update_epoch + .hash_tree_root_internal(), + ); + result.append(&mut self.latest_randao_mixes.hash_tree_root_internal()); + result.append(&mut self.previous_epoch_start_shard.hash_tree_root_internal()); + result.append(&mut self.current_epoch_start_shard.hash_tree_root_internal()); + result.append(&mut self.previous_calculation_epoch.hash_tree_root_internal()); + result.append(&mut self.current_calculation_epoch.hash_tree_root_internal()); + result.append(&mut self.previous_epoch_seed.hash_tree_root_internal()); + result.append(&mut self.current_epoch_seed.hash_tree_root_internal()); + result.append(&mut self.previous_justified_epoch.hash_tree_root_internal()); + result.append(&mut self.justified_epoch.hash_tree_root_internal()); + result.append(&mut self.justification_bitfield.hash_tree_root_internal()); + result.append(&mut self.finalized_epoch.hash_tree_root_internal()); + result.append(&mut self.latest_crosslinks.hash_tree_root_internal()); + result.append(&mut self.latest_block_roots.hash_tree_root_internal()); + result.append(&mut self.latest_index_roots.hash_tree_root_internal()); + result.append(&mut self.latest_penalized_balances.hash_tree_root_internal()); + result.append(&mut self.latest_attestations.hash_tree_root_internal()); + result.append(&mut self.batched_block_roots.hash_tree_root_internal()); + result.append(&mut self.latest_eth1_data.hash_tree_root_internal()); + result.append(&mut self.eth1_data_votes.hash_tree_root_internal()); + hash(&result) + } +} + +impl TestRandom for FastBeaconState { + fn random_for_test(rng: &mut T) -> Self { + Self { + slot: <_>::random_for_test(rng), + genesis_time: <_>::random_for_test(rng), + fork: <_>::random_for_test(rng), + validator_registry: <_>::random_for_test(rng), + validator_balances: <_>::random_for_test(rng), + validator_registry_update_epoch: <_>::random_for_test(rng), + latest_randao_mixes: <_>::random_for_test(rng), + previous_epoch_start_shard: <_>::random_for_test(rng), + current_epoch_start_shard: <_>::random_for_test(rng), + previous_calculation_epoch: <_>::random_for_test(rng), + current_calculation_epoch: <_>::random_for_test(rng), + previous_epoch_seed: <_>::random_for_test(rng), + current_epoch_seed: <_>::random_for_test(rng), + previous_justified_epoch: <_>::random_for_test(rng), + justified_epoch: <_>::random_for_test(rng), + justification_bitfield: <_>::random_for_test(rng), + finalized_epoch: <_>::random_for_test(rng), + latest_crosslinks: <_>::random_for_test(rng), + latest_block_roots: <_>::random_for_test(rng), + latest_index_roots: <_>::random_for_test(rng), + latest_penalized_balances: <_>::random_for_test(rng), + latest_attestations: <_>::random_for_test(rng), + batched_block_roots: <_>::random_for_test(rng), + latest_eth1_data: <_>::random_for_test(rng), + eth1_data_votes: <_>::random_for_test(rng), + } + } +} diff --git a/eth2/types/src/fast_beacon_state/committees_cache.rs b/eth2/types/src/fast_beacon_state/committees_cache.rs new file mode 100644 index 000000000..42dd101ba --- /dev/null +++ b/eth2/types/src/fast_beacon_state/committees_cache.rs @@ -0,0 +1,128 @@ +use crate::{ + validator_registry::get_active_validator_indices, BeaconState, ChainSpec, Epoch, Hash256, +}; +use honey_badger_split::SplitExt; +use serde_derive::Serialize; +use swap_or_not_shuffle::get_permutated_index; + +pub const CACHED_EPOCHS: usize = 3; + +#[derive(Debug, PartialEq, Clone, Default, Serialize)] +pub struct CommitteesCache { + cache_index_offset: usize, + active_validator_indices_cache: Vec>>, + shuffling_cache: Vec>>>>, + previous_epoch: Epoch, + current_epoch: Epoch, + next_epoch: Epoch, +} + +impl CommitteesCache { + pub fn new(current_epoch: Epoch, spec: &ChainSpec) -> Self { + let previous_epoch = if current_epoch == spec.genesis_epoch { + current_epoch + } else { + current_epoch - 1 + }; + let next_epoch = current_epoch + 1; + + Self { + cache_index_offset: 0, + active_validator_indices_cache: vec![None; CACHED_EPOCHS], + shuffling_cache: vec![None; CACHED_EPOCHS], + previous_epoch, + current_epoch, + next_epoch, + } + } + + pub fn advance_epoch(&mut self) { + let previous_cache_index = self.cache_index(self.previous_epoch); + + self.active_validator_indices_cache[previous_cache_index] = None; + self.shuffling_cache[previous_cache_index] = None; + + self.cache_index_offset += 1; + self.cache_index_offset %= CACHED_EPOCHS; + } + + pub fn cache_index(&self, epoch: Epoch) -> usize { + let base_index = match epoch { + e if e == self.previous_epoch => 0, + e if e == self.current_epoch => 1, + e if e == self.next_epoch => 2, + _ => panic!("Bad cache index."), + }; + + (base_index + self.cache_index_offset) % CACHED_EPOCHS + } + + pub fn get_active_validator_indices( + &mut self, + state: &BeaconState, + epoch: Epoch, + ) -> &Vec { + let i = self.cache_index(epoch); + + if self.active_validator_indices_cache[i] == None { + self.active_validator_indices_cache[i] = Some(get_active_validator_indices( + &state.validator_registry, + epoch, + )); + } + + self.active_validator_indices_cache[i] + .as_ref() + .expect("Cache cannot be None") + } + + pub fn get_shuffling( + &mut self, + state: &BeaconState, + seed: Hash256, + epoch: Epoch, + spec: &ChainSpec, + ) -> Option<&Vec>> { + let cache_index = self.cache_index(epoch); + + if self.shuffling_cache[cache_index] == None { + let active_validator_indices = self.get_active_validator_indices(&state, epoch); + + if active_validator_indices.is_empty() { + return None; + } + + let committees_per_epoch = + state.get_epoch_committee_count(active_validator_indices.len(), spec); + + let mut shuffled_active_validator_indices = vec![0; active_validator_indices.len()]; + for &i in active_validator_indices { + let shuffled_i = get_permutated_index( + i, + active_validator_indices.len(), + &seed[..], + spec.shuffle_round_count, + )?; + shuffled_active_validator_indices[i] = active_validator_indices[shuffled_i] + } + + let committees: Vec> = shuffled_active_validator_indices + .honey_badger_split(committees_per_epoch as usize) + .map(|slice: &[usize]| slice.to_vec()) + .collect(); + + self.shuffling_cache[cache_index] = Some(Some(committees)); + } + + match self.shuffling_cache[cache_index] { + Some(_) => Some( + self.shuffling_cache[cache_index] + .as_ref() + .expect("Cache cannot be None") + .as_ref() + .expect("Shuffling cannot be None."), + ), + None => None, + } + } +} diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index f2c128440..820c26ad6 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -16,6 +16,7 @@ pub mod deposit_input; pub mod eth1_data; pub mod eth1_data_vote; pub mod exit; +pub mod fast_beacon_state; pub mod fork; pub mod free_attestation; pub mod pending_attestation; @@ -52,6 +53,7 @@ pub use crate::deposit_input::DepositInput; pub use crate::eth1_data::Eth1Data; pub use crate::eth1_data_vote::Eth1DataVote; pub use crate::exit::Exit; +pub use crate::fast_beacon_state::FastBeaconState; pub use crate::fork::Fork; pub use crate::free_attestation::FreeAttestation; pub use crate::pending_attestation::PendingAttestation; From cdc03f1749e5f27a1516269232bd15b0bca440f8 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 22 Feb 2019 17:55:36 +1300 Subject: [PATCH 005/132] Remove FastBeaconState --- eth2/types/src/fast_beacon_state.rs | 1136 ----------------- .../src/fast_beacon_state/committees_cache.rs | 128 -- eth2/types/src/lib.rs | 2 - 3 files changed, 1266 deletions(-) delete mode 100644 eth2/types/src/fast_beacon_state.rs delete mode 100644 eth2/types/src/fast_beacon_state/committees_cache.rs diff --git a/eth2/types/src/fast_beacon_state.rs b/eth2/types/src/fast_beacon_state.rs deleted file mode 100644 index 7e56df8ae..000000000 --- a/eth2/types/src/fast_beacon_state.rs +++ /dev/null @@ -1,1136 +0,0 @@ -use crate::test_utils::TestRandom; -use crate::{ - validator::StatusFlags, validator_registry::get_active_validator_indices, AttestationData, - Bitfield, ChainSpec, Crosslink, Deposit, Epoch, Eth1Data, Eth1DataVote, Fork, Hash256, - PendingAttestation, PublicKey, Signature, Slot, Validator, -}; -use bls::verify_proof_of_possession; -use committees_cache::CommitteesCache; -use honey_badger_split::SplitExt; -use log::trace; -use rand::RngCore; -use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; -use swap_or_not_shuffle::get_permutated_index; - -mod committees_cache; - -#[derive(Debug, PartialEq)] -pub enum FastBeaconStateError { - EpochOutOfBounds, - UnableToShuffle, - InsufficientRandaoMixes, - InsufficientValidators, - InsufficientBlockRoots, - InsufficientIndexRoots, - InsufficientAttestations, - InsufficientCommittees, -} - -#[derive(Debug, PartialEq)] -pub enum InclusionError { - /// The validator did not participate in an attestation in this period. - NoAttestationsForValidator, - AttestationParticipantsError(AttestationParticipantsError), -} - -#[derive(Debug, PartialEq)] -pub enum AttestationParticipantsError { - /// There is no committee for the given shard in the given epoch. - NoCommitteeForShard, - FastBeaconStateError(FastBeaconStateError), -} - -macro_rules! safe_add_assign { - ($a: expr, $b: expr) => { - $a = $a.saturating_add($b); - }; -} -macro_rules! safe_sub_assign { - ($a: expr, $b: expr) => { - $a = $a.saturating_sub($b); - }; -} - -#[derive(Debug, PartialEq, Clone, Default, Serialize)] -pub struct FastBeaconState { - // Misc - pub slot: Slot, - pub genesis_time: u64, - pub fork: Fork, - - // Validator registry - pub validator_registry: Vec, - pub validator_balances: Vec, - pub validator_registry_update_epoch: Epoch, - - // Randomness and committees - pub latest_randao_mixes: Vec, - pub previous_epoch_start_shard: u64, - pub current_epoch_start_shard: u64, - pub previous_calculation_epoch: Epoch, - pub current_calculation_epoch: Epoch, - pub previous_epoch_seed: Hash256, - pub current_epoch_seed: Hash256, - - // Finality - pub previous_justified_epoch: Epoch, - pub justified_epoch: Epoch, - pub justification_bitfield: u64, - pub finalized_epoch: Epoch, - - // Recent state - pub latest_crosslinks: Vec, - pub latest_block_roots: Vec, - pub latest_index_roots: Vec, - pub latest_penalized_balances: Vec, - pub latest_attestations: Vec, - pub batched_block_roots: Vec, - - // Ethereum 1.0 chain data - pub latest_eth1_data: Eth1Data, - pub eth1_data_votes: Vec, - - // Cache - committees_cache: CommitteesCache, -} - -impl FastBeaconState { - /// Produce the first state of the Beacon Chain. - pub fn genesis( - genesis_time: u64, - initial_validator_deposits: Vec, - latest_eth1_data: Eth1Data, - spec: &ChainSpec, - ) -> Result { - let initial_crosslink = Crosslink { - epoch: spec.genesis_epoch, - shard_block_root: spec.zero_hash, - }; - - let mut genesis_state = FastBeaconState { - /* - * Misc - */ - slot: spec.genesis_slot, - genesis_time, - fork: Fork { - previous_version: spec.genesis_fork_version, - current_version: spec.genesis_fork_version, - epoch: spec.genesis_epoch, - }, - - /* - * Validator registry - */ - validator_registry: vec![], // Set later in the function. - validator_balances: vec![], // Set later in the function. - validator_registry_update_epoch: spec.genesis_epoch, - - /* - * Randomness and committees - */ - latest_randao_mixes: vec![spec.zero_hash; spec.latest_randao_mixes_length as usize], - previous_epoch_start_shard: spec.genesis_start_shard, - current_epoch_start_shard: spec.genesis_start_shard, - previous_calculation_epoch: spec.genesis_epoch, - current_calculation_epoch: spec.genesis_epoch, - previous_epoch_seed: spec.zero_hash, - current_epoch_seed: spec.zero_hash, - - /* - * Finality - */ - previous_justified_epoch: spec.genesis_epoch, - justified_epoch: spec.genesis_epoch, - justification_bitfield: 0, - finalized_epoch: spec.genesis_epoch, - - /* - * Recent state - */ - latest_crosslinks: vec![initial_crosslink; spec.shard_count as usize], - latest_block_roots: vec![spec.zero_hash; spec.latest_block_roots_length as usize], - latest_index_roots: vec![spec.zero_hash; spec.latest_index_roots_length as usize], - latest_penalized_balances: vec![0; spec.latest_penalized_exit_length as usize], - latest_attestations: vec![], - batched_block_roots: vec![], - - /* - * PoW receipt root - */ - latest_eth1_data, - eth1_data_votes: vec![], - - /* - * Caches (not in spec) - */ - committees_cache: CommitteesCache::new(spec.genesis_epoch, spec), - }; - - for deposit in initial_validator_deposits { - let _index = genesis_state.process_deposit( - deposit.deposit_data.deposit_input.pubkey, - deposit.deposit_data.amount, - deposit.deposit_data.deposit_input.proof_of_possession, - deposit.deposit_data.deposit_input.withdrawal_credentials, - spec, - ); - } - - for validator_index in 0..genesis_state.validator_registry.len() { - if genesis_state.get_effective_balance(validator_index, spec) >= spec.max_deposit_amount - { - genesis_state.activate_validator(validator_index, true, spec); - } - } - - let genesis_active_index_root = hash_tree_root(get_active_validator_indices( - &genesis_state.validator_registry, - spec.genesis_epoch, - )); - genesis_state.latest_index_roots = - vec![genesis_active_index_root; spec.latest_index_roots_length]; - genesis_state.current_epoch_seed = genesis_state.generate_seed(spec.genesis_epoch, spec)?; - - Ok(genesis_state) - } - - /// Return the tree hash root for this `FastBeaconState`. - /// - /// Spec v0.2.0 - pub fn canonical_root(&self) -> Hash256 { - Hash256::from(&self.hash_tree_root()[..]) - } - - /// The epoch corresponding to `self.slot`. - /// - /// Spec v0.2.0 - pub fn current_epoch(&self, spec: &ChainSpec) -> Epoch { - self.slot.epoch(spec.epoch_length) - } - - /// The epoch prior to `self.current_epoch()`. - /// - /// Spec v0.2.0 - pub fn previous_epoch(&self, spec: &ChainSpec) -> Epoch { - let current_epoch = self.current_epoch(&spec); - if current_epoch == spec.genesis_epoch { - current_epoch - } else { - current_epoch - 1 - } - } - - /// The epoch following `self.current_epoch()`. - /// - /// Spec v0.2.0 - pub fn next_epoch(&self, spec: &ChainSpec) -> Epoch { - self.current_epoch(spec).saturating_add(1_u64) - } - - /// The first slot of the epoch corresponding to `self.slot`. - /// - /// Spec v0.2.0 - pub fn current_epoch_start_slot(&self, spec: &ChainSpec) -> Slot { - self.current_epoch(spec).start_slot(spec.epoch_length) - } - - /// The first slot of the epoch preceeding the one corresponding to `self.slot`. - /// - /// Spec v0.2.0 - pub fn previous_epoch_start_slot(&self, spec: &ChainSpec) -> Slot { - self.previous_epoch(spec).start_slot(spec.epoch_length) - } - - /// Return the number of committees in one epoch. - /// - /// TODO: this should probably be a method on `ChainSpec`. - /// - /// Spec v0.2.0 - pub fn get_epoch_committee_count( - &self, - active_validator_count: usize, - spec: &ChainSpec, - ) -> u64 { - std::cmp::max( - 1, - std::cmp::min( - spec.shard_count / spec.epoch_length, - active_validator_count as u64 / spec.epoch_length / spec.target_committee_size, - ), - ) * spec.epoch_length - } - - /// Shuffle ``validators`` into crosslink committees seeded by ``seed`` and ``epoch``. - /// Return a list of ``committees_per_epoch`` committees where each - /// committee is itself a list of validator indices. - /// - /// Spec v0.1 - pub fn get_shuffling( - &self, - seed: Hash256, - epoch: Epoch, - spec: &ChainSpec, - ) -> Option>> { - let active_validator_indices = - get_active_validator_indices(&self.validator_registry, epoch); - - if active_validator_indices.is_empty() { - return None; - } - - trace!( - "get_shuffling: active_validator_indices.len() == {}", - active_validator_indices.len() - ); - - let committees_per_epoch = - self.get_epoch_committee_count(active_validator_indices.len(), spec); - - trace!( - "get_shuffling: active_validator_indices.len() == {}, committees_per_epoch: {}", - active_validator_indices.len(), - committees_per_epoch - ); - - let mut shuffled_active_validator_indices = vec![0; active_validator_indices.len()]; - for &i in &active_validator_indices { - let shuffled_i = get_permutated_index( - i, - active_validator_indices.len(), - &seed[..], - spec.shuffle_round_count, - )?; - shuffled_active_validator_indices[i] = active_validator_indices[shuffled_i] - } - - Some( - shuffled_active_validator_indices - .honey_badger_split(committees_per_epoch as usize) - .map(|slice: &[usize]| slice.to_vec()) - .collect(), - ) - } - - /// Return the number of committees in the previous epoch. - /// - /// Spec v0.2.0 - fn get_previous_epoch_committee_count(&self, spec: &ChainSpec) -> u64 { - let previous_active_validators = - get_active_validator_indices(&self.validator_registry, self.previous_calculation_epoch); - self.get_epoch_committee_count(previous_active_validators.len(), spec) - } - - /// Return the number of committees in the current epoch. - /// - /// Spec v0.2.0 - pub fn get_current_epoch_committee_count(&self, spec: &ChainSpec) -> u64 { - let current_active_validators = - get_active_validator_indices(&self.validator_registry, self.current_calculation_epoch); - self.get_epoch_committee_count(current_active_validators.len(), spec) - } - - /// Return the number of committees in the next epoch. - /// - /// Spec v0.2.0 - pub fn get_next_epoch_committee_count(&self, spec: &ChainSpec) -> u64 { - let current_active_validators = - get_active_validator_indices(&self.validator_registry, self.next_epoch(spec)); - self.get_epoch_committee_count(current_active_validators.len(), spec) - } - - pub fn get_active_index_root(&self, epoch: Epoch, spec: &ChainSpec) -> Option { - let current_epoch = self.current_epoch(spec); - - let earliest_index_root = current_epoch - Epoch::from(spec.latest_index_roots_length) - + Epoch::from(spec.entry_exit_delay) - + 1; - let latest_index_root = current_epoch + spec.entry_exit_delay; - - trace!( - "get_active_index_root: epoch: {}, earliest: {}, latest: {}", - epoch, - earliest_index_root, - latest_index_root - ); - - if (epoch >= earliest_index_root) & (epoch <= latest_index_root) { - Some(self.latest_index_roots[epoch.as_usize() % spec.latest_index_roots_length]) - } else { - trace!("get_active_index_root: epoch out of range."); - None - } - } - - /// Generate a seed for the given ``epoch``. - /// - /// Spec v0.2.0 - pub fn generate_seed( - &self, - epoch: Epoch, - spec: &ChainSpec, - ) -> Result { - let mut input = self - .get_randao_mix(epoch, spec) - .ok_or_else(|| FastBeaconStateError::InsufficientRandaoMixes)? - .to_vec(); - - input.append( - &mut self - .get_active_index_root(epoch, spec) - .ok_or_else(|| FastBeaconStateError::InsufficientIndexRoots)? - .to_vec(), - ); - - // TODO: ensure `Hash256::from(u64)` == `int_to_bytes32`. - input.append(&mut Hash256::from(epoch.as_u64()).to_vec()); - - Ok(Hash256::from(&hash(&input[..])[..])) - } - - /// Return the list of ``(committee, shard)`` tuples for the ``slot``. - /// - /// Note: There are two possible shufflings for crosslink committees for a - /// `slot` in the next epoch: with and without a `registry_change` - /// - /// Spec v0.2.0 - pub fn get_crosslink_committees_at_slot( - &self, - slot: Slot, - registry_change: bool, - spec: &ChainSpec, - ) -> Result, u64)>, FastBeaconStateError> { - let epoch = slot.epoch(spec.epoch_length); - let current_epoch = self.current_epoch(spec); - let previous_epoch = self.previous_epoch(spec); - let next_epoch = self.next_epoch(spec); - - let (committees_per_epoch, seed, shuffling_epoch, shuffling_start_shard) = - if epoch == current_epoch { - trace!("get_crosslink_committees_at_slot: current_epoch"); - ( - self.get_current_epoch_committee_count(spec), - self.current_epoch_seed, - self.current_calculation_epoch, - self.current_epoch_start_shard, - ) - } else if epoch == previous_epoch { - trace!("get_crosslink_committees_at_slot: previous_epoch"); - ( - self.get_previous_epoch_committee_count(spec), - self.previous_epoch_seed, - self.previous_calculation_epoch, - self.previous_epoch_start_shard, - ) - } else if epoch == next_epoch { - trace!("get_crosslink_committees_at_slot: next_epoch"); - let current_committees_per_epoch = self.get_current_epoch_committee_count(spec); - let epochs_since_last_registry_update = - current_epoch - self.validator_registry_update_epoch; - let (seed, shuffling_start_shard) = if registry_change { - let next_seed = self.generate_seed(next_epoch, spec)?; - ( - next_seed, - (self.current_epoch_start_shard + current_committees_per_epoch) - % spec.shard_count, - ) - } else if (epochs_since_last_registry_update > 1) - & epochs_since_last_registry_update.is_power_of_two() - { - let next_seed = self.generate_seed(next_epoch, spec)?; - (next_seed, self.current_epoch_start_shard) - } else { - (self.current_epoch_seed, self.current_epoch_start_shard) - }; - ( - self.get_next_epoch_committee_count(spec), - seed, - next_epoch, - shuffling_start_shard, - ) - } else { - return Err(FastBeaconStateError::EpochOutOfBounds); - }; - - let shuffling = self - .get_shuffling(seed, shuffling_epoch, spec) - .ok_or_else(|| FastBeaconStateError::UnableToShuffle)?; - let offset = slot.as_u64() % spec.epoch_length; - let committees_per_slot = committees_per_epoch / spec.epoch_length; - let slot_start_shard = - (shuffling_start_shard + committees_per_slot * offset) % spec.shard_count; - - trace!( - "get_crosslink_committees_at_slot: committees_per_slot: {}, slot_start_shard: {}, seed: {}", - committees_per_slot, - slot_start_shard, - seed - ); - - let mut crosslinks_at_slot = vec![]; - for i in 0..committees_per_slot { - let tuple = ( - shuffling[(committees_per_slot * offset + i) as usize].clone(), - (slot_start_shard + i) % spec.shard_count, - ); - crosslinks_at_slot.push(tuple) - } - Ok(crosslinks_at_slot) - } - - /// Returns the `slot`, `shard` and `committee_index` for which a validator must produce an - /// attestation. - /// - /// Spec v0.2.0 - pub fn attestation_slot_and_shard_for_validator( - &self, - validator_index: usize, - spec: &ChainSpec, - ) -> Result, FastBeaconStateError> { - let mut result = None; - for slot in self.current_epoch(spec).slot_iter(spec.epoch_length) { - for (committee, shard) in self.get_crosslink_committees_at_slot(slot, false, spec)? { - if let Some(committee_index) = committee.iter().position(|&i| i == validator_index) - { - result = Some((slot, shard, committee_index as u64)); - } - } - } - Ok(result) - } - - /// An entry or exit triggered in the ``epoch`` given by the input takes effect at - /// the epoch given by the output. - /// - /// Spec v0.2.0 - pub fn get_entry_exit_effect_epoch(&self, epoch: Epoch, spec: &ChainSpec) -> Epoch { - epoch + 1 + spec.entry_exit_delay - } - - /// Returns the beacon proposer index for the `slot`. - /// - /// If the state does not contain an index for a beacon proposer at the requested `slot`, then `None` is returned. - /// - /// Spec v0.2.0 - pub fn get_beacon_proposer_index( - &self, - slot: Slot, - spec: &ChainSpec, - ) -> Result { - let committees = self.get_crosslink_committees_at_slot(slot, false, spec)?; - trace!( - "get_beacon_proposer_index: slot: {}, committees_count: {}", - slot, - committees.len() - ); - committees - .first() - .ok_or(FastBeaconStateError::InsufficientValidators) - .and_then(|(first_committee, _)| { - let index = (slot.as_usize()) - .checked_rem(first_committee.len()) - .ok_or(FastBeaconStateError::InsufficientValidators)?; - Ok(first_committee[index]) - }) - } - - /// Process the penalties and prepare the validators who are eligible to withdrawal. - /// - /// Spec v0.2.0 - pub fn process_penalties_and_exits(&mut self, spec: &ChainSpec) { - let current_epoch = self.current_epoch(spec); - let active_validator_indices = - get_active_validator_indices(&self.validator_registry, current_epoch); - let total_balance = self.get_total_balance(&active_validator_indices[..], spec); - - for index in 0..self.validator_balances.len() { - let validator = &self.validator_registry[index]; - - if current_epoch - == validator.penalized_epoch + Epoch::from(spec.latest_penalized_exit_length / 2) - { - let epoch_index: usize = - current_epoch.as_usize() % spec.latest_penalized_exit_length; - - let total_at_start = self.latest_penalized_balances - [(epoch_index + 1) % spec.latest_penalized_exit_length]; - let total_at_end = self.latest_penalized_balances[epoch_index]; - let total_penalities = total_at_end.saturating_sub(total_at_start); - let penalty = self.get_effective_balance(index, spec) - * std::cmp::min(total_penalities * 3, total_balance) - / total_balance; - safe_sub_assign!(self.validator_balances[index], penalty); - } - } - - let eligible = |index: usize| { - let validator = &self.validator_registry[index]; - - if validator.penalized_epoch <= current_epoch { - let penalized_withdrawal_epochs = spec.latest_penalized_exit_length / 2; - current_epoch >= validator.penalized_epoch + penalized_withdrawal_epochs as u64 - } else { - current_epoch >= validator.exit_epoch + spec.min_validator_withdrawal_epochs - } - }; - - let mut eligable_indices: Vec = (0..self.validator_registry.len()) - .filter(|i| eligible(*i)) - .collect(); - eligable_indices.sort_by_key(|i| self.validator_registry[*i].exit_epoch); - for (withdrawn_so_far, index) in eligable_indices.iter().enumerate() { - self.prepare_validator_for_withdrawal(*index); - if withdrawn_so_far as u64 >= spec.max_withdrawals_per_epoch { - break; - } - } - } - - /// Return the randao mix at a recent ``epoch``. - /// - /// Returns `None` if the epoch is out-of-bounds of `self.latest_randao_mixes`. - /// - /// Spec v0.2.0 - pub fn get_randao_mix(&self, epoch: Epoch, spec: &ChainSpec) -> Option<&Hash256> { - self.latest_randao_mixes - .get(epoch.as_usize() % spec.latest_randao_mixes_length) - } - - /// Update validator registry, activating/exiting validators if possible. - /// - /// Spec v0.2.0 - pub fn update_validator_registry(&mut self, spec: &ChainSpec) { - let current_epoch = self.current_epoch(spec); - let active_validator_indices = - get_active_validator_indices(&self.validator_registry, current_epoch); - let total_balance = self.get_total_balance(&active_validator_indices[..], spec); - - let max_balance_churn = std::cmp::max( - spec.max_deposit_amount, - total_balance / (2 * spec.max_balance_churn_quotient), - ); - - let mut balance_churn = 0; - for index in 0..self.validator_registry.len() { - let validator = &self.validator_registry[index]; - - if (validator.activation_epoch > self.get_entry_exit_effect_epoch(current_epoch, spec)) - && self.validator_balances[index] >= spec.max_deposit_amount - { - balance_churn += self.get_effective_balance(index, spec); - if balance_churn > max_balance_churn { - break; - } - self.activate_validator(index, false, spec); - } - } - - let mut balance_churn = 0; - for index in 0..self.validator_registry.len() { - let validator = &self.validator_registry[index]; - - if (validator.exit_epoch > self.get_entry_exit_effect_epoch(current_epoch, spec)) - && validator.status_flags == Some(StatusFlags::InitiatedExit) - { - balance_churn += self.get_effective_balance(index, spec); - if balance_churn > max_balance_churn { - break; - } - - self.exit_validator(index, spec); - } - } - - self.validator_registry_update_epoch = current_epoch; - } - /// Process a validator deposit, returning the validator index if the deposit is valid. - /// - /// Spec v0.2.0 - pub fn process_deposit( - &mut self, - pubkey: PublicKey, - amount: u64, - proof_of_possession: Signature, - withdrawal_credentials: Hash256, - spec: &ChainSpec, - ) -> Result { - // TODO: ensure verify proof-of-possession represents the spec accurately. - if !verify_proof_of_possession(&proof_of_possession, &pubkey) { - return Err(()); - } - - if let Some(index) = self - .validator_registry - .iter() - .position(|v| v.pubkey == pubkey) - { - if self.validator_registry[index].withdrawal_credentials == withdrawal_credentials { - safe_add_assign!(self.validator_balances[index], amount); - Ok(index) - } else { - Err(()) - } - } else { - let validator = Validator { - pubkey, - withdrawal_credentials, - activation_epoch: spec.far_future_epoch, - exit_epoch: spec.far_future_epoch, - withdrawal_epoch: spec.far_future_epoch, - penalized_epoch: spec.far_future_epoch, - status_flags: None, - }; - self.validator_registry.push(validator); - self.validator_balances.push(amount); - Ok(self.validator_registry.len() - 1) - } - } - - /// Activate the validator of the given ``index``. - /// - /// Spec v0.2.0 - pub fn activate_validator( - &mut self, - validator_index: usize, - is_genesis: bool, - spec: &ChainSpec, - ) { - let current_epoch = self.current_epoch(spec); - - self.validator_registry[validator_index].activation_epoch = if is_genesis { - spec.genesis_epoch - } else { - self.get_entry_exit_effect_epoch(current_epoch, spec) - } - } - - /// Initiate an exit for the validator of the given `index`. - /// - /// Spec v0.2.0 - pub fn initiate_validator_exit(&mut self, validator_index: usize) { - // TODO: the spec does an `|=` here, ensure this isn't buggy. - self.validator_registry[validator_index].status_flags = Some(StatusFlags::InitiatedExit); - } - - /// Exit the validator of the given `index`. - /// - /// Spec v0.2.0 - fn exit_validator(&mut self, validator_index: usize, spec: &ChainSpec) { - let current_epoch = self.current_epoch(spec); - - if self.validator_registry[validator_index].exit_epoch - <= self.get_entry_exit_effect_epoch(current_epoch, spec) - { - return; - } - - self.validator_registry[validator_index].exit_epoch = - self.get_entry_exit_effect_epoch(current_epoch, spec); - } - - /// Penalize the validator of the given ``index``. - /// - /// Exits the validator and assigns its effective balance to the block producer for this - /// state. - /// - /// Spec v0.2.0 - pub fn penalize_validator( - &mut self, - validator_index: usize, - spec: &ChainSpec, - ) -> Result<(), FastBeaconStateError> { - self.exit_validator(validator_index, spec); - let current_epoch = self.current_epoch(spec); - - self.latest_penalized_balances - [current_epoch.as_usize() % spec.latest_penalized_exit_length] += - self.get_effective_balance(validator_index, spec); - - let whistleblower_index = self.get_beacon_proposer_index(self.slot, spec)?; - let whistleblower_reward = self.get_effective_balance(validator_index, spec); - safe_add_assign!( - self.validator_balances[whistleblower_index as usize], - whistleblower_reward - ); - safe_sub_assign!( - self.validator_balances[validator_index], - whistleblower_reward - ); - self.validator_registry[validator_index].penalized_epoch = current_epoch; - Ok(()) - } - - /// Initiate an exit for the validator of the given `index`. - /// - /// Spec v0.2.0 - pub fn prepare_validator_for_withdrawal(&mut self, validator_index: usize) { - //TODO: we're not ANDing here, we're setting. Potentially wrong. - self.validator_registry[validator_index].status_flags = Some(StatusFlags::Withdrawable); - } - - /// Iterate through the validator registry and eject active validators with balance below - /// ``EJECTION_BALANCE``. - /// - /// Spec v0.2.0 - pub fn process_ejections(&mut self, spec: &ChainSpec) { - for validator_index in - get_active_validator_indices(&self.validator_registry, self.current_epoch(spec)) - { - if self.validator_balances[validator_index] < spec.ejection_balance { - self.exit_validator(validator_index, spec) - } - } - } - - /// Returns the penality that should be applied to some validator for inactivity. - /// - /// Note: this is defined "inline" in the spec, not as a helper function. - /// - /// Spec v0.2.0 - pub fn inactivity_penalty( - &self, - validator_index: usize, - epochs_since_finality: Epoch, - base_reward_quotient: u64, - spec: &ChainSpec, - ) -> u64 { - let effective_balance = self.get_effective_balance(validator_index, spec); - self.base_reward(validator_index, base_reward_quotient, spec) - + effective_balance * epochs_since_finality.as_u64() - / spec.inactivity_penalty_quotient - / 2 - } - - /// Returns the distance between the first included attestation for some validator and this - /// slot. - /// - /// Note: In the spec this is defined "inline", not as a helper function. - /// - /// Spec v0.2.0 - pub fn inclusion_distance( - &self, - attestations: &[&PendingAttestation], - validator_index: usize, - spec: &ChainSpec, - ) -> Result { - let attestation = - self.earliest_included_attestation(attestations, validator_index, spec)?; - Ok((attestation.inclusion_slot - attestation.data.slot).as_u64()) - } - - /// Returns the slot of the earliest included attestation for some validator. - /// - /// Note: In the spec this is defined "inline", not as a helper function. - /// - /// Spec v0.2.0 - pub fn inclusion_slot( - &self, - attestations: &[&PendingAttestation], - validator_index: usize, - spec: &ChainSpec, - ) -> Result { - let attestation = - self.earliest_included_attestation(attestations, validator_index, spec)?; - Ok(attestation.inclusion_slot) - } - - /// Finds the earliest included attestation for some validator. - /// - /// Note: In the spec this is defined "inline", not as a helper function. - /// - /// Spec v0.2.0 - fn earliest_included_attestation( - &self, - attestations: &[&PendingAttestation], - validator_index: usize, - spec: &ChainSpec, - ) -> Result { - let mut included_attestations = vec![]; - - for (i, a) in attestations.iter().enumerate() { - let participants = - self.get_attestation_participants(&a.data, &a.aggregation_bitfield, spec)?; - if participants.iter().any(|i| *i == validator_index) { - included_attestations.push(i); - } - } - - let earliest_attestation_index = included_attestations - .iter() - .min_by_key(|i| attestations[**i].inclusion_slot) - .ok_or_else(|| InclusionError::NoAttestationsForValidator)?; - Ok(attestations[*earliest_attestation_index].clone()) - } - - /// Returns the base reward for some validator. - /// - /// Note: In the spec this is defined "inline", not as a helper function. - /// - /// Spec v0.2.0 - pub fn base_reward( - &self, - validator_index: usize, - base_reward_quotient: u64, - spec: &ChainSpec, - ) -> u64 { - self.get_effective_balance(validator_index, spec) / base_reward_quotient / 5 - } - - /// Return the combined effective balance of an array of validators. - /// - /// Spec v0.2.0 - pub fn get_total_balance(&self, validator_indices: &[usize], spec: &ChainSpec) -> u64 { - validator_indices - .iter() - .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)) - } - - /// Return the effective balance (also known as "balance at stake") for a validator with the given ``index``. - /// - /// Spec v0.2.0 - pub fn get_effective_balance(&self, validator_index: usize, spec: &ChainSpec) -> u64 { - std::cmp::min( - self.validator_balances[validator_index], - spec.max_deposit_amount, - ) - } - - /// Return the block root at a recent `slot`. - /// - /// Spec v0.2.0 - pub fn get_block_root(&self, slot: Slot, spec: &ChainSpec) -> Option<&Hash256> { - self.latest_block_roots - .get(slot.as_usize() % spec.latest_block_roots_length) - } - - pub fn get_attestation_participants_union( - &self, - attestations: &[&PendingAttestation], - spec: &ChainSpec, - ) -> Result, AttestationParticipantsError> { - let mut all_participants = attestations.iter().try_fold::<_, _, Result< - Vec, - AttestationParticipantsError, - >>(vec![], |mut acc, a| { - acc.append(&mut self.get_attestation_participants( - &a.data, - &a.aggregation_bitfield, - spec, - )?); - Ok(acc) - })?; - all_participants.sort_unstable(); - all_participants.dedup(); - Ok(all_participants) - } - - /// Return the participant indices at for the ``attestation_data`` and ``bitfield``. - /// - /// In effect, this converts the "committee indices" on the bitfield into "validator indices" - /// for self.validator_registy. - /// - /// Spec v0.2.0 - pub fn get_attestation_participants( - &self, - attestation_data: &AttestationData, - bitfield: &Bitfield, - spec: &ChainSpec, - ) -> Result, AttestationParticipantsError> { - let crosslink_committees = - self.get_crosslink_committees_at_slot(attestation_data.slot, false, spec)?; - - let committee_index: usize = crosslink_committees - .iter() - .position(|(_committee, shard)| *shard == attestation_data.shard) - .ok_or_else(|| AttestationParticipantsError::NoCommitteeForShard)?; - let (crosslink_committee, _shard) = &crosslink_committees[committee_index]; - - /* - * TODO: verify bitfield length is valid. - */ - - let mut participants = vec![]; - for (i, validator_index) in crosslink_committee.iter().enumerate() { - if bitfield.get(i).unwrap() { - participants.push(*validator_index); - } - } - Ok(participants) - } -} - -fn hash_tree_root(input: Vec) -> Hash256 { - Hash256::from(&input.hash_tree_root()[..]) -} - -impl From for AttestationParticipantsError { - fn from(e: FastBeaconStateError) -> AttestationParticipantsError { - AttestationParticipantsError::FastBeaconStateError(e) - } -} - -impl From for InclusionError { - fn from(e: AttestationParticipantsError) -> InclusionError { - InclusionError::AttestationParticipantsError(e) - } -} - -impl Encodable for FastBeaconState { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.slot); - s.append(&self.genesis_time); - s.append(&self.fork); - s.append(&self.validator_registry); - s.append(&self.validator_balances); - s.append(&self.validator_registry_update_epoch); - s.append(&self.latest_randao_mixes); - s.append(&self.previous_epoch_start_shard); - s.append(&self.current_epoch_start_shard); - s.append(&self.previous_calculation_epoch); - s.append(&self.current_calculation_epoch); - s.append(&self.previous_epoch_seed); - s.append(&self.current_epoch_seed); - s.append(&self.previous_justified_epoch); - s.append(&self.justified_epoch); - s.append(&self.justification_bitfield); - s.append(&self.finalized_epoch); - s.append(&self.latest_crosslinks); - s.append(&self.latest_block_roots); - s.append(&self.latest_index_roots); - s.append(&self.latest_penalized_balances); - s.append(&self.latest_attestations); - s.append(&self.batched_block_roots); - s.append(&self.latest_eth1_data); - s.append(&self.eth1_data_votes); - } -} - -impl Decodable for FastBeaconState { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (slot, i) = <_>::ssz_decode(bytes, i)?; - let (genesis_time, i) = <_>::ssz_decode(bytes, i)?; - let (fork, i) = <_>::ssz_decode(bytes, i)?; - let (validator_registry, i) = <_>::ssz_decode(bytes, i)?; - let (validator_balances, i) = <_>::ssz_decode(bytes, i)?; - let (validator_registry_update_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (latest_randao_mixes, i) = <_>::ssz_decode(bytes, i)?; - let (previous_epoch_start_shard, i) = <_>::ssz_decode(bytes, i)?; - let (current_epoch_start_shard, i) = <_>::ssz_decode(bytes, i)?; - let (previous_calculation_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (current_calculation_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (previous_epoch_seed, i) = <_>::ssz_decode(bytes, i)?; - let (current_epoch_seed, i) = <_>::ssz_decode(bytes, i)?; - let (previous_justified_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (justified_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (justification_bitfield, i) = <_>::ssz_decode(bytes, i)?; - let (finalized_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (latest_crosslinks, i) = <_>::ssz_decode(bytes, i)?; - let (latest_block_roots, i) = <_>::ssz_decode(bytes, i)?; - let (latest_index_roots, i) = <_>::ssz_decode(bytes, i)?; - let (latest_penalized_balances, i) = <_>::ssz_decode(bytes, i)?; - let (latest_attestations, i) = <_>::ssz_decode(bytes, i)?; - let (batched_block_roots, i) = <_>::ssz_decode(bytes, i)?; - let (latest_eth1_data, i) = <_>::ssz_decode(bytes, i)?; - let (eth1_data_votes, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - Self { - slot, - genesis_time, - fork, - validator_registry, - validator_balances, - validator_registry_update_epoch, - latest_randao_mixes, - previous_epoch_start_shard, - current_epoch_start_shard, - previous_calculation_epoch, - current_calculation_epoch, - previous_epoch_seed, - current_epoch_seed, - previous_justified_epoch, - justified_epoch, - justification_bitfield, - finalized_epoch, - latest_crosslinks, - latest_block_roots, - latest_index_roots, - latest_penalized_balances, - latest_attestations, - batched_block_roots, - latest_eth1_data, - eth1_data_votes, - }, - i, - )) - } -} - -impl TreeHash for FastBeaconState { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.slot.hash_tree_root_internal()); - result.append(&mut self.genesis_time.hash_tree_root_internal()); - result.append(&mut self.fork.hash_tree_root_internal()); - result.append(&mut self.validator_registry.hash_tree_root_internal()); - result.append(&mut self.validator_balances.hash_tree_root_internal()); - result.append( - &mut self - .validator_registry_update_epoch - .hash_tree_root_internal(), - ); - result.append(&mut self.latest_randao_mixes.hash_tree_root_internal()); - result.append(&mut self.previous_epoch_start_shard.hash_tree_root_internal()); - result.append(&mut self.current_epoch_start_shard.hash_tree_root_internal()); - result.append(&mut self.previous_calculation_epoch.hash_tree_root_internal()); - result.append(&mut self.current_calculation_epoch.hash_tree_root_internal()); - result.append(&mut self.previous_epoch_seed.hash_tree_root_internal()); - result.append(&mut self.current_epoch_seed.hash_tree_root_internal()); - result.append(&mut self.previous_justified_epoch.hash_tree_root_internal()); - result.append(&mut self.justified_epoch.hash_tree_root_internal()); - result.append(&mut self.justification_bitfield.hash_tree_root_internal()); - result.append(&mut self.finalized_epoch.hash_tree_root_internal()); - result.append(&mut self.latest_crosslinks.hash_tree_root_internal()); - result.append(&mut self.latest_block_roots.hash_tree_root_internal()); - result.append(&mut self.latest_index_roots.hash_tree_root_internal()); - result.append(&mut self.latest_penalized_balances.hash_tree_root_internal()); - result.append(&mut self.latest_attestations.hash_tree_root_internal()); - result.append(&mut self.batched_block_roots.hash_tree_root_internal()); - result.append(&mut self.latest_eth1_data.hash_tree_root_internal()); - result.append(&mut self.eth1_data_votes.hash_tree_root_internal()); - hash(&result) - } -} - -impl TestRandom for FastBeaconState { - fn random_for_test(rng: &mut T) -> Self { - Self { - slot: <_>::random_for_test(rng), - genesis_time: <_>::random_for_test(rng), - fork: <_>::random_for_test(rng), - validator_registry: <_>::random_for_test(rng), - validator_balances: <_>::random_for_test(rng), - validator_registry_update_epoch: <_>::random_for_test(rng), - latest_randao_mixes: <_>::random_for_test(rng), - previous_epoch_start_shard: <_>::random_for_test(rng), - current_epoch_start_shard: <_>::random_for_test(rng), - previous_calculation_epoch: <_>::random_for_test(rng), - current_calculation_epoch: <_>::random_for_test(rng), - previous_epoch_seed: <_>::random_for_test(rng), - current_epoch_seed: <_>::random_for_test(rng), - previous_justified_epoch: <_>::random_for_test(rng), - justified_epoch: <_>::random_for_test(rng), - justification_bitfield: <_>::random_for_test(rng), - finalized_epoch: <_>::random_for_test(rng), - latest_crosslinks: <_>::random_for_test(rng), - latest_block_roots: <_>::random_for_test(rng), - latest_index_roots: <_>::random_for_test(rng), - latest_penalized_balances: <_>::random_for_test(rng), - latest_attestations: <_>::random_for_test(rng), - batched_block_roots: <_>::random_for_test(rng), - latest_eth1_data: <_>::random_for_test(rng), - eth1_data_votes: <_>::random_for_test(rng), - } - } -} diff --git a/eth2/types/src/fast_beacon_state/committees_cache.rs b/eth2/types/src/fast_beacon_state/committees_cache.rs deleted file mode 100644 index 42dd101ba..000000000 --- a/eth2/types/src/fast_beacon_state/committees_cache.rs +++ /dev/null @@ -1,128 +0,0 @@ -use crate::{ - validator_registry::get_active_validator_indices, BeaconState, ChainSpec, Epoch, Hash256, -}; -use honey_badger_split::SplitExt; -use serde_derive::Serialize; -use swap_or_not_shuffle::get_permutated_index; - -pub const CACHED_EPOCHS: usize = 3; - -#[derive(Debug, PartialEq, Clone, Default, Serialize)] -pub struct CommitteesCache { - cache_index_offset: usize, - active_validator_indices_cache: Vec>>, - shuffling_cache: Vec>>>>, - previous_epoch: Epoch, - current_epoch: Epoch, - next_epoch: Epoch, -} - -impl CommitteesCache { - pub fn new(current_epoch: Epoch, spec: &ChainSpec) -> Self { - let previous_epoch = if current_epoch == spec.genesis_epoch { - current_epoch - } else { - current_epoch - 1 - }; - let next_epoch = current_epoch + 1; - - Self { - cache_index_offset: 0, - active_validator_indices_cache: vec![None; CACHED_EPOCHS], - shuffling_cache: vec![None; CACHED_EPOCHS], - previous_epoch, - current_epoch, - next_epoch, - } - } - - pub fn advance_epoch(&mut self) { - let previous_cache_index = self.cache_index(self.previous_epoch); - - self.active_validator_indices_cache[previous_cache_index] = None; - self.shuffling_cache[previous_cache_index] = None; - - self.cache_index_offset += 1; - self.cache_index_offset %= CACHED_EPOCHS; - } - - pub fn cache_index(&self, epoch: Epoch) -> usize { - let base_index = match epoch { - e if e == self.previous_epoch => 0, - e if e == self.current_epoch => 1, - e if e == self.next_epoch => 2, - _ => panic!("Bad cache index."), - }; - - (base_index + self.cache_index_offset) % CACHED_EPOCHS - } - - pub fn get_active_validator_indices( - &mut self, - state: &BeaconState, - epoch: Epoch, - ) -> &Vec { - let i = self.cache_index(epoch); - - if self.active_validator_indices_cache[i] == None { - self.active_validator_indices_cache[i] = Some(get_active_validator_indices( - &state.validator_registry, - epoch, - )); - } - - self.active_validator_indices_cache[i] - .as_ref() - .expect("Cache cannot be None") - } - - pub fn get_shuffling( - &mut self, - state: &BeaconState, - seed: Hash256, - epoch: Epoch, - spec: &ChainSpec, - ) -> Option<&Vec>> { - let cache_index = self.cache_index(epoch); - - if self.shuffling_cache[cache_index] == None { - let active_validator_indices = self.get_active_validator_indices(&state, epoch); - - if active_validator_indices.is_empty() { - return None; - } - - let committees_per_epoch = - state.get_epoch_committee_count(active_validator_indices.len(), spec); - - let mut shuffled_active_validator_indices = vec![0; active_validator_indices.len()]; - for &i in active_validator_indices { - let shuffled_i = get_permutated_index( - i, - active_validator_indices.len(), - &seed[..], - spec.shuffle_round_count, - )?; - shuffled_active_validator_indices[i] = active_validator_indices[shuffled_i] - } - - let committees: Vec> = shuffled_active_validator_indices - .honey_badger_split(committees_per_epoch as usize) - .map(|slice: &[usize]| slice.to_vec()) - .collect(); - - self.shuffling_cache[cache_index] = Some(Some(committees)); - } - - match self.shuffling_cache[cache_index] { - Some(_) => Some( - self.shuffling_cache[cache_index] - .as_ref() - .expect("Cache cannot be None") - .as_ref() - .expect("Shuffling cannot be None."), - ), - None => None, - } - } -} diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index 820c26ad6..f2c128440 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -16,7 +16,6 @@ pub mod deposit_input; pub mod eth1_data; pub mod eth1_data_vote; pub mod exit; -pub mod fast_beacon_state; pub mod fork; pub mod free_attestation; pub mod pending_attestation; @@ -53,7 +52,6 @@ pub use crate::deposit_input::DepositInput; pub use crate::eth1_data::Eth1Data; pub use crate::eth1_data_vote::Eth1DataVote; pub use crate::exit::Exit; -pub use crate::fast_beacon_state::FastBeaconState; pub use crate::fork::Fork; pub use crate::free_attestation::FreeAttestation; pub use crate::pending_attestation::PendingAttestation; From a5de6a1915b3d9b8afd24d22f0c1acb2987abf40 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 22 Feb 2019 18:14:16 +1300 Subject: [PATCH 006/132] Add caching to BeaconState. Removes CachingBeaconState --- .../src/attestation_aggregator.rs | 111 +++---- beacon_node/beacon_chain/src/beacon_chain.rs | 35 ++- .../beacon_chain/src/cached_beacon_state.rs | 150 --------- beacon_node/beacon_chain/src/checkpoint.rs | 2 +- beacon_node/beacon_chain/src/lib.rs | 5 +- .../test_harness/src/beacon_chain_harness.rs | 13 +- .../beacon_chain/test_harness/tests/chain.rs | 3 + .../state_processing/src/block_processable.rs | 29 +- .../state_processing/src/epoch_processable.rs | 65 ++-- eth2/state_processing/src/slot_processable.rs | 2 +- eth2/types/src/beacon_state.rs | 286 +++++++++++------- eth2/types/src/beacon_state/epoch_cache.rs | 77 +++++ eth2/types/src/beacon_state/tests.rs | 51 +++- eth2/types/src/lib.rs | 4 +- 14 files changed, 459 insertions(+), 374 deletions(-) delete mode 100644 beacon_node/beacon_chain/src/cached_beacon_state.rs create mode 100644 eth2/types/src/beacon_state/epoch_cache.rs diff --git a/beacon_node/beacon_chain/src/attestation_aggregator.rs b/beacon_node/beacon_chain/src/attestation_aggregator.rs index fa2ec87ab..70348dc94 100644 --- a/beacon_node/beacon_chain/src/attestation_aggregator.rs +++ b/beacon_node/beacon_chain/src/attestation_aggregator.rs @@ -1,9 +1,9 @@ -use crate::cached_beacon_state::CachedBeaconState; +use log::trace; use state_processing::validate_attestation_without_signature; use std::collections::{HashMap, HashSet}; use types::{ - beacon_state::BeaconStateError, AggregateSignature, Attestation, AttestationData, BeaconState, - Bitfield, ChainSpec, FreeAttestation, Signature, + AggregateSignature, Attestation, AttestationData, BeaconState, BeaconStateError, Bitfield, + ChainSpec, FreeAttestation, Signature, }; const PHASE_0_CUSTODY_BIT: bool = false; @@ -42,21 +42,28 @@ pub enum Message { BadSignature, /// The given `slot` does not match the validators committee assignment. BadSlot, - /// The given `shard` does not match the validators committee assignment. + /// The given `shard` does not match the validators committee assignment, or is not included in + /// a committee for the given slot. BadShard, + /// Attestation is from the epoch prior to this, ignoring. + TooOld, } -macro_rules! some_or_invalid { - ($expression: expr, $error: expr) => { - match $expression { - Some(x) => x, - None => { - return Ok(Outcome { - valid: false, - message: $error, - }); - } - } +macro_rules! valid_outcome { + ($error: expr) => { + return Ok(Outcome { + valid: true, + message: $error, + }); + }; +} + +macro_rules! invalid_outcome { + ($error: expr) => { + return Ok(Outcome { + valid: false, + message: $error, + }); }; } @@ -77,49 +84,56 @@ impl AttestationAggregator { /// - The signature is verified against that of the validator at `validator_index`. pub fn process_free_attestation( &mut self, - cached_state: &CachedBeaconState, + cached_state: &BeaconState, free_attestation: &FreeAttestation, spec: &ChainSpec, ) -> Result { - let (slot, shard, committee_index) = some_or_invalid!( - cached_state.attestation_slot_and_shard_for_validator( - free_attestation.validator_index as usize, - spec, - )?, - Message::BadValidatorIndex + let attestation_duties = match cached_state.attestation_slot_and_shard_for_validator( + free_attestation.validator_index as usize, + spec, + ) { + Err(BeaconStateError::EpochCacheUninitialized(e)) => { + panic!("Attempted to access unbuilt cache {:?}.", e) + } + Err(BeaconStateError::EpochOutOfBounds) => invalid_outcome!(Message::TooOld), + Err(BeaconStateError::ShardOutOfBounds) => invalid_outcome!(Message::BadShard), + Err(e) => return Err(e), + Ok(None) => invalid_outcome!(Message::BadValidatorIndex), + Ok(Some(attestation_duties)) => attestation_duties, + }; + + let (slot, shard, committee_index) = attestation_duties; + + trace!( + "slot: {}, shard: {}, committee_index: {}, val_index: {}", + slot, + shard, + committee_index, + free_attestation.validator_index ); if free_attestation.data.slot != slot { - return Ok(Outcome { - valid: false, - message: Message::BadSlot, - }); + invalid_outcome!(Message::BadSlot); } if free_attestation.data.shard != shard { - return Ok(Outcome { - valid: false, - message: Message::BadShard, - }); + invalid_outcome!(Message::BadShard); } let signable_message = free_attestation.data.signable_message(PHASE_0_CUSTODY_BIT); - let validator_record = some_or_invalid!( - cached_state - .state - .validator_registry - .get(free_attestation.validator_index as usize), - Message::BadValidatorIndex - ); + let validator_record = match cached_state + .validator_registry + .get(free_attestation.validator_index as usize) + { + None => invalid_outcome!(Message::BadValidatorIndex), + Some(validator_record) => validator_record, + }; if !free_attestation .signature .verify(&signable_message, &validator_record.pubkey) { - return Ok(Outcome { - valid: false, - message: Message::BadSignature, - }); + invalid_outcome!(Message::BadSignature); } if let Some(existing_attestation) = self.store.get(&signable_message) { @@ -129,15 +143,9 @@ impl AttestationAggregator { committee_index as usize, ) { self.store.insert(signable_message, updated_attestation); - Ok(Outcome { - valid: true, - message: Message::Aggregated, - }) + valid_outcome!(Message::Aggregated); } else { - Ok(Outcome { - valid: true, - message: Message::AggregationNotRequired, - }) + valid_outcome!(Message::AggregationNotRequired); } } else { let mut aggregate_signature = AggregateSignature::new(); @@ -151,10 +159,7 @@ impl AttestationAggregator { aggregate_signature, }; self.store.insert(signable_message, new_attestation); - Ok(Outcome { - valid: true, - message: Message::NewAttestationCreated, - }) + valid_outcome!(Message::NewAttestationCreated); } } diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index b2d041654..9ee55e5a3 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1,5 +1,4 @@ use crate::attestation_aggregator::{AttestationAggregator, Outcome as AggregationOutcome}; -use crate::cached_beacon_state::CachedBeaconState; use crate::checkpoint::CheckPoint; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, @@ -15,10 +14,10 @@ use state_processing::{ }; use std::sync::Arc; use types::{ - beacon_state::BeaconStateError, readers::{BeaconBlockReader, BeaconStateReader}, - AttestationData, BeaconBlock, BeaconBlockBody, BeaconState, ChainSpec, Crosslink, Deposit, - Epoch, Eth1Data, FreeAttestation, Hash256, PublicKey, Signature, Slot, + AttestationData, BeaconBlock, BeaconBlockBody, BeaconState, BeaconStateError, ChainSpec, + Crosslink, Deposit, Epoch, Eth1Data, FreeAttestation, Hash256, PublicKey, RelativeEpoch, + Signature, Slot, }; #[derive(Debug, PartialEq)] @@ -70,7 +69,6 @@ pub struct BeaconChain { canonical_head: RwLock, finalized_head: RwLock, pub state: RwLock, - pub cached_state: RwLock, pub spec: ChainSpec, pub fork_choice: RwLock, } @@ -96,7 +94,7 @@ where return Err(Error::InsufficientValidators); } - let genesis_state = BeaconState::genesis( + let mut genesis_state = BeaconState::genesis( genesis_time, initial_validator_deposits, latest_eth1_data, @@ -109,32 +107,32 @@ where let block_root = genesis_block.canonical_root(); block_store.put(&block_root, &ssz_encode(&genesis_block)[..])?; - let cached_state = RwLock::new(CachedBeaconState::from_beacon_state( - genesis_state.clone(), - spec.clone(), - )?); - let finalized_head = RwLock::new(CheckPoint::new( genesis_block.clone(), block_root, + // TODO: this is a memory waste; remove full clone. genesis_state.clone(), state_root, )); let canonical_head = RwLock::new(CheckPoint::new( genesis_block.clone(), block_root, + // TODO: this is a memory waste; remove full clone. genesis_state.clone(), state_root, )); let attestation_aggregator = RwLock::new(AttestationAggregator::new()); + genesis_state.build_epoch_cache(RelativeEpoch::Previous, &spec)?; + genesis_state.build_epoch_cache(RelativeEpoch::Current, &spec)?; + genesis_state.build_epoch_cache(RelativeEpoch::Next, &spec)?; + Ok(Self { block_store, state_store, slot_clock, attestation_aggregator, - state: RwLock::new(genesis_state.clone()), - cached_state, + state: RwLock::new(genesis_state), finalized_head, canonical_head, spec, @@ -150,6 +148,10 @@ where new_beacon_state: BeaconState, new_beacon_state_root: Hash256, ) { + debug!( + "Updating canonical head with block at slot: {}", + new_beacon_block.slot + ); let mut head = self.canonical_head.write(); head.update( new_beacon_block, @@ -288,7 +290,7 @@ where validator_index ); if let Some((slot, shard, _committee)) = self - .cached_state + .state .read() .attestation_slot_and_shard_for_validator(validator_index, &self.spec)? { @@ -346,7 +348,7 @@ where let aggregation_outcome = self .attestation_aggregator .write() - .process_free_attestation(&self.cached_state.read(), &free_attestation, &self.spec)?; + .process_free_attestation(&self.state.read(), &free_attestation, &self.spec)?; // return if the attestation is invalid if !aggregation_outcome.valid { @@ -501,9 +503,6 @@ where ); // Update the local state variable. *self.state.write() = state.clone(); - // Update the cached state variable. - *self.cached_state.write() = - CachedBeaconState::from_beacon_state(state.clone(), self.spec.clone())?; } Ok(BlockProcessingOutcome::ValidBlock(ValidBlock::Processed)) diff --git a/beacon_node/beacon_chain/src/cached_beacon_state.rs b/beacon_node/beacon_chain/src/cached_beacon_state.rs deleted file mode 100644 index e14e9fe99..000000000 --- a/beacon_node/beacon_chain/src/cached_beacon_state.rs +++ /dev/null @@ -1,150 +0,0 @@ -use log::{debug, trace}; -use std::collections::HashMap; -use types::{beacon_state::BeaconStateError, BeaconState, ChainSpec, Epoch, Slot}; - -pub const CACHE_PREVIOUS: bool = false; -pub const CACHE_CURRENT: bool = true; -pub const CACHE_NEXT: bool = false; - -pub type CrosslinkCommittees = Vec<(Vec, u64)>; -pub type Shard = u64; -pub type CommitteeIndex = u64; -pub type AttestationDuty = (Slot, Shard, CommitteeIndex); -pub type AttestationDutyMap = HashMap; - -// TODO: CachedBeaconState is presently duplicating `BeaconState` and `ChainSpec`. This is a -// massive memory waste, switch them to references. - -pub struct CachedBeaconState { - pub state: BeaconState, - committees: Vec>, - attestation_duties: Vec, - next_epoch: Epoch, - current_epoch: Epoch, - previous_epoch: Epoch, - spec: ChainSpec, -} - -impl CachedBeaconState { - pub fn from_beacon_state( - state: BeaconState, - spec: ChainSpec, - ) -> Result { - let current_epoch = state.current_epoch(&spec); - let previous_epoch = if current_epoch == spec.genesis_epoch { - current_epoch - } else { - current_epoch.saturating_sub(1_u64) - }; - let next_epoch = state.next_epoch(&spec); - - let mut committees: Vec> = Vec::with_capacity(3); - let mut attestation_duties: Vec = Vec::with_capacity(3); - - if CACHE_PREVIOUS { - debug!("from_beacon_state: building previous epoch cache."); - let cache = build_epoch_cache(&state, previous_epoch, &spec)?; - committees.push(cache.committees); - attestation_duties.push(cache.attestation_duty_map); - } else { - committees.push(vec![]); - attestation_duties.push(HashMap::new()); - } - if CACHE_CURRENT { - debug!("from_beacon_state: building current epoch cache."); - let cache = build_epoch_cache(&state, current_epoch, &spec)?; - committees.push(cache.committees); - attestation_duties.push(cache.attestation_duty_map); - } else { - committees.push(vec![]); - attestation_duties.push(HashMap::new()); - } - if CACHE_NEXT { - debug!("from_beacon_state: building next epoch cache."); - let cache = build_epoch_cache(&state, next_epoch, &spec)?; - committees.push(cache.committees); - attestation_duties.push(cache.attestation_duty_map); - } else { - committees.push(vec![]); - attestation_duties.push(HashMap::new()); - } - - Ok(Self { - state, - committees, - attestation_duties, - next_epoch, - current_epoch, - previous_epoch, - spec, - }) - } - - fn slot_to_cache_index(&self, slot: Slot) -> Option { - trace!("slot_to_cache_index: cache lookup"); - match slot.epoch(self.spec.epoch_length) { - epoch if (epoch == self.previous_epoch) & CACHE_PREVIOUS => Some(0), - epoch if (epoch == self.current_epoch) & CACHE_CURRENT => Some(1), - epoch if (epoch == self.next_epoch) & CACHE_NEXT => Some(2), - _ => None, - } - } - - /// Returns the `slot`, `shard` and `committee_index` for which a validator must produce an - /// attestation. - /// - /// Cached method. - /// - /// Spec v0.2.0 - pub fn attestation_slot_and_shard_for_validator( - &self, - validator_index: usize, - _spec: &ChainSpec, - ) -> Result, BeaconStateError> { - // Get the result for this epoch. - let cache_index = self - .slot_to_cache_index(self.state.slot) - .expect("Current epoch should always have a cache index."); - - let duties = self.attestation_duties[cache_index] - .get(&(validator_index as u64)) - .and_then(|tuple| Some(*tuple)); - - Ok(duties) - } -} - -struct EpochCacheResult { - committees: Vec, - attestation_duty_map: AttestationDutyMap, -} - -fn build_epoch_cache( - state: &BeaconState, - epoch: Epoch, - spec: &ChainSpec, -) -> Result { - let mut epoch_committees: Vec = - Vec::with_capacity(spec.epoch_length as usize); - let mut attestation_duty_map: AttestationDutyMap = HashMap::new(); - - for slot in epoch.slot_iter(spec.epoch_length) { - let slot_committees = state.get_crosslink_committees_at_slot(slot, false, spec)?; - - for (committee, shard) in slot_committees { - for (committee_index, validator_index) in committee.iter().enumerate() { - attestation_duty_map.insert( - *validator_index as u64, - (slot, shard, committee_index as u64), - ); - } - } - - epoch_committees.push(state.get_crosslink_committees_at_slot(slot, false, spec)?) - } - - Ok(EpochCacheResult { - committees: epoch_committees, - attestation_duty_map, - }) -} diff --git a/beacon_node/beacon_chain/src/checkpoint.rs b/beacon_node/beacon_chain/src/checkpoint.rs index bef97d2ed..828e462de 100644 --- a/beacon_node/beacon_chain/src/checkpoint.rs +++ b/beacon_node/beacon_chain/src/checkpoint.rs @@ -3,7 +3,7 @@ use types::{BeaconBlock, BeaconState, Hash256}; /// Represents some block and it's associated state. Generally, this will be used for tracking the /// head, justified head and finalized head. -#[derive(PartialEq, Clone, Serialize)] +#[derive(Clone, Serialize)] pub struct CheckPoint { pub beacon_block: BeaconBlock, pub beacon_block_root: Hash256, diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index bc9085fbe..d7d1d9664 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -1,8 +1,9 @@ mod attestation_aggregator; mod beacon_chain; -mod cached_beacon_state; mod checkpoint; -pub use self::beacon_chain::{BeaconChain, Error}; +pub use self::beacon_chain::{ + BeaconChain, BlockProcessingOutcome, Error, InvalidBlock, ValidBlock, +}; pub use self::checkpoint::CheckPoint; pub use fork_choice::{ForkChoice, ForkChoiceAlgorithms, ForkChoiceError}; diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index 9d61952f0..a15e82aa2 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -1,5 +1,5 @@ use super::ValidatorHarness; -use beacon_chain::BeaconChain; +use beacon_chain::{BeaconChain, BlockProcessingOutcome}; pub use beacon_chain::{CheckPoint, Error as BeaconChainError}; use bls::create_proof_of_possession; use db::{ @@ -157,7 +157,7 @@ impl BeaconChainHarness { .beacon_chain .state .read() - .get_crosslink_committees_at_slot(present_slot, false, &self.spec) + .get_crosslink_committees_at_slot(present_slot, &self.spec) .unwrap() .iter() .fold(vec![], |mut acc, (committee, _slot)| { @@ -223,7 +223,10 @@ impl BeaconChainHarness { debug!("Producing block..."); let block = self.produce_block(); debug!("Submitting block for processing..."); - self.beacon_chain.process_block(block).unwrap(); + match self.beacon_chain.process_block(block) { + Ok(BlockProcessingOutcome::ValidBlock(_)) => {} + other => panic!("block processing failed with {:?}", other), + }; debug!("...block processed by BeaconChain."); debug!("Producing free attestations..."); @@ -242,6 +245,10 @@ impl BeaconChainHarness { debug!("Free attestations processed."); } + pub fn run_fork_choice(&mut self) { + self.beacon_chain.fork_choice().unwrap() + } + /// Dump all blocks and states from the canonical beacon chain. pub fn chain_dump(&self) -> Result, BeaconChainError> { self.beacon_chain.chain_dump() diff --git a/beacon_node/beacon_chain/test_harness/tests/chain.rs b/beacon_node/beacon_chain/test_harness/tests/chain.rs index 1a08ffcf1..1b29a412f 100644 --- a/beacon_node/beacon_chain/test_harness/tests/chain.rs +++ b/beacon_node/beacon_chain/test_harness/tests/chain.rs @@ -35,6 +35,9 @@ fn it_can_produce_past_first_epoch_boundary() { harness.advance_chain_with_block(); debug!("Produced block {}/{}.", i + 1, blocks); } + + harness.run_fork_choice(); + let dump = harness.chain_dump().expect("Chain dump failed."); assert_eq!(dump.len() as u64, blocks + 1); // + 1 for genesis block. diff --git a/eth2/state_processing/src/block_processable.rs b/eth2/state_processing/src/block_processable.rs index 539711c69..0e6b57cf0 100644 --- a/eth2/state_processing/src/block_processable.rs +++ b/eth2/state_processing/src/block_processable.rs @@ -4,9 +4,8 @@ use int_to_bytes::int_to_bytes32; use log::{debug, trace}; use ssz::{ssz_encode, TreeHash}; use types::{ - beacon_state::{AttestationParticipantsError, BeaconStateError}, - AggregatePublicKey, Attestation, BeaconBlock, BeaconState, ChainSpec, Crosslink, Epoch, Exit, - Fork, Hash256, PendingAttestation, PublicKey, Signature, + AggregatePublicKey, Attestation, BeaconBlock, BeaconState, BeaconStateError, ChainSpec, + Crosslink, Epoch, Exit, Fork, Hash256, PendingAttestation, PublicKey, RelativeEpoch, Signature, }; // TODO: define elsehwere. @@ -27,7 +26,6 @@ pub enum Error { MissingBeaconBlock(Hash256), InvalidBeaconBlock(Hash256), MissingParentBlock(Hash256), - NoBlockProducer, StateSlotMismatch, BadBlockSignature, BadRandaoSignature, @@ -56,7 +54,7 @@ pub enum AttestationValidationError { BadSignature, ShardBlockRootNotZero, NoBlockRoot, - AttestationParticipantsError(AttestationParticipantsError), + BeaconStateError(BeaconStateError), } macro_rules! ensure { @@ -98,12 +96,15 @@ fn per_block_processing_signature_optional( ) -> Result<(), Error> { ensure!(block.slot == state.slot, Error::StateSlotMismatch); + // Building the previous epoch could be delayed until an attestation from a previous epoch is + // included. This is left for future optimisation. + state.build_epoch_cache(RelativeEpoch::Previous, spec)?; + state.build_epoch_cache(RelativeEpoch::Current, spec)?; + /* * Proposer Signature */ - let block_proposer_index = state - .get_beacon_proposer_index(block.slot, spec) - .map_err(|_| Error::NoBlockProducer)?; + let block_proposer_index = state.get_beacon_proposer_index(block.slot, spec)?; let block_proposer = &state.validator_registry[block_proposer_index]; if verify_block_signature { @@ -361,6 +362,12 @@ fn validate_attestation_signature_optional( &attestation.aggregation_bitfield, spec, )?; + trace!( + "slot: {}, shard: {}, participants: {:?}", + attestation.data.slot, + attestation.data.shard, + participants + ); let mut group_public_key = AggregatePublicKey::new(); for participant in participants { group_public_key.add( @@ -417,8 +424,8 @@ impl From for Error { } } -impl From for AttestationValidationError { - fn from(e: AttestationParticipantsError) -> AttestationValidationError { - AttestationValidationError::AttestationParticipantsError(e) +impl From for AttestationValidationError { + fn from(e: BeaconStateError) -> AttestationValidationError { + AttestationValidationError::BeaconStateError(e) } } diff --git a/eth2/state_processing/src/epoch_processable.rs b/eth2/state_processing/src/epoch_processable.rs index 11b2b224d..409d40a2c 100644 --- a/eth2/state_processing/src/epoch_processable.rs +++ b/eth2/state_processing/src/epoch_processable.rs @@ -5,9 +5,8 @@ use ssz::TreeHash; use std::collections::{HashMap, HashSet}; use std::iter::FromIterator; use types::{ - beacon_state::{AttestationParticipantsError, BeaconStateError, InclusionError}, - validator_registry::get_active_validator_indices, - BeaconState, ChainSpec, Crosslink, Epoch, Hash256, PendingAttestation, + validator_registry::get_active_validator_indices, BeaconState, BeaconStateError, ChainSpec, + Crosslink, Epoch, Hash256, InclusionError, PendingAttestation, RelativeEpoch, }; macro_rules! safe_add_assign { @@ -28,7 +27,6 @@ pub enum Error { BaseRewardQuotientIsZero, NoRandaoSeed, BeaconStateError(BeaconStateError), - AttestationParticipantsError(AttestationParticipantsError), InclusionError(InclusionError), WinningRootError(WinningRootError), } @@ -36,7 +34,7 @@ pub enum Error { #[derive(Debug, PartialEq)] pub enum WinningRootError { NoWinningRoot, - AttestationParticipantsError(AttestationParticipantsError), + BeaconStateError(BeaconStateError), } #[derive(Clone)] @@ -66,6 +64,11 @@ impl EpochProcessable for BeaconState { self.current_epoch(spec) ); + // Ensure all of the caches are built. + self.build_epoch_cache(RelativeEpoch::Previous, spec)?; + self.build_epoch_cache(RelativeEpoch::Current, spec)?; + self.build_epoch_cache(RelativeEpoch::Next, spec)?; + /* * Validators attesting during the current epoch. */ @@ -322,8 +325,11 @@ impl EpochProcessable for BeaconState { slot, slot.epoch(spec.epoch_length) ); + + // Clone is used to remove the borrow. It becomes an issue later when trying to mutate + // `self.balances`. let crosslink_committees_at_slot = - self.get_crosslink_committees_at_slot(slot, false, spec)?; + self.get_crosslink_committees_at_slot(slot, spec)?.clone(); for (crosslink_committee, shard) in crosslink_committees_at_slot { let shard = shard as u64; @@ -499,8 +505,10 @@ impl EpochProcessable for BeaconState { * Crosslinks */ for slot in self.previous_epoch(spec).slot_iter(spec.epoch_length) { + // Clone is used to remove the borrow. It becomes an issue later when trying to mutate + // `self.balances`. let crosslink_committees_at_slot = - self.get_crosslink_committees_at_slot(slot, false, spec)?; + self.get_crosslink_committees_at_slot(slot, spec)?.clone(); for (_crosslink_committee, shard) in crosslink_committees_at_slot { let shard = shard as u64; @@ -609,6 +617,12 @@ impl EpochProcessable for BeaconState { .cloned() .collect(); + /* + * Manage the beacon state caches + */ + self.advance_caches(); + self.build_epoch_cache(RelativeEpoch::Next, spec)?; + debug!("Epoch transition complete."); Ok(()) @@ -645,19 +659,18 @@ fn winning_root( } // TODO: `cargo fmt` makes this rather ugly; tidy up. - let attesting_validator_indices = attestations.iter().try_fold::<_, _, Result< - _, - AttestationParticipantsError, - >>(vec![], |mut acc, a| { - if (a.data.shard == shard) && (a.data.shard_block_root == *shard_block_root) { - acc.append(&mut state.get_attestation_participants( - &a.data, - &a.aggregation_bitfield, - spec, - )?); - } - Ok(acc) - })?; + let attesting_validator_indices = attestations + .iter() + .try_fold::<_, _, Result<_, BeaconStateError>>(vec![], |mut acc, a| { + if (a.data.shard == shard) && (a.data.shard_block_root == *shard_block_root) { + acc.append(&mut state.get_attestation_participants( + &a.data, + &a.aggregation_bitfield, + spec, + )?); + } + Ok(acc) + })?; let total_balance: u64 = attesting_validator_indices .iter() @@ -708,15 +721,9 @@ impl From for Error { } } -impl From for Error { - fn from(e: AttestationParticipantsError) -> Error { - Error::AttestationParticipantsError(e) - } -} - -impl From for WinningRootError { - fn from(e: AttestationParticipantsError) -> WinningRootError { - WinningRootError::AttestationParticipantsError(e) +impl From for WinningRootError { + fn from(e: BeaconStateError) -> WinningRootError { + WinningRootError::BeaconStateError(e) } } diff --git a/eth2/state_processing/src/slot_processable.rs b/eth2/state_processing/src/slot_processable.rs index 9e3b611fd..0bbc79ab0 100644 --- a/eth2/state_processing/src/slot_processable.rs +++ b/eth2/state_processing/src/slot_processable.rs @@ -1,5 +1,5 @@ use crate::{EpochProcessable, EpochProcessingError}; -use types::{beacon_state::BeaconStateError, BeaconState, ChainSpec, Hash256}; +use types::{BeaconState, BeaconStateError, ChainSpec, Hash256}; #[derive(Debug, PartialEq)] pub enum Error { diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 2216e9516..47805da80 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -1,3 +1,4 @@ +use self::epoch_cache::EpochCache; use crate::test_utils::TestRandom; use crate::{ validator::StatusFlags, validator_registry::get_active_validator_indices, AttestationData, @@ -10,13 +11,35 @@ use log::trace; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use std::collections::HashMap; use swap_or_not_shuffle::get_permutated_index; +mod epoch_cache; mod tests; +pub type Committee = Vec; +pub type CrosslinkCommittees = Vec<(Committee, u64)>; +pub type Shard = u64; +pub type CommitteeIndex = u64; +pub type AttestationDuty = (Slot, Shard, CommitteeIndex); +pub type AttestationDutyMap = HashMap; +pub type ShardCommitteeIndexMap = HashMap; + +pub const CACHED_EPOCHS: usize = 3; + +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum RelativeEpoch { + Previous, + Current, + Next, +} + #[derive(Debug, PartialEq)] -pub enum BeaconStateError { +pub enum Error { EpochOutOfBounds, + /// The supplied shard is unknown. It may be larger than the maximum shard count, or not in a + /// committee for the given slot. + ShardOutOfBounds, UnableToShuffle, InsufficientRandaoMixes, InsufficientValidators, @@ -24,20 +47,14 @@ pub enum BeaconStateError { InsufficientIndexRoots, InsufficientAttestations, InsufficientCommittees, + EpochCacheUninitialized(RelativeEpoch), } #[derive(Debug, PartialEq)] pub enum InclusionError { /// The validator did not participate in an attestation in this period. NoAttestationsForValidator, - AttestationParticipantsError(AttestationParticipantsError), -} - -#[derive(Debug, PartialEq)] -pub enum AttestationParticipantsError { - /// There is no committee for the given shard in the given epoch. - NoCommitteeForShard, - BeaconStateError(BeaconStateError), + Error(Error), } macro_rules! safe_add_assign { @@ -89,6 +106,10 @@ pub struct BeaconState { // Ethereum 1.0 chain data pub latest_eth1_data: Eth1Data, pub eth1_data_votes: Vec, + + // Caching + pub cache_index_offset: usize, + pub caches: Vec, } impl BeaconState { @@ -98,7 +119,7 @@ impl BeaconState { initial_validator_deposits: Vec, latest_eth1_data: Eth1Data, spec: &ChainSpec, - ) -> Result { + ) -> Result { let initial_crosslink = Crosslink { epoch: spec.genesis_epoch, shard_block_root: spec.zero_hash, @@ -157,6 +178,12 @@ impl BeaconState { */ latest_eth1_data, eth1_data_votes: vec![], + + /* + * Caching (not in spec) + */ + cache_index_offset: 0, + caches: vec![EpochCache::empty(); CACHED_EPOCHS], }; for deposit in initial_validator_deposits { @@ -187,6 +214,81 @@ impl BeaconState { Ok(genesis_state) } + /// Build an epoch cache, unless it is has already been built. + pub fn build_epoch_cache( + &mut self, + relative_epoch: RelativeEpoch, + spec: &ChainSpec, + ) -> Result<(), Error> { + let cache_index = self.cache_index(relative_epoch); + + if self.caches[cache_index].initialized == false { + self.force_build_epoch_cache(relative_epoch, spec) + } else { + Ok(()) + } + } + + /// Always builds an epoch cache, even if it is alread initialized. + pub fn force_build_epoch_cache( + &mut self, + relative_epoch: RelativeEpoch, + spec: &ChainSpec, + ) -> Result<(), Error> { + let epoch = self.absolute_epoch(relative_epoch, spec); + let cache_index = self.cache_index(relative_epoch); + + self.caches[cache_index] = EpochCache::initialized(&self, epoch, spec)?; + + Ok(()) + } + + fn absolute_epoch(&self, relative_epoch: RelativeEpoch, spec: &ChainSpec) -> Epoch { + match relative_epoch { + RelativeEpoch::Previous => self.previous_epoch(spec), + RelativeEpoch::Current => self.current_epoch(spec), + RelativeEpoch::Next => self.next_epoch(spec), + } + } + + fn relative_epoch(&self, epoch: Epoch, spec: &ChainSpec) -> Result { + match epoch { + e if e == self.current_epoch(spec) => Ok(RelativeEpoch::Current), + e if e == self.previous_epoch(spec) => Ok(RelativeEpoch::Previous), + e if e == self.next_epoch(spec) => Ok(RelativeEpoch::Next), + _ => Err(Error::EpochOutOfBounds), + } + } + + pub fn advance_caches(&mut self) { + let previous_cache_index = self.cache_index(RelativeEpoch::Previous); + + self.caches[previous_cache_index] = EpochCache::empty(); + + self.cache_index_offset += 1; + self.cache_index_offset %= CACHED_EPOCHS; + } + + fn cache_index(&self, relative_epoch: RelativeEpoch) -> usize { + let base_index = match relative_epoch { + RelativeEpoch::Current => 1, + RelativeEpoch::Previous => 0, + RelativeEpoch::Next => 2, + }; + + (base_index + self.cache_index_offset) % CACHED_EPOCHS + } + + fn cache<'a>(&'a self, relative_epoch: RelativeEpoch) -> Result<&'a EpochCache, Error> { + let cache = &self.caches[self.cache_index(relative_epoch)]; + + if cache.initialized == false { + Err(Error::EpochCacheUninitialized(relative_epoch)) + } else { + Ok(cache) + } + } + /// Return the tree hash root for this `BeaconState`. /// /// Spec v0.2.0 @@ -258,7 +360,7 @@ impl BeaconState { /// committee is itself a list of validator indices. /// /// Spec v0.1 - pub fn get_shuffling( + pub(crate) fn get_shuffling( &self, seed: Hash256, epoch: Epoch, @@ -271,11 +373,6 @@ impl BeaconState { return None; } - trace!( - "get_shuffling: active_validator_indices.len() == {}", - active_validator_indices.len() - ); - let committees_per_epoch = self.get_epoch_committee_count(active_validator_indices.len(), spec); @@ -339,17 +436,9 @@ impl BeaconState { + 1; let latest_index_root = current_epoch + spec.entry_exit_delay; - trace!( - "get_active_index_root: epoch: {}, earliest: {}, latest: {}", - epoch, - earliest_index_root, - latest_index_root - ); - if (epoch >= earliest_index_root) & (epoch <= latest_index_root) { Some(self.latest_index_roots[epoch.as_usize() % spec.latest_index_roots_length]) } else { - trace!("get_active_index_root: epoch out of range."); None } } @@ -357,20 +446,16 @@ impl BeaconState { /// Generate a seed for the given ``epoch``. /// /// Spec v0.2.0 - pub fn generate_seed( - &self, - epoch: Epoch, - spec: &ChainSpec, - ) -> Result { + pub fn generate_seed(&self, epoch: Epoch, spec: &ChainSpec) -> Result { let mut input = self .get_randao_mix(epoch, spec) - .ok_or_else(|| BeaconStateError::InsufficientRandaoMixes)? + .ok_or_else(|| Error::InsufficientRandaoMixes)? .to_vec(); input.append( &mut self .get_active_index_root(epoch, spec) - .ok_or_else(|| BeaconStateError::InsufficientIndexRoots)? + .ok_or_else(|| Error::InsufficientIndexRoots)? .to_vec(), ); @@ -380,18 +465,34 @@ impl BeaconState { Ok(Hash256::from(&hash(&input[..])[..])) } + pub fn get_crosslink_committees_at_slot( + &self, + slot: Slot, + spec: &ChainSpec, + ) -> Result<&CrosslinkCommittees, Error> { + let epoch = slot.epoch(spec.epoch_length); + let relative_epoch = self.relative_epoch(epoch, spec)?; + let cache = self.cache(relative_epoch)?; + + let slot_offset = slot - epoch.start_slot(spec.epoch_length); + + Ok(&cache.committees[slot_offset.as_usize()]) + } + /// Return the list of ``(committee, shard)`` tuples for the ``slot``. /// /// Note: There are two possible shufflings for crosslink committees for a /// `slot` in the next epoch: with and without a `registry_change` /// + /// Note: this is equivalent to the `get_crosslink_committees_at_slot` function in the spec. + /// /// Spec v0.2.0 - pub fn get_crosslink_committees_at_slot( + pub(crate) fn calculate_crosslink_committees_at_slot( &self, slot: Slot, registry_change: bool, spec: &ChainSpec, - ) -> Result, u64)>, BeaconStateError> { + ) -> Result, u64)>, Error> { let epoch = slot.epoch(spec.epoch_length); let current_epoch = self.current_epoch(spec); let previous_epoch = self.previous_epoch(spec); @@ -441,24 +542,17 @@ impl BeaconState { shuffling_start_shard, ) } else { - return Err(BeaconStateError::EpochOutOfBounds); + return Err(Error::EpochOutOfBounds); }; let shuffling = self .get_shuffling(seed, shuffling_epoch, spec) - .ok_or_else(|| BeaconStateError::UnableToShuffle)?; + .ok_or_else(|| Error::UnableToShuffle)?; let offset = slot.as_u64() % spec.epoch_length; let committees_per_slot = committees_per_epoch / spec.epoch_length; let slot_start_shard = (shuffling_start_shard + committees_per_slot * offset) % spec.shard_count; - trace!( - "get_crosslink_committees_at_slot: committees_per_slot: {}, slot_start_shard: {}, seed: {}", - committees_per_slot, - slot_start_shard, - seed - ); - let mut crosslinks_at_slot = vec![]; for i in 0..committees_per_slot { let tuple = ( @@ -473,22 +567,20 @@ impl BeaconState { /// Returns the `slot`, `shard` and `committee_index` for which a validator must produce an /// attestation. /// + /// Only reads the current epoch. + /// /// Spec v0.2.0 pub fn attestation_slot_and_shard_for_validator( &self, validator_index: usize, - spec: &ChainSpec, - ) -> Result, BeaconStateError> { - let mut result = None; - for slot in self.current_epoch(spec).slot_iter(spec.epoch_length) { - for (committee, shard) in self.get_crosslink_committees_at_slot(slot, false, spec)? { - if let Some(committee_index) = committee.iter().position(|&i| i == validator_index) - { - result = Some((slot, shard, committee_index as u64)); - } - } - } - Ok(result) + _spec: &ChainSpec, + ) -> Result, Error> { + let cache = self.cache(RelativeEpoch::Current)?; + + Ok(cache + .attestation_duty_map + .get(&(validator_index as u64)) + .and_then(|tuple| Some(*tuple))) } /// An entry or exit triggered in the ``epoch`` given by the input takes effect at @@ -504,12 +596,8 @@ impl BeaconState { /// If the state does not contain an index for a beacon proposer at the requested `slot`, then `None` is returned. /// /// Spec v0.2.0 - pub fn get_beacon_proposer_index( - &self, - slot: Slot, - spec: &ChainSpec, - ) -> Result { - let committees = self.get_crosslink_committees_at_slot(slot, false, spec)?; + pub fn get_beacon_proposer_index(&self, slot: Slot, spec: &ChainSpec) -> Result { + let committees = self.get_crosslink_committees_at_slot(slot, spec)?; trace!( "get_beacon_proposer_index: slot: {}, committees_count: {}", slot, @@ -517,11 +605,12 @@ impl BeaconState { ); committees .first() - .ok_or(BeaconStateError::InsufficientValidators) + .ok_or(Error::InsufficientValidators) .and_then(|(first_committee, _)| { - let index = (slot.as_usize()) + let index = slot + .as_usize() .checked_rem(first_committee.len()) - .ok_or(BeaconStateError::InsufficientValidators)?; + .ok_or(Error::InsufficientValidators)?; Ok(first_committee[index]) }) } @@ -730,7 +819,7 @@ impl BeaconState { &mut self, validator_index: usize, spec: &ChainSpec, - ) -> Result<(), BeaconStateError> { + ) -> Result<(), Error> { self.exit_validator(validator_index, spec); let current_epoch = self.current_epoch(spec); @@ -899,54 +988,47 @@ impl BeaconState { &self, attestations: &[&PendingAttestation], spec: &ChainSpec, - ) -> Result, AttestationParticipantsError> { - let mut all_participants = attestations.iter().try_fold::<_, _, Result< - Vec, - AttestationParticipantsError, - >>(vec![], |mut acc, a| { - acc.append(&mut self.get_attestation_participants( - &a.data, - &a.aggregation_bitfield, - spec, - )?); - Ok(acc) - })?; + ) -> Result, Error> { + let mut all_participants = attestations + .iter() + .try_fold::<_, _, Result, Error>>(vec![], |mut acc, a| { + acc.append(&mut self.get_attestation_participants( + &a.data, + &a.aggregation_bitfield, + spec, + )?); + Ok(acc) + })?; all_participants.sort_unstable(); all_participants.dedup(); Ok(all_participants) } - /// Return the participant indices at for the ``attestation_data`` and ``bitfield``. - /// - /// In effect, this converts the "committee indices" on the bitfield into "validator indices" - /// for self.validator_registy. - /// - /// Spec v0.2.0 pub fn get_attestation_participants( &self, attestation_data: &AttestationData, bitfield: &Bitfield, spec: &ChainSpec, - ) -> Result, AttestationParticipantsError> { - let crosslink_committees = - self.get_crosslink_committees_at_slot(attestation_data.slot, false, spec)?; + ) -> Result, Error> { + let epoch = attestation_data.slot.epoch(spec.epoch_length); + let relative_epoch = self.relative_epoch(epoch, spec)?; + let cache = self.cache(relative_epoch)?; - let committee_index: usize = crosslink_committees - .iter() - .position(|(_committee, shard)| *shard == attestation_data.shard) - .ok_or_else(|| AttestationParticipantsError::NoCommitteeForShard)?; - let (crosslink_committee, _shard) = &crosslink_committees[committee_index]; + let (committee_slot_index, committee_index) = cache + .shard_committee_index_map + .get(&attestation_data.shard) + .ok_or_else(|| Error::ShardOutOfBounds)?; + let (committee, shard) = &cache.committees[*committee_slot_index][*committee_index]; - /* - * TODO: verify bitfield length is valid. - */ + assert_eq!(*shard, attestation_data.shard, "Bad epoch cache build."); let mut participants = vec![]; - for (i, validator_index) in crosslink_committee.iter().enumerate() { + for (i, validator_index) in committee.iter().enumerate() { if bitfield.get(i).unwrap() { participants.push(*validator_index); } } + Ok(participants) } } @@ -955,15 +1037,9 @@ fn hash_tree_root(input: Vec) -> Hash256 { Hash256::from(&input.hash_tree_root()[..]) } -impl From for AttestationParticipantsError { - fn from(e: BeaconStateError) -> AttestationParticipantsError { - AttestationParticipantsError::BeaconStateError(e) - } -} - -impl From for InclusionError { - fn from(e: AttestationParticipantsError) -> InclusionError { - InclusionError::AttestationParticipantsError(e) +impl From for InclusionError { + fn from(e: Error) -> InclusionError { + InclusionError::Error(e) } } @@ -1052,6 +1128,8 @@ impl Decodable for BeaconState { batched_block_roots, latest_eth1_data, eth1_data_votes, + cache_index_offset: 0, + caches: vec![EpochCache::empty(); CACHED_EPOCHS], }, i, )) @@ -1122,6 +1200,8 @@ impl TestRandom for BeaconState { batched_block_roots: <_>::random_for_test(rng), latest_eth1_data: <_>::random_for_test(rng), eth1_data_votes: <_>::random_for_test(rng), + cache_index_offset: 0, + caches: vec![EpochCache::empty(); CACHED_EPOCHS], } } } diff --git a/eth2/types/src/beacon_state/epoch_cache.rs b/eth2/types/src/beacon_state/epoch_cache.rs new file mode 100644 index 000000000..0653aaa3d --- /dev/null +++ b/eth2/types/src/beacon_state/epoch_cache.rs @@ -0,0 +1,77 @@ +use super::{AttestationDutyMap, BeaconState, CrosslinkCommittees, Error, ShardCommitteeIndexMap}; +use crate::{ChainSpec, Epoch}; +use log::trace; +use serde_derive::Serialize; +use std::collections::HashMap; + +#[derive(Debug, PartialEq, Clone, Serialize)] +pub struct EpochCache { + /// True if this cache has been initialized. + pub initialized: bool, + /// The crosslink committees for an epoch. + pub committees: Vec, + /// Maps validator index to a slot, shard and committee index for attestation. + pub attestation_duty_map: AttestationDutyMap, + /// Maps a shard to an index of `self.committees`. + pub shard_committee_index_map: ShardCommitteeIndexMap, +} + +impl EpochCache { + pub fn empty() -> EpochCache { + EpochCache { + initialized: false, + committees: vec![], + attestation_duty_map: AttestationDutyMap::new(), + shard_committee_index_map: ShardCommitteeIndexMap::new(), + } + } + + pub fn initialized( + state: &BeaconState, + epoch: Epoch, + spec: &ChainSpec, + ) -> Result { + let mut epoch_committees: Vec = + Vec::with_capacity(spec.epoch_length as usize); + let mut attestation_duty_map: AttestationDutyMap = HashMap::new(); + let mut shard_committee_index_map: ShardCommitteeIndexMap = HashMap::new(); + + for (epoch_committeess_index, slot) in epoch.slot_iter(spec.epoch_length).enumerate() { + let slot_committees = + state.calculate_crosslink_committees_at_slot(slot, false, spec)?; + + for (slot_committees_index, (committee, shard)) in slot_committees.iter().enumerate() { + // Empty committees are not permitted. + if committee.is_empty() { + return Err(Error::InsufficientValidators); + } + + trace!( + "shard: {}, epoch_i: {}, slot_i: {}", + shard, + epoch_committeess_index, + slot_committees_index + ); + + shard_committee_index_map + .insert(*shard, (epoch_committeess_index, slot_committees_index)); + + for (committee_index, validator_index) in committee.iter().enumerate() { + attestation_duty_map.insert( + *validator_index as u64, + (slot, *shard, committee_index as u64), + ); + } + } + + epoch_committees.push(slot_committees) + } + + Ok(EpochCache { + initialized: true, + committees: epoch_committees, + attestation_duty_map, + shard_committee_index_map, + }) + } +} diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs index 2b7c5b539..d503d8c6a 100644 --- a/eth2/types/src/beacon_state/tests.rs +++ b/eth2/types/src/beacon_state/tests.rs @@ -3,8 +3,8 @@ use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; use crate::{ - beacon_state::BeaconStateError, BeaconState, ChainSpec, Deposit, DepositData, DepositInput, - Eth1Data, Hash256, Keypair, + BeaconState, BeaconStateError, ChainSpec, Deposit, DepositData, DepositInput, Eth1Data, + Hash256, Keypair, }; use bls::create_proof_of_possession; use ssz::ssz_encode; @@ -73,6 +73,53 @@ pub fn can_produce_genesis_block() { builder.build().unwrap(); } +/// Tests that `get_attestation_participants` is consistent with the result of +/// get_crosslink_committees_at_slot` with a full bitfield. +#[test] +pub fn get_attestation_participants_consistency() { + let mut rng = XorShiftRng::from_seed([42; 16]); + + let mut builder = BeaconStateTestBuilder::with_random_validators(8); + builder.spec = ChainSpec::few_validators(); + + let mut state = builder.build().unwrap(); + let spec = builder.spec.clone(); + + state + .build_epoch_cache(RelativeEpoch::Previous, &spec) + .unwrap(); + state + .build_epoch_cache(RelativeEpoch::Current, &spec) + .unwrap(); + state.build_epoch_cache(RelativeEpoch::Next, &spec).unwrap(); + + for slot in state + .slot + .epoch(spec.epoch_length) + .slot_iter(spec.epoch_length) + { + let committees = state.get_crosslink_committees_at_slot(slot, &spec).unwrap(); + + for (committee, shard) in committees { + let mut attestation_data = AttestationData::random_for_test(&mut rng); + attestation_data.slot = slot; + attestation_data.shard = *shard; + + let mut bitfield = Bitfield::new(); + for (i, _) in committee.iter().enumerate() { + bitfield.set(i, true); + } + + assert_eq!( + state + .get_attestation_participants(&attestation_data, &bitfield, &spec) + .unwrap(), + *committee + ); + } + } +} + #[test] pub fn test_ssz_round_trip() { let mut rng = XorShiftRng::from_seed([42; 16]); diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index f2c128440..4f196b9e9 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -42,7 +42,9 @@ pub use crate::attestation_data_and_custody_bit::AttestationDataAndCustodyBit; pub use crate::attester_slashing::AttesterSlashing; pub use crate::beacon_block::BeaconBlock; pub use crate::beacon_block_body::BeaconBlockBody; -pub use crate::beacon_state::BeaconState; +pub use crate::beacon_state::{ + BeaconState, Error as BeaconStateError, InclusionError, RelativeEpoch, +}; pub use crate::casper_slashing::CasperSlashing; pub use crate::chain_spec::ChainSpec; pub use crate::crosslink::Crosslink; From 89ab0f683e08f54cea033c78bbb3b6ebe7a2424d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 22 Feb 2019 18:19:47 +1300 Subject: [PATCH 007/132] Change "few_validators" spec to be 8 shards. A bug arises when the number of shards is less than the slots in an epoch. --- eth2/types/src/chain_spec.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs index b5d5689e3..706ad417a 100644 --- a/eth2/types/src/chain_spec.rs +++ b/eth2/types/src/chain_spec.rs @@ -199,7 +199,7 @@ impl ChainSpec { let genesis_epoch = genesis_slot.epoch(epoch_length); Self { - shard_count: 1, + shard_count: 8, target_committee_size: 1, genesis_slot, genesis_epoch, From 7a28893bab2fbe13f6145497a9633406837f98bf Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 22 Feb 2019 18:48:35 +1300 Subject: [PATCH 008/132] Fix bug in Epoch.slot_iter() It wasn't running the whole range, plus it could get into a loop when used near the u64::max_value --- eth2/types/src/slot_epoch.rs | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/eth2/types/src/slot_epoch.rs b/eth2/types/src/slot_epoch.rs index eb5a8dced..ff4fd5b9b 100644 --- a/eth2/types/src/slot_epoch.rs +++ b/eth2/types/src/slot_epoch.rs @@ -72,7 +72,7 @@ impl Epoch { pub fn slot_iter(&self, epoch_length: u64) -> SlotIter { SlotIter { - current: self.start_slot(epoch_length), + current_iteration: 0, epoch: self, epoch_length, } @@ -80,7 +80,7 @@ impl Epoch { } pub struct SlotIter<'a> { - current: Slot, + current_iteration: u64, epoch: &'a Epoch, epoch_length: u64, } @@ -89,12 +89,13 @@ impl<'a> Iterator for SlotIter<'a> { type Item = Slot; fn next(&mut self) -> Option { - if self.current == self.epoch.end_slot(self.epoch_length) { + if self.current_iteration >= self.epoch_length { None } else { - let previous = self.current; - self.current += 1; - Some(previous) + let start_slot = self.epoch.start_slot(self.epoch_length); + let previous = self.current_iteration; + self.current_iteration += 1; + Some(start_slot + previous) } } } @@ -115,4 +116,22 @@ mod epoch_tests { use ssz::ssz_encode; all_tests!(Epoch); + + #[test] + fn slot_iter() { + let epoch_length = 8; + + let epoch = Epoch::new(0); + + let mut slots = vec![]; + for slot in epoch.slot_iter(epoch_length) { + slots.push(slot); + } + + assert_eq!(slots.len(), epoch_length as usize); + + for i in 0..epoch_length { + assert_eq!(Slot::from(i), slots[i as usize]) + } + } } From 5f3ba42b97a2b641f1979eb871b1eb20dc938722 Mon Sep 17 00:00:00 2001 From: thojest Date: Fri, 22 Feb 2019 12:16:11 +0100 Subject: [PATCH 009/132] added first draft for lib-crates for test_random and test_random_derive (lighthouse-246) --- Cargo.toml | 2 ++ eth2/utils/test_random/Cargo.toml | 8 +++++ eth2/utils/test_random/src/lib.rs | 39 ++++++++++++++++++++++++ eth2/utils/test_random_derive/Cargo.toml | 15 +++++++++ eth2/utils/test_random_derive/src/lib.rs | 15 +++++++++ 5 files changed, 79 insertions(+) create mode 100644 eth2/utils/test_random/Cargo.toml create mode 100644 eth2/utils/test_random/src/lib.rs create mode 100644 eth2/utils/test_random_derive/Cargo.toml create mode 100644 eth2/utils/test_random_derive/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index d92b1a303..a557339a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,8 @@ members = [ "eth2/utils/ssz_derive", "eth2/utils/swap_or_not_shuffle", "eth2/utils/fisher_yates_shuffle", + "eth2/utils/test_random", + "eth2/utils/test_random_derive", "beacon_node", "beacon_node/db", "beacon_node/beacon_chain", diff --git a/eth2/utils/test_random/Cargo.toml b/eth2/utils/test_random/Cargo.toml new file mode 100644 index 000000000..6346aa628 --- /dev/null +++ b/eth2/utils/test_random/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "test_random" +version = "0.1.0" +authors = ["thojest "] +edition = "2018" + +[dependencies] +rand = "0.5.5" \ No newline at end of file diff --git a/eth2/utils/test_random/src/lib.rs b/eth2/utils/test_random/src/lib.rs new file mode 100644 index 000000000..aa0d38a5a --- /dev/null +++ b/eth2/utils/test_random/src/lib.rs @@ -0,0 +1,39 @@ +use rand::RngCore; + +pub trait TestRandom +where + T: RngCore, +{ + fn random_for_test(rng: &mut T) -> Self; +} + +impl TestRandom for u64 { + fn random_for_test(rng: &mut T) -> Self { + rng.next_u64() + } +} + +impl TestRandom for u32 { + fn random_for_test(rng: &mut T) -> Self { + rng.next_u32() + } +} + +impl TestRandom for usize { + fn random_for_test(rng: &mut T) -> Self { + rng.next_u32() as usize + } +} + +impl TestRandom for Vec +where + U: TestRandom, +{ + fn random_for_test(rng: &mut T) -> Self { + vec![ + ::random_for_test(rng), + ::random_for_test(rng), + ::random_for_test(rng), + ] + } +} diff --git a/eth2/utils/test_random_derive/Cargo.toml b/eth2/utils/test_random_derive/Cargo.toml new file mode 100644 index 000000000..e596ad29c --- /dev/null +++ b/eth2/utils/test_random_derive/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "test_random_derive" +version = "0.1.0" +authors = ["thojest "] +edition = "2018" +description = "Procedural derive macros for implementation of TestRandom trait" + +[lib] +proc-macro = true + +[dependencies] +syn = "0.15" +quote = "0.6" +test_random = {path = "../test_random"} + diff --git a/eth2/utils/test_random_derive/src/lib.rs b/eth2/utils/test_random_derive/src/lib.rs new file mode 100644 index 000000000..27e62d31a --- /dev/null +++ b/eth2/utils/test_random_derive/src/lib.rs @@ -0,0 +1,15 @@ +extern crate proc_macro; + +use crate::proc_macro::TokenStream; +use quote::quote; +use syn; +use syn::DeriveInput; + +#[proc_macro_derive(TestRandom)] +pub fn test_random_derive(input: TokenStream) -> TokenStream { + let ast = syn::parse(input).unwrap(); + + impl_test_random(&ast) +} + +fn impl_test_random(ast: &DeriveInput) -> TokenStream {} From 7a382043e173b5ae9a7cc8e6f12323790b8e3089 Mon Sep 17 00:00:00 2001 From: thojest Date: Fri, 22 Feb 2019 15:54:18 +0100 Subject: [PATCH 010/132] added test_random_derive implementation (lighthouse-246) --- eth2/utils/test_random_derive/src/lib.rs | 38 ++++++++++++++++++++---- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/eth2/utils/test_random_derive/src/lib.rs b/eth2/utils/test_random_derive/src/lib.rs index 27e62d31a..2c244e448 100644 --- a/eth2/utils/test_random_derive/src/lib.rs +++ b/eth2/utils/test_random_derive/src/lib.rs @@ -2,14 +2,42 @@ extern crate proc_macro; use crate::proc_macro::TokenStream; use quote::quote; -use syn; -use syn::DeriveInput; +use syn::{parse_macro_input, DeriveInput}; #[proc_macro_derive(TestRandom)] pub fn test_random_derive(input: TokenStream) -> TokenStream { - let ast = syn::parse(input).unwrap(); + let ast = parse_macro_input!(input as DeriveInput); + let name = &ast.ident; - impl_test_random(&ast) + let struct_data = match &ast.data { + syn::Data::Struct(s) => s, + _ => panic!("test_random_derive only supports structs."), + }; + + let field_names = get_named_field_idents_and_types(&struct_data); + + let output = quote! { + impl TestRandom for #name { + fn random_for_test(rng: &mut T) -> Self { + Self { + #( + #field_names: <_>::random_for_test(rng) + )* + } + } + } + }; + + output.into() } -fn impl_test_random(ast: &DeriveInput) -> TokenStream {} +fn get_named_field_idents_and_types(struct_data: &syn::DataStruct) -> Vec<(&syn::Ident)> { + struct_data + .fields + .iter() + .map(|f| match &f.ident { + Some(ref ident) => ident, + _ => panic!("test_random_derive only supports named struct fields."), + }) + .collect() +} From 278b41c8efff564850131cd5a58815b042095a0d Mon Sep 17 00:00:00 2001 From: thojest Date: Fri, 22 Feb 2019 16:05:52 +0100 Subject: [PATCH 011/132] decided against moving test_utils (lighthouse-246) --- Cargo.toml | 1 - eth2/utils/test_random/Cargo.toml | 8 ----- eth2/utils/test_random/src/lib.rs | 39 ------------------------ eth2/utils/test_random_derive/Cargo.toml | 2 -- 4 files changed, 50 deletions(-) delete mode 100644 eth2/utils/test_random/Cargo.toml delete mode 100644 eth2/utils/test_random/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index a557339a5..42d69489b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,6 @@ members = [ "eth2/utils/ssz_derive", "eth2/utils/swap_or_not_shuffle", "eth2/utils/fisher_yates_shuffle", - "eth2/utils/test_random", "eth2/utils/test_random_derive", "beacon_node", "beacon_node/db", diff --git a/eth2/utils/test_random/Cargo.toml b/eth2/utils/test_random/Cargo.toml deleted file mode 100644 index 6346aa628..000000000 --- a/eth2/utils/test_random/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "test_random" -version = "0.1.0" -authors = ["thojest "] -edition = "2018" - -[dependencies] -rand = "0.5.5" \ No newline at end of file diff --git a/eth2/utils/test_random/src/lib.rs b/eth2/utils/test_random/src/lib.rs deleted file mode 100644 index aa0d38a5a..000000000 --- a/eth2/utils/test_random/src/lib.rs +++ /dev/null @@ -1,39 +0,0 @@ -use rand::RngCore; - -pub trait TestRandom -where - T: RngCore, -{ - fn random_for_test(rng: &mut T) -> Self; -} - -impl TestRandom for u64 { - fn random_for_test(rng: &mut T) -> Self { - rng.next_u64() - } -} - -impl TestRandom for u32 { - fn random_for_test(rng: &mut T) -> Self { - rng.next_u32() - } -} - -impl TestRandom for usize { - fn random_for_test(rng: &mut T) -> Self { - rng.next_u32() as usize - } -} - -impl TestRandom for Vec -where - U: TestRandom, -{ - fn random_for_test(rng: &mut T) -> Self { - vec![ - ::random_for_test(rng), - ::random_for_test(rng), - ::random_for_test(rng), - ] - } -} diff --git a/eth2/utils/test_random_derive/Cargo.toml b/eth2/utils/test_random_derive/Cargo.toml index e596ad29c..4559befaf 100644 --- a/eth2/utils/test_random_derive/Cargo.toml +++ b/eth2/utils/test_random_derive/Cargo.toml @@ -11,5 +11,3 @@ proc-macro = true [dependencies] syn = "0.15" quote = "0.6" -test_random = {path = "../test_random"} - From 66b5accdc2a8369d804248670fc81a09ba2f7582 Mon Sep 17 00:00:00 2001 From: thojest Date: Fri, 22 Feb 2019 17:10:26 +0100 Subject: [PATCH 012/132] replaced manual TestRandom implementation with macro when possible; fixed typo in TestRandom macro (lighthouse-246) --- eth2/types/Cargo.toml | 1 + eth2/types/src/attestation.rs | 14 ++------ eth2/types/src/attestation_data.rs | 18 ++-------- eth2/types/src/attester_slashing.rs | 12 ++----- eth2/types/src/beacon_block.rs | 17 ++------- eth2/types/src/beacon_block_body.rs | 15 ++------ eth2/types/src/beacon_state.rs | 35 ++----------------- eth2/types/src/casper_slashing.rs | 12 ++----- eth2/types/src/crosslink.rs | 12 ++----- eth2/types/src/deposit.rs | 13 ++----- eth2/types/src/deposit_data.rs | 13 ++----- eth2/types/src/deposit_input.rs | 13 ++----- eth2/types/src/eth1_data.rs | 12 ++----- eth2/types/src/eth1_data_vote.rs | 12 ++----- eth2/types/src/exit.rs | 13 ++----- eth2/types/src/fork.rs | 13 ++----- eth2/types/src/pending_attestation.rs | 14 ++------ eth2/types/src/proposal_signed_data.rs | 13 ++----- eth2/types/src/proposer_slashing.rs | 15 ++------ eth2/types/src/shard_reassignment_record.rs | 13 ++----- eth2/types/src/slashable_attestation.rs | 14 ++------ eth2/types/src/slashable_vote_data.rs | 14 ++------ .../src/validator_registry_delta_block.rs | 15 ++------ eth2/utils/test_random_derive/src/lib.rs | 12 +++---- 24 files changed, 51 insertions(+), 284 deletions(-) diff --git a/eth2/types/Cargo.toml b/eth2/types/Cargo.toml index f51e20236..f70e8b490 100644 --- a/eth2/types/Cargo.toml +++ b/eth2/types/Cargo.toml @@ -20,6 +20,7 @@ slog = "^2.2.3" ssz = { path = "../utils/ssz" } ssz_derive = { path = "../utils/ssz_derive" } swap_or_not_shuffle = { path = "../utils/swap_or_not_shuffle" } +test_random_derive = { path = "../utils/test_random_derive" } [dev-dependencies] env_logger = "0.6.0" diff --git a/eth2/types/src/attestation.rs b/eth2/types/src/attestation.rs index 7388a8e49..ee573dcc0 100644 --- a/eth2/types/src/attestation.rs +++ b/eth2/types/src/attestation.rs @@ -4,8 +4,9 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; -#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode)] +#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, TestRandom)] pub struct Attestation { pub aggregation_bitfield: Bitfield, pub data: AttestationData, @@ -45,17 +46,6 @@ impl TreeHash for Attestation { } } -impl TestRandom for Attestation { - fn random_for_test(rng: &mut T) -> Self { - Self { - data: <_>::random_for_test(rng), - aggregation_bitfield: <_>::random_for_test(rng), - custody_bitfield: <_>::random_for_test(rng), - aggregate_signature: <_>::random_for_test(rng), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data.rs index 7edb0b72b..ec67e30b6 100644 --- a/eth2/types/src/attestation_data.rs +++ b/eth2/types/src/attestation_data.rs @@ -4,6 +4,7 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; pub const SSZ_ATTESTION_DATA_LENGTH: usize = { 8 + // slot @@ -16,7 +17,7 @@ pub const SSZ_ATTESTION_DATA_LENGTH: usize = { 32 // justified_block_root }; -#[derive(Debug, Clone, PartialEq, Default, Serialize, Hash, Encode, Decode)] +#[derive(Debug, Clone, PartialEq, Default, Serialize, Hash, Encode, Decode, TestRandom)] pub struct AttestationData { pub slot: Slot, pub shard: u64, @@ -59,21 +60,6 @@ impl TreeHash for AttestationData { } } -impl TestRandom for AttestationData { - fn random_for_test(rng: &mut T) -> Self { - Self { - slot: <_>::random_for_test(rng), - shard: <_>::random_for_test(rng), - beacon_block_root: <_>::random_for_test(rng), - epoch_boundary_root: <_>::random_for_test(rng), - shard_block_root: <_>::random_for_test(rng), - latest_crosslink: <_>::random_for_test(rng), - justified_epoch: <_>::random_for_test(rng), - justified_block_root: <_>::random_for_test(rng), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/attester_slashing.rs b/eth2/types/src/attester_slashing.rs index f84998324..8ea6b39e8 100644 --- a/eth2/types/src/attester_slashing.rs +++ b/eth2/types/src/attester_slashing.rs @@ -3,8 +3,9 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TestRandom)] pub struct AttesterSlashing { pub slashable_attestation_1: SlashableAttestation, pub slashable_attestation_2: SlashableAttestation, @@ -19,15 +20,6 @@ impl TreeHash for AttesterSlashing { } } -impl TestRandom for AttesterSlashing { - fn random_for_test(rng: &mut T) -> Self { - Self { - slashable_attestation_1: <_>::random_for_test(rng), - slashable_attestation_2: <_>::random_for_test(rng), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/beacon_block.rs b/eth2/types/src/beacon_block.rs index c252d03f7..e5dc9e238 100644 --- a/eth2/types/src/beacon_block.rs +++ b/eth2/types/src/beacon_block.rs @@ -5,8 +5,9 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TestRandom)] pub struct BeaconBlock { pub slot: Slot, pub parent_root: Hash256, @@ -74,20 +75,6 @@ impl TreeHash for BeaconBlock { } } -impl TestRandom for BeaconBlock { - fn random_for_test(rng: &mut T) -> Self { - Self { - slot: <_>::random_for_test(rng), - parent_root: <_>::random_for_test(rng), - state_root: <_>::random_for_test(rng), - randao_reveal: <_>::random_for_test(rng), - eth1_data: <_>::random_for_test(rng), - signature: <_>::random_for_test(rng), - body: <_>::random_for_test(rng), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/beacon_block_body.rs b/eth2/types/src/beacon_block_body.rs index e051f5940..b492ba747 100644 --- a/eth2/types/src/beacon_block_body.rs +++ b/eth2/types/src/beacon_block_body.rs @@ -4,8 +4,9 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; -#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, TestRandom)] pub struct BeaconBlockBody { pub proposer_slashings: Vec, pub attester_slashings: Vec, @@ -26,18 +27,6 @@ impl TreeHash for BeaconBlockBody { } } -impl TestRandom for BeaconBlockBody { - fn random_for_test(rng: &mut T) -> Self { - Self { - proposer_slashings: <_>::random_for_test(rng), - attester_slashings: <_>::random_for_test(rng), - attestations: <_>::random_for_test(rng), - deposits: <_>::random_for_test(rng), - exits: <_>::random_for_test(rng), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 21deb6fe7..df40a966a 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -12,6 +12,7 @@ use serde_derive::Serialize; use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; use swap_or_not_shuffle::get_permutated_index; +use test_random_derive::TestRandom; mod tests; @@ -52,7 +53,7 @@ macro_rules! safe_sub_assign { }; } -#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, TestRandom)] pub struct BeaconState { // Misc pub slot: Slot, @@ -1003,35 +1004,3 @@ impl TreeHash for BeaconState { hash(&result) } } - -impl TestRandom for BeaconState { - fn random_for_test(rng: &mut T) -> Self { - Self { - slot: <_>::random_for_test(rng), - genesis_time: <_>::random_for_test(rng), - fork: <_>::random_for_test(rng), - validator_registry: <_>::random_for_test(rng), - validator_balances: <_>::random_for_test(rng), - validator_registry_update_epoch: <_>::random_for_test(rng), - latest_randao_mixes: <_>::random_for_test(rng), - previous_epoch_start_shard: <_>::random_for_test(rng), - current_epoch_start_shard: <_>::random_for_test(rng), - previous_calculation_epoch: <_>::random_for_test(rng), - current_calculation_epoch: <_>::random_for_test(rng), - previous_epoch_seed: <_>::random_for_test(rng), - current_epoch_seed: <_>::random_for_test(rng), - previous_justified_epoch: <_>::random_for_test(rng), - justified_epoch: <_>::random_for_test(rng), - justification_bitfield: <_>::random_for_test(rng), - finalized_epoch: <_>::random_for_test(rng), - latest_crosslinks: <_>::random_for_test(rng), - latest_block_roots: <_>::random_for_test(rng), - latest_index_roots: <_>::random_for_test(rng), - latest_penalized_balances: <_>::random_for_test(rng), - latest_attestations: <_>::random_for_test(rng), - batched_block_roots: <_>::random_for_test(rng), - latest_eth1_data: <_>::random_for_test(rng), - eth1_data_votes: <_>::random_for_test(rng), - } - } -} diff --git a/eth2/types/src/casper_slashing.rs b/eth2/types/src/casper_slashing.rs index 6346db65c..bfc87e01d 100644 --- a/eth2/types/src/casper_slashing.rs +++ b/eth2/types/src/casper_slashing.rs @@ -4,8 +4,9 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TestRandom)] pub struct CasperSlashing { pub slashable_vote_data_1: SlashableVoteData, pub slashable_vote_data_2: SlashableVoteData, @@ -20,15 +21,6 @@ impl TreeHash for CasperSlashing { } } -impl TestRandom for CasperSlashing { - fn random_for_test(rng: &mut T) -> Self { - Self { - slashable_vote_data_1: <_>::random_for_test(rng), - slashable_vote_data_2: <_>::random_for_test(rng), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/crosslink.rs b/eth2/types/src/crosslink.rs index 19c71f604..8b0d2bc18 100644 --- a/eth2/types/src/crosslink.rs +++ b/eth2/types/src/crosslink.rs @@ -4,8 +4,9 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; -#[derive(Debug, Clone, PartialEq, Default, Serialize, Hash, Encode, Decode)] +#[derive(Debug, Clone, PartialEq, Default, Serialize, Hash, Encode, Decode, TestRandom)] pub struct Crosslink { pub epoch: Epoch, pub shard_block_root: Hash256, @@ -30,15 +31,6 @@ impl TreeHash for Crosslink { } } -impl TestRandom for Crosslink { - fn random_for_test(rng: &mut T) -> Self { - Self { - epoch: <_>::random_for_test(rng), - shard_block_root: <_>::random_for_test(rng), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/deposit.rs b/eth2/types/src/deposit.rs index 78f43532a..2b126b900 100644 --- a/eth2/types/src/deposit.rs +++ b/eth2/types/src/deposit.rs @@ -4,8 +4,9 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TestRandom)] pub struct Deposit { pub branch: Vec, pub index: u64, @@ -22,16 +23,6 @@ impl TreeHash for Deposit { } } -impl TestRandom for Deposit { - fn random_for_test(rng: &mut T) -> Self { - Self { - branch: <_>::random_for_test(rng), - index: <_>::random_for_test(rng), - deposit_data: <_>::random_for_test(rng), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/deposit_data.rs b/eth2/types/src/deposit_data.rs index 8f49deb3c..0cce11cec 100644 --- a/eth2/types/src/deposit_data.rs +++ b/eth2/types/src/deposit_data.rs @@ -4,8 +4,9 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TestRandom)] pub struct DepositData { pub amount: u64, pub timestamp: u64, @@ -22,16 +23,6 @@ impl TreeHash for DepositData { } } -impl TestRandom for DepositData { - fn random_for_test(rng: &mut T) -> Self { - Self { - amount: <_>::random_for_test(rng), - timestamp: <_>::random_for_test(rng), - deposit_input: <_>::random_for_test(rng), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/deposit_input.rs b/eth2/types/src/deposit_input.rs index 7556fc2ca..d14ba68ac 100644 --- a/eth2/types/src/deposit_input.rs +++ b/eth2/types/src/deposit_input.rs @@ -5,8 +5,9 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TestRandom)] pub struct DepositInput { pub pubkey: PublicKey, pub withdrawal_credentials: Hash256, @@ -23,16 +24,6 @@ impl TreeHash for DepositInput { } } -impl TestRandom for DepositInput { - fn random_for_test(rng: &mut T) -> Self { - Self { - pubkey: <_>::random_for_test(rng), - withdrawal_credentials: <_>::random_for_test(rng), - proof_of_possession: <_>::random_for_test(rng), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/eth1_data.rs b/eth2/types/src/eth1_data.rs index b0dc14e7a..e1b968303 100644 --- a/eth2/types/src/eth1_data.rs +++ b/eth2/types/src/eth1_data.rs @@ -4,9 +4,10 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; // Note: this is refer to as DepositRootVote in specs -#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, TestRandom)] pub struct Eth1Data { pub deposit_root: Hash256, pub block_hash: Hash256, @@ -21,15 +22,6 @@ impl TreeHash for Eth1Data { } } -impl TestRandom for Eth1Data { - fn random_for_test(rng: &mut T) -> Self { - Self { - deposit_root: <_>::random_for_test(rng), - block_hash: <_>::random_for_test(rng), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/eth1_data_vote.rs b/eth2/types/src/eth1_data_vote.rs index eda6e6a6a..09d462999 100644 --- a/eth2/types/src/eth1_data_vote.rs +++ b/eth2/types/src/eth1_data_vote.rs @@ -4,9 +4,10 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; // Note: this is refer to as DepositRootVote in specs -#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, TestRandom)] pub struct Eth1DataVote { pub eth1_data: Eth1Data, pub vote_count: u64, @@ -21,15 +22,6 @@ impl TreeHash for Eth1DataVote { } } -impl TestRandom for Eth1DataVote { - fn random_for_test(rng: &mut T) -> Self { - Self { - eth1_data: <_>::random_for_test(rng), - vote_count: <_>::random_for_test(rng), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/exit.rs b/eth2/types/src/exit.rs index 18d743b83..c96319d55 100644 --- a/eth2/types/src/exit.rs +++ b/eth2/types/src/exit.rs @@ -4,8 +4,9 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TestRandom)] pub struct Exit { pub epoch: Epoch, pub validator_index: u64, @@ -22,16 +23,6 @@ impl TreeHash for Exit { } } -impl TestRandom for Exit { - fn random_for_test(rng: &mut T) -> Self { - Self { - epoch: <_>::random_for_test(rng), - validator_index: <_>::random_for_test(rng), - signature: <_>::random_for_test(rng), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs index 85d530e19..555237b57 100644 --- a/eth2/types/src/fork.rs +++ b/eth2/types/src/fork.rs @@ -3,8 +3,9 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; -#[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode)] +#[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode, TestRandom)] pub struct Fork { pub previous_version: u64, pub current_version: u64, @@ -21,16 +22,6 @@ impl TreeHash for Fork { } } -impl TestRandom for Fork { - fn random_for_test(rng: &mut T) -> Self { - Self { - previous_version: <_>::random_for_test(rng), - current_version: <_>::random_for_test(rng), - epoch: <_>::random_for_test(rng), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/pending_attestation.rs b/eth2/types/src/pending_attestation.rs index 42f990210..3bf6ff1ad 100644 --- a/eth2/types/src/pending_attestation.rs +++ b/eth2/types/src/pending_attestation.rs @@ -4,8 +4,9 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; -#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode)] +#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, TestRandom)] pub struct PendingAttestation { pub aggregation_bitfield: Bitfield, pub data: AttestationData, @@ -24,17 +25,6 @@ impl TreeHash for PendingAttestation { } } -impl TestRandom for PendingAttestation { - fn random_for_test(rng: &mut T) -> Self { - Self { - data: <_>::random_for_test(rng), - aggregation_bitfield: <_>::random_for_test(rng), - custody_bitfield: <_>::random_for_test(rng), - inclusion_slot: <_>::random_for_test(rng), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/proposal_signed_data.rs b/eth2/types/src/proposal_signed_data.rs index 63c0f1ce6..f3f369b7f 100644 --- a/eth2/types/src/proposal_signed_data.rs +++ b/eth2/types/src/proposal_signed_data.rs @@ -4,8 +4,9 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; -#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, TestRandom)] pub struct ProposalSignedData { pub slot: Slot, pub shard: u64, @@ -22,16 +23,6 @@ impl TreeHash for ProposalSignedData { } } -impl TestRandom for ProposalSignedData { - fn random_for_test(rng: &mut T) -> Self { - Self { - slot: <_>::random_for_test(rng), - shard: <_>::random_for_test(rng), - block_root: <_>::random_for_test(rng), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/proposer_slashing.rs b/eth2/types/src/proposer_slashing.rs index b3a819a7f..08ae27abf 100644 --- a/eth2/types/src/proposer_slashing.rs +++ b/eth2/types/src/proposer_slashing.rs @@ -5,8 +5,9 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TestRandom)] pub struct ProposerSlashing { pub proposer_index: u64, pub proposal_data_1: ProposalSignedData, @@ -27,18 +28,6 @@ impl TreeHash for ProposerSlashing { } } -impl TestRandom for ProposerSlashing { - fn random_for_test(rng: &mut T) -> Self { - Self { - proposer_index: <_>::random_for_test(rng), - proposal_data_1: <_>::random_for_test(rng), - proposal_signature_1: <_>::random_for_test(rng), - proposal_data_2: <_>::random_for_test(rng), - proposal_signature_2: <_>::random_for_test(rng), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/shard_reassignment_record.rs b/eth2/types/src/shard_reassignment_record.rs index 511fe13ca..eb67a10c2 100644 --- a/eth2/types/src/shard_reassignment_record.rs +++ b/eth2/types/src/shard_reassignment_record.rs @@ -3,8 +3,9 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TestRandom)] pub struct ShardReassignmentRecord { pub validator_index: u64, pub shard: u64, @@ -21,16 +22,6 @@ impl TreeHash for ShardReassignmentRecord { } } -impl TestRandom for ShardReassignmentRecord { - fn random_for_test(rng: &mut T) -> Self { - Self { - validator_index: <_>::random_for_test(rng), - shard: <_>::random_for_test(rng), - slot: <_>::random_for_test(rng), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/slashable_attestation.rs b/eth2/types/src/slashable_attestation.rs index 676954ec2..db1e7fe79 100644 --- a/eth2/types/src/slashable_attestation.rs +++ b/eth2/types/src/slashable_attestation.rs @@ -3,8 +3,9 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TestRandom)] pub struct SlashableAttestation { pub validator_indices: Vec, pub data: AttestationData, @@ -23,17 +24,6 @@ impl TreeHash for SlashableAttestation { } } -impl TestRandom for SlashableAttestation { - fn random_for_test(rng: &mut T) -> Self { - Self { - validator_indices: <_>::random_for_test(rng), - data: <_>::random_for_test(rng), - custody_bitfield: <_>::random_for_test(rng), - aggregate_signature: <_>::random_for_test(rng), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/slashable_vote_data.rs b/eth2/types/src/slashable_vote_data.rs index bdd1d0619..0a79166da 100644 --- a/eth2/types/src/slashable_vote_data.rs +++ b/eth2/types/src/slashable_vote_data.rs @@ -6,8 +6,9 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TestRandom)] pub struct SlashableVoteData { pub custody_bit_0_indices: Vec, pub custody_bit_1_indices: Vec, @@ -47,17 +48,6 @@ impl TreeHash for SlashableVoteData { } } -impl TestRandom for SlashableVoteData { - fn random_for_test(rng: &mut T) -> Self { - Self { - custody_bit_0_indices: <_>::random_for_test(rng), - custody_bit_1_indices: <_>::random_for_test(rng), - data: <_>::random_for_test(rng), - aggregate_signature: <_>::random_for_test(rng), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/validator_registry_delta_block.rs b/eth2/types/src/validator_registry_delta_block.rs index 14f9c6ce5..35316a356 100644 --- a/eth2/types/src/validator_registry_delta_block.rs +++ b/eth2/types/src/validator_registry_delta_block.rs @@ -4,9 +4,10 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, TreeHash}; use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; // The information gathered from the PoW chain validator registration function. -#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode)] +#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, TestRandom)] pub struct ValidatorRegistryDeltaBlock { pub latest_registry_delta_root: Hash256, pub validator_index: u32, @@ -40,18 +41,6 @@ impl TreeHash for ValidatorRegistryDeltaBlock { } } -impl TestRandom for ValidatorRegistryDeltaBlock { - fn random_for_test(rng: &mut T) -> Self { - Self { - latest_registry_delta_root: <_>::random_for_test(rng), - validator_index: <_>::random_for_test(rng), - pubkey: <_>::random_for_test(rng), - slot: <_>::random_for_test(rng), - flag: <_>::random_for_test(rng), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/utils/test_random_derive/src/lib.rs b/eth2/utils/test_random_derive/src/lib.rs index 2c244e448..9a456606c 100644 --- a/eth2/utils/test_random_derive/src/lib.rs +++ b/eth2/utils/test_random_derive/src/lib.rs @@ -6,22 +6,22 @@ use syn::{parse_macro_input, DeriveInput}; #[proc_macro_derive(TestRandom)] pub fn test_random_derive(input: TokenStream) -> TokenStream { - let ast = parse_macro_input!(input as DeriveInput); - let name = &ast.ident; + let derived_input = parse_macro_input!(input as DeriveInput); + let name = &derived_input.ident; - let struct_data = match &ast.data { + let struct_data = match &derived_input.data { syn::Data::Struct(s) => s, _ => panic!("test_random_derive only supports structs."), }; - let field_names = get_named_field_idents_and_types(&struct_data); + let field_names = get_named_field_idents(&struct_data); let output = quote! { impl TestRandom for #name { fn random_for_test(rng: &mut T) -> Self { Self { #( - #field_names: <_>::random_for_test(rng) + #field_names: <_>::random_for_test(rng), )* } } @@ -31,7 +31,7 @@ pub fn test_random_derive(input: TokenStream) -> TokenStream { output.into() } -fn get_named_field_idents_and_types(struct_data: &syn::DataStruct) -> Vec<(&syn::Ident)> { +fn get_named_field_idents(struct_data: &syn::DataStruct) -> Vec<(&syn::Ident)> { struct_data .fields .iter() From f95a0134e679e1f7ea191625dcced70f654f5d18 Mon Sep 17 00:00:00 2001 From: mjkeating Date: Fri, 22 Feb 2019 13:07:04 -0800 Subject: [PATCH 013/132] now using the Hashtree macro for most struct types --- eth2/types/src/attestation.rs | 19 ++------- eth2/types/src/attestation_data.rs | 23 ++--------- .../src/attestation_data_and_custody_bit.rs | 17 ++------ eth2/types/src/attester_slashing.rs | 16 ++------ eth2/types/src/beacon_block.rs | 22 ++-------- eth2/types/src/beacon_block_body.rs | 19 ++------- eth2/types/src/beacon_state.rs | 40 +------------------ eth2/types/src/casper_slashing.rs | 16 ++------ eth2/types/src/crosslink.rs | 16 ++------ eth2/types/src/deposit.rs | 17 ++------ eth2/types/src/deposit_data.rs | 17 ++------ eth2/types/src/deposit_input.rs | 17 ++------ eth2/types/src/eth1_data.rs | 16 ++------ eth2/types/src/eth1_data_vote.rs | 16 ++------ eth2/types/src/exit.rs | 17 ++------ eth2/types/src/fork.rs | 17 ++------ eth2/types/src/pending_attestation.rs | 18 ++------- eth2/types/src/proposal_signed_data.rs | 17 ++------ eth2/types/src/proposer_slashing.rs | 19 ++------- eth2/types/src/shard_reassignment_record.rs | 17 ++------ eth2/types/src/slashable_attestation.rs | 18 ++------- eth2/types/src/slashable_vote_data.rs | 18 ++------- .../src/validator_registry_delta_block.rs | 19 ++------- eth2/utils/ssz/src/impl_tree_hash.rs | 6 +++ eth2/utils/ssz/src/tree_hash.rs | 4 +- eth2/utils/ssz_derive/src/lib.rs | 32 +++++++++++++++ 26 files changed, 110 insertions(+), 363 deletions(-) diff --git a/eth2/types/src/attestation.rs b/eth2/types/src/attestation.rs index 7388a8e49..a8523dfe8 100644 --- a/eth2/types/src/attestation.rs +++ b/eth2/types/src/attestation.rs @@ -2,10 +2,10 @@ use super::{AggregatePublicKey, AggregateSignature, AttestationData, Bitfield, H use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, TreeHash}; -use ssz_derive::{Decode, Encode}; +use ssz::TreeHash; +use ssz_derive::{Decode, Encode, Hashtree}; -#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode)] +#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, Hashtree)] pub struct Attestation { pub aggregation_bitfield: Bitfield, pub data: AttestationData, @@ -34,17 +34,6 @@ impl Attestation { } } -impl TreeHash for Attestation { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.aggregation_bitfield.hash_tree_root_internal()); - result.append(&mut self.data.hash_tree_root_internal()); - result.append(&mut self.custody_bitfield.hash_tree_root_internal()); - result.append(&mut self.aggregate_signature.hash_tree_root_internal()); - hash(&result) - } -} - impl TestRandom for Attestation { fn random_for_test(rng: &mut T) -> Self { Self { @@ -60,7 +49,7 @@ impl TestRandom for Attestation { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable}; + use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data.rs index 7edb0b72b..d2da31102 100644 --- a/eth2/types/src/attestation_data.rs +++ b/eth2/types/src/attestation_data.rs @@ -2,8 +2,8 @@ use crate::test_utils::TestRandom; use crate::{AttestationDataAndCustodyBit, Crosslink, Epoch, Hash256, Slot}; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, TreeHash}; -use ssz_derive::{Decode, Encode}; +use ssz::TreeHash; +use ssz_derive::{Decode, Encode, Hashtree}; pub const SSZ_ATTESTION_DATA_LENGTH: usize = { 8 + // slot @@ -16,7 +16,7 @@ pub const SSZ_ATTESTION_DATA_LENGTH: usize = { 32 // justified_block_root }; -#[derive(Debug, Clone, PartialEq, Default, Serialize, Hash, Encode, Decode)] +#[derive(Debug, Clone, PartialEq, Default, Serialize, Hash, Encode, Decode, Hashtree)] pub struct AttestationData { pub slot: Slot, pub shard: u64, @@ -44,21 +44,6 @@ impl AttestationData { } } -impl TreeHash for AttestationData { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.slot.hash_tree_root_internal()); - result.append(&mut self.shard.hash_tree_root_internal()); - result.append(&mut self.beacon_block_root.hash_tree_root_internal()); - result.append(&mut self.epoch_boundary_root.hash_tree_root_internal()); - result.append(&mut self.shard_block_root.hash_tree_root_internal()); - result.append(&mut self.latest_crosslink.hash_tree_root_internal()); - result.append(&mut self.justified_epoch.hash_tree_root_internal()); - result.append(&mut self.justified_block_root.hash_tree_root_internal()); - hash(&result) - } -} - impl TestRandom for AttestationData { fn random_for_test(rng: &mut T) -> Self { Self { @@ -78,7 +63,7 @@ impl TestRandom for AttestationData { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable}; + use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/attestation_data_and_custody_bit.rs b/eth2/types/src/attestation_data_and_custody_bit.rs index 3f107be82..33b7f3715 100644 --- a/eth2/types/src/attestation_data_and_custody_bit.rs +++ b/eth2/types/src/attestation_data_and_custody_bit.rs @@ -2,25 +2,14 @@ use super::AttestationData; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz::TreeHash; -use ssz_derive::{Decode, Encode}; +use ssz_derive::{Decode, Encode, Hashtree}; -#[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode)] +#[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode, Hashtree)] pub struct AttestationDataAndCustodyBit { pub data: AttestationData, pub custody_bit: bool, } -impl TreeHash for AttestationDataAndCustodyBit { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.data.hash_tree_root_internal()); - // TODO: add bool ssz - // result.append(custody_bit.hash_tree_root_internal()); - ssz::hash(&result) - } -} - impl TestRandom for AttestationDataAndCustodyBit { fn random_for_test(rng: &mut T) -> Self { Self { @@ -35,7 +24,7 @@ impl TestRandom for AttestationDataAndCustodyBit { mod test { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable}; + use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/attester_slashing.rs b/eth2/types/src/attester_slashing.rs index f84998324..76c0c2b2c 100644 --- a/eth2/types/src/attester_slashing.rs +++ b/eth2/types/src/attester_slashing.rs @@ -1,24 +1,14 @@ use crate::{test_utils::TestRandom, SlashableAttestation}; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, TreeHash}; -use ssz_derive::{Decode, Encode}; +use ssz_derive::{Decode, Encode, Hashtree}; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, Hashtree)] pub struct AttesterSlashing { pub slashable_attestation_1: SlashableAttestation, pub slashable_attestation_2: SlashableAttestation, } -impl TreeHash for AttesterSlashing { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.slashable_attestation_1.hash_tree_root_internal()); - result.append(&mut self.slashable_attestation_2.hash_tree_root_internal()); - hash(&result) - } -} - impl TestRandom for AttesterSlashing { fn random_for_test(rng: &mut T) -> Self { Self { @@ -32,7 +22,7 @@ impl TestRandom for AttesterSlashing { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable}; + use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/beacon_block.rs b/eth2/types/src/beacon_block.rs index c252d03f7..d1dcd58df 100644 --- a/eth2/types/src/beacon_block.rs +++ b/eth2/types/src/beacon_block.rs @@ -3,10 +3,10 @@ use crate::{BeaconBlockBody, ChainSpec, Eth1Data, Hash256, ProposalSignedData, S use bls::Signature; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, TreeHash}; -use ssz_derive::{Decode, Encode}; +use ssz::TreeHash; +use ssz_derive::{Decode, Encode, Hashtree}; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, Hashtree)] pub struct BeaconBlock { pub slot: Slot, pub parent_root: Hash256, @@ -60,20 +60,6 @@ impl BeaconBlock { } } -impl TreeHash for BeaconBlock { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.slot.hash_tree_root_internal()); - result.append(&mut self.parent_root.hash_tree_root_internal()); - result.append(&mut self.state_root.hash_tree_root_internal()); - result.append(&mut self.randao_reveal.hash_tree_root_internal()); - result.append(&mut self.eth1_data.hash_tree_root_internal()); - result.append(&mut self.signature.hash_tree_root_internal()); - result.append(&mut self.body.hash_tree_root_internal()); - hash(&result) - } -} - impl TestRandom for BeaconBlock { fn random_for_test(rng: &mut T) -> Self { Self { @@ -92,7 +78,7 @@ impl TestRandom for BeaconBlock { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable}; + use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/beacon_block_body.rs b/eth2/types/src/beacon_block_body.rs index e051f5940..62e8e1e90 100644 --- a/eth2/types/src/beacon_block_body.rs +++ b/eth2/types/src/beacon_block_body.rs @@ -2,10 +2,9 @@ use super::{Attestation, AttesterSlashing, Deposit, Exit, ProposerSlashing}; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, TreeHash}; -use ssz_derive::{Decode, Encode}; +use ssz_derive::{Decode, Encode, Hashtree}; -#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, Hashtree)] pub struct BeaconBlockBody { pub proposer_slashings: Vec, pub attester_slashings: Vec, @@ -14,18 +13,6 @@ pub struct BeaconBlockBody { pub exits: Vec, } -impl TreeHash for BeaconBlockBody { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.proposer_slashings.hash_tree_root_internal()); - result.append(&mut self.attester_slashings.hash_tree_root_internal()); - result.append(&mut self.attestations.hash_tree_root_internal()); - result.append(&mut self.deposits.hash_tree_root_internal()); - result.append(&mut self.exits.hash_tree_root_internal()); - hash(&result) - } -} - impl TestRandom for BeaconBlockBody { fn random_for_test(rng: &mut T) -> Self { Self { @@ -42,7 +29,7 @@ impl TestRandom for BeaconBlockBody { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable}; + use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 21deb6fe7..ee3d42d80 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -10,7 +10,7 @@ use log::trace; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, TreeHash}; -use ssz_derive::{Decode, Encode}; +use ssz_derive::{Decode, Encode, Hashtree}; use swap_or_not_shuffle::get_permutated_index; mod tests; @@ -52,7 +52,7 @@ macro_rules! safe_sub_assign { }; } -#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, Hashtree)] pub struct BeaconState { // Misc pub slot: Slot, @@ -968,42 +968,6 @@ impl From for InclusionError { } } -impl TreeHash for BeaconState { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.slot.hash_tree_root_internal()); - result.append(&mut self.genesis_time.hash_tree_root_internal()); - result.append(&mut self.fork.hash_tree_root_internal()); - result.append(&mut self.validator_registry.hash_tree_root_internal()); - result.append(&mut self.validator_balances.hash_tree_root_internal()); - result.append( - &mut self - .validator_registry_update_epoch - .hash_tree_root_internal(), - ); - result.append(&mut self.latest_randao_mixes.hash_tree_root_internal()); - result.append(&mut self.previous_epoch_start_shard.hash_tree_root_internal()); - result.append(&mut self.current_epoch_start_shard.hash_tree_root_internal()); - result.append(&mut self.previous_calculation_epoch.hash_tree_root_internal()); - result.append(&mut self.current_calculation_epoch.hash_tree_root_internal()); - result.append(&mut self.previous_epoch_seed.hash_tree_root_internal()); - result.append(&mut self.current_epoch_seed.hash_tree_root_internal()); - result.append(&mut self.previous_justified_epoch.hash_tree_root_internal()); - result.append(&mut self.justified_epoch.hash_tree_root_internal()); - result.append(&mut self.justification_bitfield.hash_tree_root_internal()); - result.append(&mut self.finalized_epoch.hash_tree_root_internal()); - result.append(&mut self.latest_crosslinks.hash_tree_root_internal()); - result.append(&mut self.latest_block_roots.hash_tree_root_internal()); - result.append(&mut self.latest_index_roots.hash_tree_root_internal()); - result.append(&mut self.latest_penalized_balances.hash_tree_root_internal()); - result.append(&mut self.latest_attestations.hash_tree_root_internal()); - result.append(&mut self.batched_block_roots.hash_tree_root_internal()); - result.append(&mut self.latest_eth1_data.hash_tree_root_internal()); - result.append(&mut self.eth1_data_votes.hash_tree_root_internal()); - hash(&result) - } -} - impl TestRandom for BeaconState { fn random_for_test(rng: &mut T) -> Self { Self { diff --git a/eth2/types/src/casper_slashing.rs b/eth2/types/src/casper_slashing.rs index 6346db65c..5532a8b57 100644 --- a/eth2/types/src/casper_slashing.rs +++ b/eth2/types/src/casper_slashing.rs @@ -2,24 +2,14 @@ use super::SlashableVoteData; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, TreeHash}; -use ssz_derive::{Decode, Encode}; +use ssz_derive::{Decode, Encode, Hashtree}; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, Hashtree)] pub struct CasperSlashing { pub slashable_vote_data_1: SlashableVoteData, pub slashable_vote_data_2: SlashableVoteData, } -impl TreeHash for CasperSlashing { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.slashable_vote_data_1.hash_tree_root_internal()); - result.append(&mut self.slashable_vote_data_2.hash_tree_root_internal()); - hash(&result) - } -} - impl TestRandom for CasperSlashing { fn random_for_test(rng: &mut T) -> Self { Self { @@ -33,7 +23,7 @@ impl TestRandom for CasperSlashing { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable}; + use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/crosslink.rs b/eth2/types/src/crosslink.rs index 19c71f604..45a92d787 100644 --- a/eth2/types/src/crosslink.rs +++ b/eth2/types/src/crosslink.rs @@ -2,10 +2,9 @@ use crate::test_utils::TestRandom; use crate::{Epoch, Hash256}; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, TreeHash}; -use ssz_derive::{Decode, Encode}; +use ssz_derive::{Decode, Encode, Hashtree}; -#[derive(Debug, Clone, PartialEq, Default, Serialize, Hash, Encode, Decode)] +#[derive(Debug, Clone, PartialEq, Default, Serialize, Hash, Encode, Decode, Hashtree)] pub struct Crosslink { pub epoch: Epoch, pub shard_block_root: Hash256, @@ -21,15 +20,6 @@ impl Crosslink { } } -impl TreeHash for Crosslink { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.epoch.hash_tree_root_internal()); - result.append(&mut self.shard_block_root.hash_tree_root_internal()); - hash(&result) - } -} - impl TestRandom for Crosslink { fn random_for_test(rng: &mut T) -> Self { Self { @@ -43,7 +33,7 @@ impl TestRandom for Crosslink { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable}; + use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/deposit.rs b/eth2/types/src/deposit.rs index 78f43532a..5a738d4c5 100644 --- a/eth2/types/src/deposit.rs +++ b/eth2/types/src/deposit.rs @@ -2,26 +2,15 @@ use super::{DepositData, Hash256}; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, TreeHash}; -use ssz_derive::{Decode, Encode}; +use ssz_derive::{Decode, Encode, Hashtree}; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, Hashtree)] pub struct Deposit { pub branch: Vec, pub index: u64, pub deposit_data: DepositData, } -impl TreeHash for Deposit { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.branch.hash_tree_root_internal()); - result.append(&mut self.index.hash_tree_root_internal()); - result.append(&mut self.deposit_data.hash_tree_root_internal()); - hash(&result) - } -} - impl TestRandom for Deposit { fn random_for_test(rng: &mut T) -> Self { Self { @@ -36,7 +25,7 @@ impl TestRandom for Deposit { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable}; + use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/deposit_data.rs b/eth2/types/src/deposit_data.rs index 8f49deb3c..fa0bd78b6 100644 --- a/eth2/types/src/deposit_data.rs +++ b/eth2/types/src/deposit_data.rs @@ -2,26 +2,15 @@ use super::DepositInput; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, TreeHash}; -use ssz_derive::{Decode, Encode}; +use ssz_derive::{Decode, Encode, Hashtree}; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, Hashtree)] pub struct DepositData { pub amount: u64, pub timestamp: u64, pub deposit_input: DepositInput, } -impl TreeHash for DepositData { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.amount.hash_tree_root_internal()); - result.append(&mut self.timestamp.hash_tree_root_internal()); - result.append(&mut self.deposit_input.hash_tree_root_internal()); - hash(&result) - } -} - impl TestRandom for DepositData { fn random_for_test(rng: &mut T) -> Self { Self { @@ -36,7 +25,7 @@ impl TestRandom for DepositData { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable}; + use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/deposit_input.rs b/eth2/types/src/deposit_input.rs index 7556fc2ca..24146ad44 100644 --- a/eth2/types/src/deposit_input.rs +++ b/eth2/types/src/deposit_input.rs @@ -3,26 +3,15 @@ use crate::test_utils::TestRandom; use bls::{PublicKey, Signature}; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, TreeHash}; -use ssz_derive::{Decode, Encode}; +use ssz_derive::{Decode, Encode, Hashtree}; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, Hashtree)] pub struct DepositInput { pub pubkey: PublicKey, pub withdrawal_credentials: Hash256, pub proof_of_possession: Signature, } -impl TreeHash for DepositInput { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.pubkey.hash_tree_root_internal()); - result.append(&mut self.withdrawal_credentials.hash_tree_root_internal()); - result.append(&mut self.proof_of_possession.hash_tree_root_internal()); - hash(&result) - } -} - impl TestRandom for DepositInput { fn random_for_test(rng: &mut T) -> Self { Self { @@ -37,7 +26,7 @@ impl TestRandom for DepositInput { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable}; + use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/eth1_data.rs b/eth2/types/src/eth1_data.rs index b0dc14e7a..cdf746fae 100644 --- a/eth2/types/src/eth1_data.rs +++ b/eth2/types/src/eth1_data.rs @@ -2,25 +2,15 @@ use super::Hash256; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, TreeHash}; -use ssz_derive::{Decode, Encode}; +use ssz_derive::{Decode, Encode, Hashtree}; // Note: this is refer to as DepositRootVote in specs -#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, Hashtree)] pub struct Eth1Data { pub deposit_root: Hash256, pub block_hash: Hash256, } -impl TreeHash for Eth1Data { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.deposit_root.hash_tree_root_internal()); - result.append(&mut self.block_hash.hash_tree_root_internal()); - hash(&result) - } -} - impl TestRandom for Eth1Data { fn random_for_test(rng: &mut T) -> Self { Self { @@ -34,7 +24,7 @@ impl TestRandom for Eth1Data { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable}; + use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/eth1_data_vote.rs b/eth2/types/src/eth1_data_vote.rs index eda6e6a6a..acf77ffac 100644 --- a/eth2/types/src/eth1_data_vote.rs +++ b/eth2/types/src/eth1_data_vote.rs @@ -2,25 +2,15 @@ use super::Eth1Data; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, TreeHash}; -use ssz_derive::{Decode, Encode}; +use ssz_derive::{Decode, Encode, Hashtree}; // Note: this is refer to as DepositRootVote in specs -#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, Hashtree)] pub struct Eth1DataVote { pub eth1_data: Eth1Data, pub vote_count: u64, } -impl TreeHash for Eth1DataVote { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.eth1_data.hash_tree_root_internal()); - result.append(&mut self.vote_count.hash_tree_root_internal()); - hash(&result) - } -} - impl TestRandom for Eth1DataVote { fn random_for_test(rng: &mut T) -> Self { Self { @@ -34,7 +24,7 @@ impl TestRandom for Eth1DataVote { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable}; + use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/exit.rs b/eth2/types/src/exit.rs index 18d743b83..c0af761fd 100644 --- a/eth2/types/src/exit.rs +++ b/eth2/types/src/exit.rs @@ -2,26 +2,15 @@ use crate::{test_utils::TestRandom, Epoch}; use bls::Signature; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, TreeHash}; -use ssz_derive::{Decode, Encode}; +use ssz_derive::{Decode, Encode, Hashtree}; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, Hashtree)] pub struct Exit { pub epoch: Epoch, pub validator_index: u64, pub signature: Signature, } -impl TreeHash for Exit { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.epoch.hash_tree_root_internal()); - result.append(&mut self.validator_index.hash_tree_root_internal()); - result.append(&mut self.signature.hash_tree_root_internal()); - hash(&result) - } -} - impl TestRandom for Exit { fn random_for_test(rng: &mut T) -> Self { Self { @@ -36,7 +25,7 @@ impl TestRandom for Exit { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable}; + use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs index 85d530e19..c6c2b27ff 100644 --- a/eth2/types/src/fork.rs +++ b/eth2/types/src/fork.rs @@ -1,26 +1,15 @@ use crate::{test_utils::TestRandom, Epoch}; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, TreeHash}; -use ssz_derive::{Decode, Encode}; +use ssz_derive::{Decode, Encode, Hashtree}; -#[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode)] +#[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode, Hashtree)] pub struct Fork { pub previous_version: u64, pub current_version: u64, pub epoch: Epoch, } -impl TreeHash for Fork { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.previous_version.hash_tree_root_internal()); - result.append(&mut self.current_version.hash_tree_root_internal()); - result.append(&mut self.epoch.hash_tree_root_internal()); - hash(&result) - } -} - impl TestRandom for Fork { fn random_for_test(rng: &mut T) -> Self { Self { @@ -35,7 +24,7 @@ impl TestRandom for Fork { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable}; + use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/pending_attestation.rs b/eth2/types/src/pending_attestation.rs index 42f990210..b6f152321 100644 --- a/eth2/types/src/pending_attestation.rs +++ b/eth2/types/src/pending_attestation.rs @@ -2,10 +2,9 @@ use crate::test_utils::TestRandom; use crate::{AttestationData, Bitfield, Slot}; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, TreeHash}; -use ssz_derive::{Decode, Encode}; +use ssz_derive::{Decode, Encode, Hashtree}; -#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode)] +#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, Hashtree)] pub struct PendingAttestation { pub aggregation_bitfield: Bitfield, pub data: AttestationData, @@ -13,17 +12,6 @@ pub struct PendingAttestation { pub inclusion_slot: Slot, } -impl TreeHash for PendingAttestation { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.aggregation_bitfield.hash_tree_root_internal()); - result.append(&mut self.data.hash_tree_root_internal()); - result.append(&mut self.custody_bitfield.hash_tree_root_internal()); - result.append(&mut self.inclusion_slot.hash_tree_root_internal()); - hash(&result) - } -} - impl TestRandom for PendingAttestation { fn random_for_test(rng: &mut T) -> Self { Self { @@ -39,7 +27,7 @@ impl TestRandom for PendingAttestation { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable}; + use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/proposal_signed_data.rs b/eth2/types/src/proposal_signed_data.rs index 63c0f1ce6..53f04956b 100644 --- a/eth2/types/src/proposal_signed_data.rs +++ b/eth2/types/src/proposal_signed_data.rs @@ -2,26 +2,15 @@ use crate::test_utils::TestRandom; use crate::{Hash256, Slot}; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, TreeHash}; -use ssz_derive::{Decode, Encode}; +use ssz_derive::{Decode, Encode, Hashtree}; -#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, Hashtree)] pub struct ProposalSignedData { pub slot: Slot, pub shard: u64, pub block_root: Hash256, } -impl TreeHash for ProposalSignedData { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.slot.hash_tree_root_internal()); - result.append(&mut self.shard.hash_tree_root_internal()); - result.append(&mut self.block_root.hash_tree_root_internal()); - hash(&result) - } -} - impl TestRandom for ProposalSignedData { fn random_for_test(rng: &mut T) -> Self { Self { @@ -36,7 +25,7 @@ impl TestRandom for ProposalSignedData { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable}; + use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/proposer_slashing.rs b/eth2/types/src/proposer_slashing.rs index b3a819a7f..739c5f84d 100644 --- a/eth2/types/src/proposer_slashing.rs +++ b/eth2/types/src/proposer_slashing.rs @@ -3,10 +3,9 @@ use crate::test_utils::TestRandom; use bls::Signature; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, TreeHash}; -use ssz_derive::{Decode, Encode}; +use ssz_derive::{Decode, Encode, Hashtree}; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, Hashtree)] pub struct ProposerSlashing { pub proposer_index: u64, pub proposal_data_1: ProposalSignedData, @@ -15,18 +14,6 @@ pub struct ProposerSlashing { pub proposal_signature_2: Signature, } -impl TreeHash for ProposerSlashing { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.proposer_index.hash_tree_root_internal()); - result.append(&mut self.proposal_data_1.hash_tree_root_internal()); - result.append(&mut self.proposal_signature_1.hash_tree_root_internal()); - result.append(&mut self.proposal_data_2.hash_tree_root_internal()); - result.append(&mut self.proposal_signature_2.hash_tree_root_internal()); - hash(&result) - } -} - impl TestRandom for ProposerSlashing { fn random_for_test(rng: &mut T) -> Self { Self { @@ -43,7 +30,7 @@ impl TestRandom for ProposerSlashing { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable}; + use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/shard_reassignment_record.rs b/eth2/types/src/shard_reassignment_record.rs index 511fe13ca..4fa685d26 100644 --- a/eth2/types/src/shard_reassignment_record.rs +++ b/eth2/types/src/shard_reassignment_record.rs @@ -1,26 +1,15 @@ use crate::{test_utils::TestRandom, Slot}; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, TreeHash}; -use ssz_derive::{Decode, Encode}; +use ssz_derive::{Decode, Encode, Hashtree}; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, Hashtree)] pub struct ShardReassignmentRecord { pub validator_index: u64, pub shard: u64, pub slot: Slot, } -impl TreeHash for ShardReassignmentRecord { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.validator_index.hash_tree_root_internal()); - result.append(&mut self.shard.hash_tree_root_internal()); - result.append(&mut self.slot.hash_tree_root_internal()); - hash(&result) - } -} - impl TestRandom for ShardReassignmentRecord { fn random_for_test(rng: &mut T) -> Self { Self { @@ -35,7 +24,7 @@ impl TestRandom for ShardReassignmentRecord { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable}; + use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/slashable_attestation.rs b/eth2/types/src/slashable_attestation.rs index 676954ec2..64507994b 100644 --- a/eth2/types/src/slashable_attestation.rs +++ b/eth2/types/src/slashable_attestation.rs @@ -1,10 +1,9 @@ use crate::{test_utils::TestRandom, AggregateSignature, AttestationData, Bitfield}; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, TreeHash}; -use ssz_derive::{Decode, Encode}; +use ssz_derive::{Decode, Encode, Hashtree}; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, Hashtree)] pub struct SlashableAttestation { pub validator_indices: Vec, pub data: AttestationData, @@ -12,17 +11,6 @@ pub struct SlashableAttestation { pub aggregate_signature: AggregateSignature, } -impl TreeHash for SlashableAttestation { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.validator_indices.hash_tree_root_internal()); - result.append(&mut self.data.hash_tree_root_internal()); - result.append(&mut self.custody_bitfield.hash_tree_root_internal()); - result.append(&mut self.aggregate_signature.hash_tree_root_internal()); - hash(&result) - } -} - impl TestRandom for SlashableAttestation { fn random_for_test(rng: &mut T) -> Self { Self { @@ -38,7 +26,7 @@ impl TestRandom for SlashableAttestation { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable}; + use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/slashable_vote_data.rs b/eth2/types/src/slashable_vote_data.rs index bdd1d0619..53742372c 100644 --- a/eth2/types/src/slashable_vote_data.rs +++ b/eth2/types/src/slashable_vote_data.rs @@ -4,10 +4,9 @@ use crate::test_utils::TestRandom; use bls::AggregateSignature; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, TreeHash}; -use ssz_derive::{Decode, Encode}; +use ssz_derive::{Decode, Encode, Hashtree}; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, Hashtree)] pub struct SlashableVoteData { pub custody_bit_0_indices: Vec, pub custody_bit_1_indices: Vec, @@ -36,17 +35,6 @@ impl SlashableVoteData { } } -impl TreeHash for SlashableVoteData { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.custody_bit_0_indices.hash_tree_root_internal()); - result.append(&mut self.custody_bit_1_indices.hash_tree_root_internal()); - result.append(&mut self.data.hash_tree_root_internal()); - result.append(&mut self.aggregate_signature.hash_tree_root_internal()); - hash(&result) - } -} - impl TestRandom for SlashableVoteData { fn random_for_test(rng: &mut T) -> Self { Self { @@ -64,7 +52,7 @@ mod tests { use crate::chain_spec::ChainSpec; use crate::slot_epoch::{Epoch, Slot}; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable}; + use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_is_double_vote_true() { diff --git a/eth2/types/src/validator_registry_delta_block.rs b/eth2/types/src/validator_registry_delta_block.rs index 14f9c6ce5..c42300cb4 100644 --- a/eth2/types/src/validator_registry_delta_block.rs +++ b/eth2/types/src/validator_registry_delta_block.rs @@ -2,11 +2,10 @@ use crate::{test_utils::TestRandom, Hash256, Slot}; use bls::PublicKey; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, TreeHash}; -use ssz_derive::{Decode, Encode}; +use ssz_derive::{Decode, Encode, Hashtree}; // The information gathered from the PoW chain validator registration function. -#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode)] +#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, Hashtree)] pub struct ValidatorRegistryDeltaBlock { pub latest_registry_delta_root: Hash256, pub validator_index: u32, @@ -28,18 +27,6 @@ impl Default for ValidatorRegistryDeltaBlock { } } -impl TreeHash for ValidatorRegistryDeltaBlock { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.latest_registry_delta_root.hash_tree_root_internal()); - result.append(&mut self.validator_index.hash_tree_root_internal()); - result.append(&mut self.pubkey.hash_tree_root_internal()); - result.append(&mut self.slot.hash_tree_root_internal()); - result.append(&mut self.flag.hash_tree_root_internal()); - hash(&result) - } -} - impl TestRandom for ValidatorRegistryDeltaBlock { fn random_for_test(rng: &mut T) -> Self { Self { @@ -56,7 +43,7 @@ impl TestRandom for ValidatorRegistryDeltaBlock { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable}; + use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/utils/ssz/src/impl_tree_hash.rs b/eth2/utils/ssz/src/impl_tree_hash.rs index 7c3dae596..54bd7c139 100644 --- a/eth2/utils/ssz/src/impl_tree_hash.rs +++ b/eth2/utils/ssz/src/impl_tree_hash.rs @@ -32,6 +32,12 @@ impl TreeHash for usize { } } +impl TreeHash for bool { + fn hash_tree_root_internal(&self) -> Vec { + ssz_encode(self) + } +} + impl TreeHash for Address { fn hash_tree_root_internal(&self) -> Vec { ssz_encode(self) diff --git a/eth2/utils/ssz/src/tree_hash.rs b/eth2/utils/ssz/src/tree_hash.rs index bb05f01db..7c1ab35e9 100644 --- a/eth2/utils/ssz/src/tree_hash.rs +++ b/eth2/utils/ssz/src/tree_hash.rs @@ -7,9 +7,7 @@ pub trait TreeHash { fn hash_tree_root_internal(&self) -> Vec; fn hash_tree_root(&self) -> Vec { let mut result = self.hash_tree_root_internal(); - if result.len() < HASHSIZE { - zpad(&mut result, HASHSIZE); - } + zpad(&mut result, HASHSIZE); result } } diff --git a/eth2/utils/ssz_derive/src/lib.rs b/eth2/utils/ssz_derive/src/lib.rs index 1bc5caef1..09e1b2dc7 100644 --- a/eth2/utils/ssz_derive/src/lib.rs +++ b/eth2/utils/ssz_derive/src/lib.rs @@ -2,6 +2,7 @@ //! //! - `#[derive(Encode)]` //! - `#[derive(Decode)]` +//! - `#[derive(Hashtree)]` //! //! These macros provide SSZ encoding/decoding for a `struct`. Fields are encoded/decoded in the //! order they are defined. @@ -126,3 +127,34 @@ pub fn ssz_decode_derive(input: TokenStream) -> TokenStream { }; output.into() } + +/// Implements `ssz::TreeHash` for some `struct`. +/// +/// Fields are processed in the order they are defined. +#[proc_macro_derive(Hashtree)] +pub fn ssz_hashtree_derive(input: TokenStream) -> TokenStream { + let item = parse_macro_input!(input as DeriveInput); + + let name = &item.ident; + + let struct_data = match &item.data { + syn::Data::Struct(s) => s, + _ => panic!("ssz_derive only supports structs."), + }; + + let field_idents = get_named_field_idents(&struct_data); + + let output = quote! { + impl ssz::TreeHash for #name { + fn hash_tree_root_internal(&self) -> Vec { + let mut result: Vec = vec![]; + #( + result.append(&mut self.#field_idents.hash_tree_root_internal()); + )* + + ssz::hash(&result) + } + } + }; + output.into() +} From 9f9b466f95a12867f0b4bb858fc2eab8b769f30b Mon Sep 17 00:00:00 2001 From: Kirk Baird Date: Sat, 23 Feb 2019 14:39:54 +1100 Subject: [PATCH 014/132] Modify attestion_aggregation to use frok version in domain --- beacon_node/beacon_chain/src/attestation_aggregator.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/attestation_aggregator.rs b/beacon_node/beacon_chain/src/attestation_aggregator.rs index d5b8c090f..d70d732f8 100644 --- a/beacon_node/beacon_chain/src/attestation_aggregator.rs +++ b/beacon_node/beacon_chain/src/attestation_aggregator.rs @@ -114,7 +114,10 @@ impl AttestationAggregator { if !free_attestation.signature.verify( &signable_message, - spec.domain_attestation, + cached_state.state.fork.get_domain( + cached_state.state.current_epoch(spec), + spec.domain_attestation, + ), &validator_record.pubkey, ) { return Ok(Outcome { From 779b6266a5da859cea327db2407d5880775a85e0 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 23 Feb 2019 18:45:32 +1300 Subject: [PATCH 015/132] Ensure shuffling is cached between slot calcs Previously it was being re-built for every slot, now it is being generated once-per-epoch. --- eth2/types/src/beacon_state.rs | 196 +++++++++++++-------- eth2/types/src/beacon_state/epoch_cache.rs | 11 +- 2 files changed, 134 insertions(+), 73 deletions(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 47805da80..03d2fd52f 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -2,12 +2,12 @@ use self::epoch_cache::EpochCache; use crate::test_utils::TestRandom; use crate::{ validator::StatusFlags, validator_registry::get_active_validator_indices, AttestationData, - Bitfield, ChainSpec, Crosslink, Deposit, Epoch, Eth1Data, Eth1DataVote, Fork, Hash256, - PendingAttestation, PublicKey, Signature, Slot, Validator, + Bitfield, ChainSpec, Crosslink, Deposit, DepositData, Epoch, Eth1Data, Eth1DataVote, Fork, + Hash256, PendingAttestation, PublicKey, Signature, Slot, Validator, }; use bls::verify_proof_of_possession; use honey_badger_split::SplitExt; -use log::trace; +use log::{debug, trace}; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; @@ -120,6 +120,7 @@ impl BeaconState { latest_eth1_data: Eth1Data, spec: &ChainSpec, ) -> Result { + debug!("Creating genesis state."); let initial_crosslink = Crosslink { epoch: spec.genesis_epoch, shard_block_root: spec.zero_hash, @@ -186,15 +187,14 @@ impl BeaconState { caches: vec![EpochCache::empty(); CACHED_EPOCHS], }; - for deposit in initial_validator_deposits { - let _index = genesis_state.process_deposit( - deposit.deposit_data.deposit_input.pubkey, - deposit.deposit_data.amount, - deposit.deposit_data.deposit_input.proof_of_possession, - deposit.deposit_data.deposit_input.withdrawal_credentials, - spec, - ); - } + let deposit_data = initial_validator_deposits + .iter() + .map(|deposit| &deposit.deposit_data) + .collect(); + + genesis_state.process_deposits_optimized(deposit_data, spec); + + trace!("Processed genesis deposits."); for validator_index in 0..genesis_state.validator_registry.len() { if genesis_state.get_effective_balance(validator_index, spec) >= spec.max_deposit_amount @@ -479,6 +479,77 @@ impl BeaconState { Ok(&cache.committees[slot_offset.as_usize()]) } + pub(crate) fn get_shuffling_for_slot( + &self, + slot: Slot, + registry_change: bool, + spec: &ChainSpec, + ) -> Result>, Error> { + let (_committees_per_epoch, seed, shuffling_epoch, _shuffling_start_shard) = + self.get_committee_params_at_slot(slot, registry_change, spec)?; + + self.get_shuffling(seed, shuffling_epoch, spec) + .ok_or_else(|| Error::UnableToShuffle) + } + + pub(crate) fn get_committee_params_at_slot( + &self, + slot: Slot, + registry_change: bool, + spec: &ChainSpec, + ) -> Result<(u64, Hash256, Epoch, u64), Error> { + let epoch = slot.epoch(spec.epoch_length); + let current_epoch = self.current_epoch(spec); + let previous_epoch = self.previous_epoch(spec); + let next_epoch = self.next_epoch(spec); + + if epoch == current_epoch { + trace!("get_crosslink_committees_at_slot: current_epoch"); + Ok(( + self.get_current_epoch_committee_count(spec), + self.current_epoch_seed, + self.current_calculation_epoch, + self.current_epoch_start_shard, + )) + } else if epoch == previous_epoch { + trace!("get_crosslink_committees_at_slot: previous_epoch"); + Ok(( + self.get_previous_epoch_committee_count(spec), + self.previous_epoch_seed, + self.previous_calculation_epoch, + self.previous_epoch_start_shard, + )) + } else if epoch == next_epoch { + trace!("get_crosslink_committees_at_slot: next_epoch"); + let current_committees_per_epoch = self.get_current_epoch_committee_count(spec); + let epochs_since_last_registry_update = + current_epoch - self.validator_registry_update_epoch; + let (seed, shuffling_start_shard) = if registry_change { + let next_seed = self.generate_seed(next_epoch, spec)?; + ( + next_seed, + (self.current_epoch_start_shard + current_committees_per_epoch) + % spec.shard_count, + ) + } else if (epochs_since_last_registry_update > 1) + & epochs_since_last_registry_update.is_power_of_two() + { + let next_seed = self.generate_seed(next_epoch, spec)?; + (next_seed, self.current_epoch_start_shard) + } else { + (self.current_epoch_seed, self.current_epoch_start_shard) + }; + Ok(( + self.get_next_epoch_committee_count(spec), + seed, + next_epoch, + shuffling_start_shard, + )) + } else { + Err(Error::EpochOutOfBounds) + } + } + /// Return the list of ``(committee, shard)`` tuples for the ``slot``. /// /// Note: There are two possible shufflings for crosslink committees for a @@ -491,63 +562,12 @@ impl BeaconState { &self, slot: Slot, registry_change: bool, + shuffling: Vec>, spec: &ChainSpec, ) -> Result, u64)>, Error> { - let epoch = slot.epoch(spec.epoch_length); - let current_epoch = self.current_epoch(spec); - let previous_epoch = self.previous_epoch(spec); - let next_epoch = self.next_epoch(spec); + let (committees_per_epoch, _seed, _shuffling_epoch, shuffling_start_shard) = + self.get_committee_params_at_slot(slot, registry_change, spec)?; - let (committees_per_epoch, seed, shuffling_epoch, shuffling_start_shard) = - if epoch == current_epoch { - trace!("get_crosslink_committees_at_slot: current_epoch"); - ( - self.get_current_epoch_committee_count(spec), - self.current_epoch_seed, - self.current_calculation_epoch, - self.current_epoch_start_shard, - ) - } else if epoch == previous_epoch { - trace!("get_crosslink_committees_at_slot: previous_epoch"); - ( - self.get_previous_epoch_committee_count(spec), - self.previous_epoch_seed, - self.previous_calculation_epoch, - self.previous_epoch_start_shard, - ) - } else if epoch == next_epoch { - trace!("get_crosslink_committees_at_slot: next_epoch"); - let current_committees_per_epoch = self.get_current_epoch_committee_count(spec); - let epochs_since_last_registry_update = - current_epoch - self.validator_registry_update_epoch; - let (seed, shuffling_start_shard) = if registry_change { - let next_seed = self.generate_seed(next_epoch, spec)?; - ( - next_seed, - (self.current_epoch_start_shard + current_committees_per_epoch) - % spec.shard_count, - ) - } else if (epochs_since_last_registry_update > 1) - & epochs_since_last_registry_update.is_power_of_two() - { - let next_seed = self.generate_seed(next_epoch, spec)?; - (next_seed, self.current_epoch_start_shard) - } else { - (self.current_epoch_seed, self.current_epoch_start_shard) - }; - ( - self.get_next_epoch_committee_count(spec), - seed, - next_epoch, - shuffling_start_shard, - ) - } else { - return Err(Error::EpochOutOfBounds); - }; - - let shuffling = self - .get_shuffling(seed, shuffling_epoch, spec) - .ok_or_else(|| Error::UnableToShuffle)?; let offset = slot.as_u64() % spec.epoch_length; let committees_per_slot = committees_per_epoch / spec.epoch_length; let slot_start_shard = @@ -724,6 +744,35 @@ impl BeaconState { self.validator_registry_update_epoch = current_epoch; } + + pub fn process_deposits_optimized( + &mut self, + deposits: Vec<&DepositData>, + spec: &ChainSpec, + ) -> Vec { + let mut added_indices = vec![]; + let mut pubkey_map: HashMap = HashMap::new(); + + for (i, validator) in self.validator_registry.iter().enumerate() { + pubkey_map.insert(validator.pubkey.clone(), i); + } + + for deposit_data in deposits { + let result = self.process_deposit( + deposit_data.deposit_input.pubkey.clone(), + deposit_data.amount, + deposit_data.deposit_input.proof_of_possession.clone(), + deposit_data.deposit_input.withdrawal_credentials, + Some(&pubkey_map), + spec, + ); + if let Ok(index) = result { + added_indices.push(index); + } + } + added_indices + } + /// Process a validator deposit, returning the validator index if the deposit is valid. /// /// Spec v0.2.0 @@ -733,6 +782,7 @@ impl BeaconState { amount: u64, proof_of_possession: Signature, withdrawal_credentials: Hash256, + pubkey_map: Option<&HashMap>, spec: &ChainSpec, ) -> Result { // TODO: ensure verify proof-of-possession represents the spec accurately. @@ -740,11 +790,15 @@ impl BeaconState { return Err(()); } - if let Some(index) = self - .validator_registry - .iter() - .position(|v| v.pubkey == pubkey) - { + let validator_index = if let Some(pubkey_map) = pubkey_map { + pubkey_map.get(&pubkey).and_then(|i| Some(*i)) + } else { + self.validator_registry + .iter() + .position(|v| v.pubkey == pubkey) + }; + + if let Some(index) = validator_index { if self.validator_registry[index].withdrawal_credentials == withdrawal_credentials { safe_add_assign!(self.validator_balances[index], amount); Ok(index) diff --git a/eth2/types/src/beacon_state/epoch_cache.rs b/eth2/types/src/beacon_state/epoch_cache.rs index 0653aaa3d..ee3a67813 100644 --- a/eth2/types/src/beacon_state/epoch_cache.rs +++ b/eth2/types/src/beacon_state/epoch_cache.rs @@ -36,9 +36,16 @@ impl EpochCache { let mut attestation_duty_map: AttestationDutyMap = HashMap::new(); let mut shard_committee_index_map: ShardCommitteeIndexMap = HashMap::new(); + let shuffling = + state.get_shuffling_for_slot(epoch.start_slot(spec.epoch_length), false, spec)?; + for (epoch_committeess_index, slot) in epoch.slot_iter(spec.epoch_length).enumerate() { - let slot_committees = - state.calculate_crosslink_committees_at_slot(slot, false, spec)?; + let slot_committees = state.calculate_crosslink_committees_at_slot( + slot, + false, + shuffling.clone(), + spec, + )?; for (slot_committees_index, (committee, shard)) in slot_committees.iter().enumerate() { // Empty committees are not permitted. From c49f425fe81f80ebecd0e061a97c33313c99f17d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 24 Feb 2019 18:25:17 +1300 Subject: [PATCH 016/132] Tidy, add comments to `BeaconState` --- eth2/types/src/beacon_state.rs | 73 +++++++++++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 5 deletions(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 03d2fd52f..88f5422f8 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -192,7 +192,7 @@ impl BeaconState { .map(|deposit| &deposit.deposit_data) .collect(); - genesis_state.process_deposits_optimized(deposit_data, spec); + genesis_state.process_deposits(deposit_data, spec); trace!("Processed genesis deposits."); @@ -243,6 +243,7 @@ impl BeaconState { Ok(()) } + /// Converts a `RelativeEpoch` into an `Epoch` with respect to the epoch of this state. fn absolute_epoch(&self, relative_epoch: RelativeEpoch, spec: &ChainSpec) -> Epoch { match relative_epoch { RelativeEpoch::Previous => self.previous_epoch(spec), @@ -251,6 +252,10 @@ impl BeaconState { } } + /// Converts an `Epoch` into a `RelativeEpoch` with respect to the epoch of this state. + /// + /// Returns an error if the given `epoch` not "previous", "current" or "next" compared to the + /// epoch of this tate. fn relative_epoch(&self, epoch: Epoch, spec: &ChainSpec) -> Result { match epoch { e if e == self.current_epoch(spec) => Ok(RelativeEpoch::Current), @@ -260,6 +265,16 @@ impl BeaconState { } } + /// Advances the cache for this state into the next epoch. + /// + /// This should be used if the `slot` of this state is advanced beyond an epoch boundary. + /// + /// The `Next` cache becomes the `Current` and the `Current` cache becomes the `Previous`. The + /// `Previous` cache is abandoned. + /// + /// Care should be taken to update the `Current` epoch in case a registry update is performed + /// -- `Next` epoch is always _without_ a registry change. If you perform a registry update, + /// you should rebuild the `Current` cache so it uses the new seed. pub fn advance_caches(&mut self) { let previous_cache_index = self.cache_index(RelativeEpoch::Previous); @@ -269,6 +284,7 @@ impl BeaconState { self.cache_index_offset %= CACHED_EPOCHS; } + /// Returns the index of `self.caches` for some `RelativeEpoch`. fn cache_index(&self, relative_epoch: RelativeEpoch) -> usize { let base_index = match relative_epoch { RelativeEpoch::Current => 1, @@ -279,6 +295,8 @@ impl BeaconState { (base_index + self.cache_index_offset) % CACHED_EPOCHS } + /// Returns the cache for some `RelativeEpoch`. Returns an error if the cache has not been + /// initialized. fn cache<'a>(&'a self, relative_epoch: RelativeEpoch) -> Result<&'a EpochCache, Error> { let cache = &self.caches[self.cache_index(relative_epoch)]; @@ -356,10 +374,11 @@ impl BeaconState { } /// Shuffle ``validators`` into crosslink committees seeded by ``seed`` and ``epoch``. + /// /// Return a list of ``committees_per_epoch`` committees where each /// committee is itself a list of validator indices. /// - /// Spec v0.1 + /// Spec v0.2.0 pub(crate) fn get_shuffling( &self, seed: Hash256, @@ -428,6 +447,9 @@ impl BeaconState { self.get_epoch_committee_count(current_active_validators.len(), spec) } + /// Return the index root at a recent `epoch`. + /// + /// Spec v0.2.0 pub fn get_active_index_root(&self, epoch: Epoch, spec: &ChainSpec) -> Option { let current_epoch = self.current_epoch(spec); @@ -443,7 +465,7 @@ impl BeaconState { } } - /// Generate a seed for the given ``epoch``. + /// Generate a seed for the given `epoch`. /// /// Spec v0.2.0 pub fn generate_seed(&self, epoch: Epoch, spec: &ChainSpec) -> Result { @@ -465,6 +487,11 @@ impl BeaconState { Ok(Hash256::from(&hash(&input[..])[..])) } + /// Returns the crosslink committees for some slot. + /// + /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. + /// + /// Spec v0.2.0 pub fn get_crosslink_committees_at_slot( &self, slot: Slot, @@ -479,6 +506,11 @@ impl BeaconState { Ok(&cache.committees[slot_offset.as_usize()]) } + /// Returns the crosslink committees for some slot. + /// + /// Utilizes the cache and will fail if the appropriate cache is not initialized. + /// + /// Spec v0.2.0 pub(crate) fn get_shuffling_for_slot( &self, slot: Slot, @@ -492,6 +524,18 @@ impl BeaconState { .ok_or_else(|| Error::UnableToShuffle) } + /// Returns the following params for the given slot: + /// + /// - epoch committee count + /// - epoch seed + /// - calculation epoch + /// - start shard + /// + /// In the spec, this functionality is included in the `get_crosslink_committees_at_slot(..)` + /// function. It is separated here to allow the division of shuffling and committee building, + /// as is required for efficient operations. + /// + /// Spec v0.2.0 pub(crate) fn get_committee_params_at_slot( &self, slot: Slot, @@ -555,7 +599,8 @@ impl BeaconState { /// Note: There are two possible shufflings for crosslink committees for a /// `slot` in the next epoch: with and without a `registry_change` /// - /// Note: this is equivalent to the `get_crosslink_committees_at_slot` function in the spec. + /// Note: does not utilize the cache, `get_crosslink_committees_at_slot` is an equivalent + /// function which uses the cache. /// /// Spec v0.2.0 pub(crate) fn calculate_crosslink_committees_at_slot( @@ -589,6 +634,8 @@ impl BeaconState { /// /// Only reads the current epoch. /// + /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. + /// /// Spec v0.2.0 pub fn attestation_slot_and_shard_for_validator( &self, @@ -745,7 +792,14 @@ impl BeaconState { self.validator_registry_update_epoch = current_epoch; } - pub fn process_deposits_optimized( + /// Process multiple deposits in sequence. + /// + /// Builds a hashmap of validator pubkeys to validator index and passes it to each successive + /// call to `process_deposit(..)`. This requires much less computation than successive calls to + /// `process_deposits(..)` without the hashmap. + /// + /// Spec v0.2.0 + pub fn process_deposits( &mut self, deposits: Vec<&DepositData>, spec: &ChainSpec, @@ -775,6 +829,10 @@ impl BeaconState { /// Process a validator deposit, returning the validator index if the deposit is valid. /// + /// Optionally accepts a hashmap of all validator pubkeys to their validator index. Without + /// this hashmap, each call to `process_deposits` requires an iteration though + /// `self.validator_registry`. This becomes highly inefficient at scale. + /// /// Spec v0.2.0 pub fn process_deposit( &mut self, @@ -1058,6 +1116,11 @@ impl BeaconState { Ok(all_participants) } + /// Returns the list of validator indices which participiated in the attestation. + /// + /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. + /// + /// Spec v0.2.0 pub fn get_attestation_participants( &self, attestation_data: &AttestationData, From ab10cbbdb5ae6d706a59e454658ded89693b2061 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 24 Feb 2019 18:52:12 +1300 Subject: [PATCH 017/132] Fix clippy lints, small typos --- beacon_node/beacon_chain/src/beacon_chain.rs | 7 +------ .../state_processing/src/epoch_processable.rs | 1 - eth2/types/src/beacon_state.rs | 20 +++++++++---------- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 9ee55e5a3..d0e7b7f8f 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -495,12 +495,7 @@ where // TODO: this is a first-in-best-dressed scenario that is not ideal; fork_choice should be // run instead. if self.head().beacon_block_root == parent_block_root { - self.update_canonical_head( - block.clone(), - block_root.clone(), - state.clone(), - state_root, - ); + self.update_canonical_head(block.clone(), block_root, state.clone(), state_root); // Update the local state variable. *self.state.write() = state.clone(); } diff --git a/eth2/state_processing/src/epoch_processable.rs b/eth2/state_processing/src/epoch_processable.rs index 409d40a2c..9b6c98c86 100644 --- a/eth2/state_processing/src/epoch_processable.rs +++ b/eth2/state_processing/src/epoch_processable.rs @@ -658,7 +658,6 @@ fn winning_root( continue; } - // TODO: `cargo fmt` makes this rather ugly; tidy up. let attesting_validator_indices = attestations .iter() .try_fold::<_, _, Result<_, BeaconStateError>>(vec![], |mut acc, a| { diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 88f5422f8..e6dc9f6a6 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -222,10 +222,10 @@ impl BeaconState { ) -> Result<(), Error> { let cache_index = self.cache_index(relative_epoch); - if self.caches[cache_index].initialized == false { - self.force_build_epoch_cache(relative_epoch, spec) - } else { + if self.caches[cache_index].initialized { Ok(()) + } else { + self.force_build_epoch_cache(relative_epoch, spec) } } @@ -297,13 +297,13 @@ impl BeaconState { /// Returns the cache for some `RelativeEpoch`. Returns an error if the cache has not been /// initialized. - fn cache<'a>(&'a self, relative_epoch: RelativeEpoch) -> Result<&'a EpochCache, Error> { + fn cache(&self, relative_epoch: RelativeEpoch) -> Result<&EpochCache, Error> { let cache = &self.caches[self.cache_index(relative_epoch)]; - if cache.initialized == false { - Err(Error::EpochCacheUninitialized(relative_epoch)) - } else { + if cache.initialized { Ok(cache) + } else { + Err(Error::EpochCacheUninitialized(relative_epoch)) } } @@ -548,7 +548,7 @@ impl BeaconState { let next_epoch = self.next_epoch(spec); if epoch == current_epoch { - trace!("get_crosslink_committees_at_slot: current_epoch"); + trace!("get_committee_params_at_slot: current_epoch"); Ok(( self.get_current_epoch_committee_count(spec), self.current_epoch_seed, @@ -556,7 +556,7 @@ impl BeaconState { self.current_epoch_start_shard, )) } else if epoch == previous_epoch { - trace!("get_crosslink_committees_at_slot: previous_epoch"); + trace!("get_committee_params_at_slot: previous_epoch"); Ok(( self.get_previous_epoch_committee_count(spec), self.previous_epoch_seed, @@ -564,7 +564,7 @@ impl BeaconState { self.previous_epoch_start_shard, )) } else if epoch == next_epoch { - trace!("get_crosslink_committees_at_slot: next_epoch"); + trace!("get_committee_params_at_slot: next_epoch"); let current_committees_per_epoch = self.get_current_epoch_committee_count(spec); let epochs_since_last_registry_update = current_epoch - self.validator_registry_update_epoch; From 27e7bbd72fe27d7d9d7440638a72509f77ae2583 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 25 Feb 2019 08:35:21 +1300 Subject: [PATCH 018/132] Fix typo in BeaconState Co-Authored-By: paulhauner --- eth2/types/src/beacon_state.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index e6dc9f6a6..0270bb87c 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -229,7 +229,7 @@ impl BeaconState { } } - /// Always builds an epoch cache, even if it is alread initialized. + /// Always builds an epoch cache, even if it is already initialized. pub fn force_build_epoch_cache( &mut self, relative_epoch: RelativeEpoch, From 4c3b0a65753d72c690652af20ef8b2f2f5c74ba5 Mon Sep 17 00:00:00 2001 From: Kirk Baird Date: Mon, 25 Feb 2019 10:38:04 +1100 Subject: [PATCH 019/132] Formatting --- .../beacon_chain/src/attestation_aggregator.rs | 18 +++++++----------- eth2/types/src/beacon_state.rs | 4 ++-- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_aggregator.rs b/beacon_node/beacon_chain/src/attestation_aggregator.rs index 3aa312c84..54f178068 100644 --- a/beacon_node/beacon_chain/src/attestation_aggregator.rs +++ b/beacon_node/beacon_chain/src/attestation_aggregator.rs @@ -129,17 +129,13 @@ impl AttestationAggregator { Some(validator_record) => validator_record, }; - if !free_attestation - .signature - .verify( - &signable_message, - cached_state.fork.get_domain( - cached_state.current_epoch(spec), - spec.domain_attestation, - ), - &validator_record.pubkey, - ) - { + if !free_attestation.signature.verify( + &signable_message, + cached_state + .fork + .get_domain(cached_state.current_epoch(spec), spec.domain_attestation), + &validator_record.pubkey, + ) { invalid_outcome!(Message::BadSignature); } diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index b8e9e1689..85b49bf01 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -2,8 +2,8 @@ use self::epoch_cache::EpochCache; use crate::test_utils::TestRandom; use crate::{ validator::StatusFlags, validator_registry::get_active_validator_indices, AttestationData, - Bitfield, ChainSpec, Crosslink, Deposit, DepositData, DepositInput, Epoch, Eth1Data, Eth1DataVote, Fork, - Hash256, PendingAttestation, PublicKey, Signature, Slot, Validator, + Bitfield, ChainSpec, Crosslink, Deposit, DepositData, DepositInput, Epoch, Eth1Data, + Eth1DataVote, Fork, Hash256, PendingAttestation, PublicKey, Signature, Slot, Validator, }; use bls::verify_proof_of_possession; use honey_badger_split::SplitExt; From 9a892bbdfd7974a302a569de301b47b70d74922a Mon Sep 17 00:00:00 2001 From: thojest Date: Mon, 25 Feb 2019 09:26:43 +0100 Subject: [PATCH 020/132] removed TestRandom import in beacon_state (lighthouse-246) --- eth2/types/src/beacon_state.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index ccce4bfa7..85b49bf01 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -13,7 +13,6 @@ use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; use std::collections::HashMap; use swap_or_not_shuffle::get_permutated_index; -use test_random_derive::TestRandom; mod epoch_cache; mod tests; From d7184345b89e073b99fd92f65e48ea3ce8fe9cf0 Mon Sep 17 00:00:00 2001 From: mjkeating Date: Mon, 25 Feb 2019 09:17:17 -0800 Subject: [PATCH 021/132] renamed the macro Hashtree to TreeHash --- eth2/types/src/attestation.rs | 4 ++-- eth2/types/src/attestation_data.rs | 4 ++-- eth2/types/src/attestation_data_and_custody_bit.rs | 4 ++-- eth2/types/src/attester_slashing.rs | 4 ++-- eth2/types/src/beacon_block.rs | 4 ++-- eth2/types/src/beacon_block_body.rs | 4 ++-- eth2/types/src/casper_slashing.rs | 4 ++-- eth2/types/src/crosslink.rs | 4 ++-- eth2/types/src/deposit.rs | 4 ++-- eth2/types/src/deposit_data.rs | 4 ++-- eth2/types/src/deposit_input.rs | 4 ++-- eth2/types/src/eth1_data.rs | 4 ++-- eth2/types/src/eth1_data_vote.rs | 4 ++-- eth2/types/src/exit.rs | 4 ++-- eth2/types/src/fork.rs | 4 ++-- eth2/types/src/pending_attestation.rs | 4 ++-- eth2/types/src/proposal_signed_data.rs | 4 ++-- eth2/types/src/proposer_slashing.rs | 4 ++-- eth2/types/src/shard_reassignment_record.rs | 4 ++-- eth2/types/src/slashable_attestation.rs | 4 ++-- eth2/types/src/slashable_vote_data.rs | 4 ++-- eth2/types/src/validator_registry_delta_block.rs | 4 ++-- eth2/utils/ssz_derive/src/lib.rs | 6 +++--- 23 files changed, 47 insertions(+), 47 deletions(-) diff --git a/eth2/types/src/attestation.rs b/eth2/types/src/attestation.rs index a8523dfe8..66140decb 100644 --- a/eth2/types/src/attestation.rs +++ b/eth2/types/src/attestation.rs @@ -3,9 +3,9 @@ use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; use ssz::TreeHash; -use ssz_derive::{Decode, Encode, Hashtree}; +use ssz_derive::{Decode, Encode, TreeHash}; -#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, Hashtree)] +#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, TreeHash)] pub struct Attestation { pub aggregation_bitfield: Bitfield, pub data: AttestationData, diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data.rs index d2da31102..868f9743a 100644 --- a/eth2/types/src/attestation_data.rs +++ b/eth2/types/src/attestation_data.rs @@ -3,7 +3,7 @@ use crate::{AttestationDataAndCustodyBit, Crosslink, Epoch, Hash256, Slot}; use rand::RngCore; use serde_derive::Serialize; use ssz::TreeHash; -use ssz_derive::{Decode, Encode, Hashtree}; +use ssz_derive::{Decode, Encode, TreeHash}; pub const SSZ_ATTESTION_DATA_LENGTH: usize = { 8 + // slot @@ -16,7 +16,7 @@ pub const SSZ_ATTESTION_DATA_LENGTH: usize = { 32 // justified_block_root }; -#[derive(Debug, Clone, PartialEq, Default, Serialize, Hash, Encode, Decode, Hashtree)] +#[derive(Debug, Clone, PartialEq, Default, Serialize, Hash, Encode, Decode, TreeHash)] pub struct AttestationData { pub slot: Slot, pub shard: u64, diff --git a/eth2/types/src/attestation_data_and_custody_bit.rs b/eth2/types/src/attestation_data_and_custody_bit.rs index 33b7f3715..9175863ae 100644 --- a/eth2/types/src/attestation_data_and_custody_bit.rs +++ b/eth2/types/src/attestation_data_and_custody_bit.rs @@ -2,9 +2,9 @@ use super::AttestationData; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, Hashtree}; +use ssz_derive::{Decode, Encode, TreeHash}; -#[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode, Hashtree)] +#[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode, TreeHash)] pub struct AttestationDataAndCustodyBit { pub data: AttestationData, pub custody_bit: bool, diff --git a/eth2/types/src/attester_slashing.rs b/eth2/types/src/attester_slashing.rs index 76c0c2b2c..96204edc5 100644 --- a/eth2/types/src/attester_slashing.rs +++ b/eth2/types/src/attester_slashing.rs @@ -1,9 +1,9 @@ use crate::{test_utils::TestRandom, SlashableAttestation}; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, Hashtree}; +use ssz_derive::{Decode, Encode, TreeHash}; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, Hashtree)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash)] pub struct AttesterSlashing { pub slashable_attestation_1: SlashableAttestation, pub slashable_attestation_2: SlashableAttestation, diff --git a/eth2/types/src/beacon_block.rs b/eth2/types/src/beacon_block.rs index d1dcd58df..9d769cbed 100644 --- a/eth2/types/src/beacon_block.rs +++ b/eth2/types/src/beacon_block.rs @@ -4,9 +4,9 @@ use bls::Signature; use rand::RngCore; use serde_derive::Serialize; use ssz::TreeHash; -use ssz_derive::{Decode, Encode, Hashtree}; +use ssz_derive::{Decode, Encode, TreeHash}; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, Hashtree)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash)] pub struct BeaconBlock { pub slot: Slot, pub parent_root: Hash256, diff --git a/eth2/types/src/beacon_block_body.rs b/eth2/types/src/beacon_block_body.rs index 62e8e1e90..915e6435c 100644 --- a/eth2/types/src/beacon_block_body.rs +++ b/eth2/types/src/beacon_block_body.rs @@ -2,9 +2,9 @@ use super::{Attestation, AttesterSlashing, Deposit, Exit, ProposerSlashing}; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, Hashtree}; +use ssz_derive::{Decode, Encode, TreeHash}; -#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, Hashtree)] +#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, TreeHash)] pub struct BeaconBlockBody { pub proposer_slashings: Vec, pub attester_slashings: Vec, diff --git a/eth2/types/src/casper_slashing.rs b/eth2/types/src/casper_slashing.rs index 5532a8b57..76a6515a2 100644 --- a/eth2/types/src/casper_slashing.rs +++ b/eth2/types/src/casper_slashing.rs @@ -2,9 +2,9 @@ use super::SlashableVoteData; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, Hashtree}; +use ssz_derive::{Decode, Encode, TreeHash}; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, Hashtree)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash)] pub struct CasperSlashing { pub slashable_vote_data_1: SlashableVoteData, pub slashable_vote_data_2: SlashableVoteData, diff --git a/eth2/types/src/crosslink.rs b/eth2/types/src/crosslink.rs index 45a92d787..9c7b8eeed 100644 --- a/eth2/types/src/crosslink.rs +++ b/eth2/types/src/crosslink.rs @@ -2,9 +2,9 @@ use crate::test_utils::TestRandom; use crate::{Epoch, Hash256}; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, Hashtree}; +use ssz_derive::{Decode, Encode, TreeHash}; -#[derive(Debug, Clone, PartialEq, Default, Serialize, Hash, Encode, Decode, Hashtree)] +#[derive(Debug, Clone, PartialEq, Default, Serialize, Hash, Encode, Decode, TreeHash)] pub struct Crosslink { pub epoch: Epoch, pub shard_block_root: Hash256, diff --git a/eth2/types/src/deposit.rs b/eth2/types/src/deposit.rs index 5a738d4c5..de485cd2b 100644 --- a/eth2/types/src/deposit.rs +++ b/eth2/types/src/deposit.rs @@ -2,9 +2,9 @@ use super::{DepositData, Hash256}; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, Hashtree}; +use ssz_derive::{Decode, Encode, TreeHash}; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, Hashtree)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash)] pub struct Deposit { pub branch: Vec, pub index: u64, diff --git a/eth2/types/src/deposit_data.rs b/eth2/types/src/deposit_data.rs index fa0bd78b6..6ac40c0f8 100644 --- a/eth2/types/src/deposit_data.rs +++ b/eth2/types/src/deposit_data.rs @@ -2,9 +2,9 @@ use super::DepositInput; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, Hashtree}; +use ssz_derive::{Decode, Encode, TreeHash}; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, Hashtree)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash)] pub struct DepositData { pub amount: u64, pub timestamp: u64, diff --git a/eth2/types/src/deposit_input.rs b/eth2/types/src/deposit_input.rs index 24146ad44..5baa2226d 100644 --- a/eth2/types/src/deposit_input.rs +++ b/eth2/types/src/deposit_input.rs @@ -3,9 +3,9 @@ use crate::test_utils::TestRandom; use bls::{PublicKey, Signature}; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, Hashtree}; +use ssz_derive::{Decode, Encode, TreeHash}; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, Hashtree)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash)] pub struct DepositInput { pub pubkey: PublicKey, pub withdrawal_credentials: Hash256, diff --git a/eth2/types/src/eth1_data.rs b/eth2/types/src/eth1_data.rs index cdf746fae..2cbfad770 100644 --- a/eth2/types/src/eth1_data.rs +++ b/eth2/types/src/eth1_data.rs @@ -2,10 +2,10 @@ use super::Hash256; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, Hashtree}; +use ssz_derive::{Decode, Encode, TreeHash}; // Note: this is refer to as DepositRootVote in specs -#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, Hashtree)] +#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, TreeHash)] pub struct Eth1Data { pub deposit_root: Hash256, pub block_hash: Hash256, diff --git a/eth2/types/src/eth1_data_vote.rs b/eth2/types/src/eth1_data_vote.rs index acf77ffac..e2a98df3c 100644 --- a/eth2/types/src/eth1_data_vote.rs +++ b/eth2/types/src/eth1_data_vote.rs @@ -2,10 +2,10 @@ use super::Eth1Data; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, Hashtree}; +use ssz_derive::{Decode, Encode, TreeHash}; // Note: this is refer to as DepositRootVote in specs -#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, Hashtree)] +#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, TreeHash)] pub struct Eth1DataVote { pub eth1_data: Eth1Data, pub vote_count: u64, diff --git a/eth2/types/src/exit.rs b/eth2/types/src/exit.rs index c0af761fd..f564c8bb5 100644 --- a/eth2/types/src/exit.rs +++ b/eth2/types/src/exit.rs @@ -2,9 +2,9 @@ use crate::{test_utils::TestRandom, Epoch}; use bls::Signature; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, Hashtree}; +use ssz_derive::{Decode, Encode, TreeHash}; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, Hashtree)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash)] pub struct Exit { pub epoch: Epoch, pub validator_index: u64, diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs index c6c2b27ff..35285d6ed 100644 --- a/eth2/types/src/fork.rs +++ b/eth2/types/src/fork.rs @@ -1,9 +1,9 @@ use crate::{test_utils::TestRandom, Epoch}; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, Hashtree}; +use ssz_derive::{Decode, Encode, TreeHash}; -#[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode, Hashtree)] +#[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode, TreeHash)] pub struct Fork { pub previous_version: u64, pub current_version: u64, diff --git a/eth2/types/src/pending_attestation.rs b/eth2/types/src/pending_attestation.rs index b6f152321..53f1868d6 100644 --- a/eth2/types/src/pending_attestation.rs +++ b/eth2/types/src/pending_attestation.rs @@ -2,9 +2,9 @@ use crate::test_utils::TestRandom; use crate::{AttestationData, Bitfield, Slot}; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, Hashtree}; +use ssz_derive::{Decode, Encode, TreeHash}; -#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, Hashtree)] +#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, TreeHash)] pub struct PendingAttestation { pub aggregation_bitfield: Bitfield, pub data: AttestationData, diff --git a/eth2/types/src/proposal_signed_data.rs b/eth2/types/src/proposal_signed_data.rs index 53f04956b..164192cc9 100644 --- a/eth2/types/src/proposal_signed_data.rs +++ b/eth2/types/src/proposal_signed_data.rs @@ -2,9 +2,9 @@ use crate::test_utils::TestRandom; use crate::{Hash256, Slot}; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, Hashtree}; +use ssz_derive::{Decode, Encode, TreeHash}; -#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, Hashtree)] +#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, TreeHash)] pub struct ProposalSignedData { pub slot: Slot, pub shard: u64, diff --git a/eth2/types/src/proposer_slashing.rs b/eth2/types/src/proposer_slashing.rs index 739c5f84d..fc5276dfe 100644 --- a/eth2/types/src/proposer_slashing.rs +++ b/eth2/types/src/proposer_slashing.rs @@ -3,9 +3,9 @@ use crate::test_utils::TestRandom; use bls::Signature; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, Hashtree}; +use ssz_derive::{Decode, Encode, TreeHash}; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, Hashtree)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash)] pub struct ProposerSlashing { pub proposer_index: u64, pub proposal_data_1: ProposalSignedData, diff --git a/eth2/types/src/shard_reassignment_record.rs b/eth2/types/src/shard_reassignment_record.rs index 4fa685d26..e7a22e1fa 100644 --- a/eth2/types/src/shard_reassignment_record.rs +++ b/eth2/types/src/shard_reassignment_record.rs @@ -1,9 +1,9 @@ use crate::{test_utils::TestRandom, Slot}; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, Hashtree}; +use ssz_derive::{Decode, Encode, TreeHash}; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, Hashtree)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash)] pub struct ShardReassignmentRecord { pub validator_index: u64, pub shard: u64, diff --git a/eth2/types/src/slashable_attestation.rs b/eth2/types/src/slashable_attestation.rs index 64507994b..3315f1d35 100644 --- a/eth2/types/src/slashable_attestation.rs +++ b/eth2/types/src/slashable_attestation.rs @@ -1,9 +1,9 @@ use crate::{test_utils::TestRandom, AggregateSignature, AttestationData, Bitfield}; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, Hashtree}; +use ssz_derive::{Decode, Encode, TreeHash}; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, Hashtree)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash)] pub struct SlashableAttestation { pub validator_indices: Vec, pub data: AttestationData, diff --git a/eth2/types/src/slashable_vote_data.rs b/eth2/types/src/slashable_vote_data.rs index 53742372c..e8ac61dbe 100644 --- a/eth2/types/src/slashable_vote_data.rs +++ b/eth2/types/src/slashable_vote_data.rs @@ -4,9 +4,9 @@ use crate::test_utils::TestRandom; use bls::AggregateSignature; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, Hashtree}; +use ssz_derive::{Decode, Encode, TreeHash}; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, Hashtree)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash)] pub struct SlashableVoteData { pub custody_bit_0_indices: Vec, pub custody_bit_1_indices: Vec, diff --git a/eth2/types/src/validator_registry_delta_block.rs b/eth2/types/src/validator_registry_delta_block.rs index c42300cb4..86f5718d6 100644 --- a/eth2/types/src/validator_registry_delta_block.rs +++ b/eth2/types/src/validator_registry_delta_block.rs @@ -2,10 +2,10 @@ use crate::{test_utils::TestRandom, Hash256, Slot}; use bls::PublicKey; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, Hashtree}; +use ssz_derive::{Decode, Encode, TreeHash}; // The information gathered from the PoW chain validator registration function. -#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, Hashtree)] +#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, TreeHash)] pub struct ValidatorRegistryDeltaBlock { pub latest_registry_delta_root: Hash256, pub validator_index: u32, diff --git a/eth2/utils/ssz_derive/src/lib.rs b/eth2/utils/ssz_derive/src/lib.rs index 09e1b2dc7..ac66526fe 100644 --- a/eth2/utils/ssz_derive/src/lib.rs +++ b/eth2/utils/ssz_derive/src/lib.rs @@ -2,7 +2,7 @@ //! //! - `#[derive(Encode)]` //! - `#[derive(Decode)]` -//! - `#[derive(Hashtree)]` +//! - `#[derive(TreeHash)]` //! //! These macros provide SSZ encoding/decoding for a `struct`. Fields are encoded/decoded in the //! order they are defined. @@ -131,8 +131,8 @@ pub fn ssz_decode_derive(input: TokenStream) -> TokenStream { /// Implements `ssz::TreeHash` for some `struct`. /// /// Fields are processed in the order they are defined. -#[proc_macro_derive(Hashtree)] -pub fn ssz_hashtree_derive(input: TokenStream) -> TokenStream { +#[proc_macro_derive(TreeHash)] +pub fn ssz_tree_hash_derive(input: TokenStream) -> TokenStream { let item = parse_macro_input!(input as DeriveInput); let name = &item.ident; From 59d6b1fdd07925e6064a7ddf6809f75ae6a4a787 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 16:25:08 +1300 Subject: [PATCH 022/132] Move `BeaconStateBuilder` out of test mods This allows it to be accessed by other crates/tests. --- eth2/types/src/beacon_state.rs | 3 + eth2/types/src/beacon_state/builder.rs | 215 +++++++++++++++++++++++++ eth2/types/src/beacon_state/tests.rs | 67 +------- 3 files changed, 221 insertions(+), 64 deletions(-) create mode 100644 eth2/types/src/beacon_state/builder.rs diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 85b49bf01..956877a61 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -14,6 +14,9 @@ use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; use std::collections::HashMap; use swap_or_not_shuffle::get_permutated_index; +pub use builder::BeaconStateBuilder; + +mod builder; mod epoch_cache; mod tests; diff --git a/eth2/types/src/beacon_state/builder.rs b/eth2/types/src/beacon_state/builder.rs new file mode 100644 index 000000000..6258d819d --- /dev/null +++ b/eth2/types/src/beacon_state/builder.rs @@ -0,0 +1,215 @@ +use crate::*; +use bls::create_proof_of_possession; + +/// Builds a `BeaconState` for use in testing or benchmarking. +/// +/// Building the `BeaconState` is a three step processes: +/// +/// 1. Create a new `BeaconStateBuilder`. +/// 2. Run the `genesis` function to generate a new BeaconState. +/// 3. (Optional) Use builder functions to modify the `BeaconState`. +/// 4. Call `build()` to obtain a cloned `BeaconState`. +/// +/// Step (2) is necessary because step (3) requires an existing `BeaconState` object. (2) is not +/// included in (1) to allow for modifying params before generating the `BeaconState` (e.g., the +/// spec). +/// +/// Step (4) produces a clone of the BeaconState and doesn't consume the `BeaconStateBuilder` to +/// allow access to the `keypairs` and `spec`. +pub struct BeaconStateBuilder { + pub state: Option, + pub genesis_time: u64, + pub initial_validator_deposits: Vec, + pub latest_eth1_data: Eth1Data, + pub spec: ChainSpec, + pub keypairs: Vec, +} + +impl BeaconStateBuilder { + /// Create a new builder with the given number of validators. + pub fn with_random_validators(validator_count: usize) -> Self { + let genesis_time = 10_000_000; + let keypairs: Vec = (0..validator_count) + .collect::>() + .iter() + .map(|_| Keypair::random()) + .collect(); + let initial_validator_deposits = keypairs + .iter() + .map(|keypair| Deposit { + branch: vec![], // branch verification is not specified. + index: 0, // index verification is not specified. + deposit_data: DepositData { + amount: 32_000_000_000, // 32 ETH (in Gwei) + timestamp: genesis_time - 1, + deposit_input: DepositInput { + pubkey: keypair.pk.clone(), + withdrawal_credentials: Hash256::zero(), // Withdrawal not possible. + proof_of_possession: create_proof_of_possession(&keypair), + }, + }, + }) + .collect(); + let latest_eth1_data = Eth1Data { + deposit_root: Hash256::zero(), + block_hash: Hash256::zero(), + }; + let spec = ChainSpec::foundation(); + + Self { + state: None, + genesis_time, + initial_validator_deposits, + latest_eth1_data, + spec, + keypairs, + } + } + + /// Runs the `BeaconState::genesis` function and produces a `BeaconState`. + pub fn genesis(&mut self) -> Result<(), BeaconStateError> { + let state = BeaconState::genesis( + self.genesis_time, + self.initial_validator_deposits.clone(), + self.latest_eth1_data.clone(), + &self.spec, + )?; + + self.state = Some(state); + + Ok(()) + } + + /// Sets the `BeaconState` to be in the last slot of the given epoch. + /// + /// Sets all justification/finalization parameters to be be as "perfect" as possible (i.e., + /// highest justified and finalized slots, full justification bitfield, etc). + pub fn teleport_to_end_of_epoch(&mut self, epoch: Epoch) { + let state = self.state.as_mut().expect("Genesis required"); + + let slot = epoch.end_slot(self.spec.epoch_length); + + state.slot = slot; + state.validator_registry_update_epoch = epoch - 1; + + state.previous_calculation_epoch = epoch - 1; + state.current_calculation_epoch = epoch; + + state.previous_epoch_seed = Hash256::from(&b"previous_seed"[..]); + state.current_epoch_seed = Hash256::from(&b"current_seed"[..]); + + state.previous_justified_epoch = epoch - 2; + state.justified_epoch = epoch - 1; + state.justification_bitfield = u64::max_value(); + state.finalized_epoch = epoch - 1; + } + + /// Creates a full set of attestations for the `BeaconState`. Each attestation has full + /// participation from its committee and references the expected beacon_block hashes. + /// + /// These attestations should be fully conducive to justification and finalization. + pub fn insert_attestations(&mut self) { + let state = self.state.as_mut().expect("Genesis required"); + + state + .build_epoch_cache(RelativeEpoch::Previous, &self.spec) + .unwrap(); + state + .build_epoch_cache(RelativeEpoch::Current, &self.spec) + .unwrap(); + + let current_epoch = state.current_epoch(&self.spec); + let previous_epoch = state.previous_epoch(&self.spec); + let current_epoch_depth = + (state.slot - current_epoch.end_slot(self.spec.epoch_length)).as_usize(); + + let previous_epoch_slots = previous_epoch.slot_iter(self.spec.epoch_length); + let current_epoch_slots = current_epoch + .slot_iter(self.spec.epoch_length) + .take(current_epoch_depth); + + for slot in previous_epoch_slots.chain(current_epoch_slots) { + let committees = state + .get_crosslink_committees_at_slot(slot, &self.spec) + .unwrap() + .clone(); + + for (committee, shard) in committees { + state + .latest_attestations + .push(committee_to_pending_attestation( + state, &committee, shard, slot, &self.spec, + )) + } + } + } + + /// Returns a cloned `BeaconState`. + pub fn build(&self) -> Result { + match &self.state { + Some(state) => Ok(state.clone()), + None => panic!("Genesis required"), + } + } +} + +/// Builds a valid PendingAttestation with full participation for some committee. +fn committee_to_pending_attestation( + state: &BeaconState, + committee: &Vec, + shard: u64, + slot: Slot, + spec: &ChainSpec, +) -> PendingAttestation { + let current_epoch = state.current_epoch(spec); + let previous_epoch = state.previous_epoch(spec); + + let mut aggregation_bitfield = Bitfield::new(); + let mut custody_bitfield = Bitfield::new(); + + for (i, _) in committee.iter().enumerate() { + aggregation_bitfield.set(i, true); + custody_bitfield.set(i, true); + } + + let is_previous_epoch = state.slot.epoch(spec.epoch_length) != slot.epoch(spec.epoch_length); + + let justified_epoch = if is_previous_epoch { + state.previous_justified_epoch + } else { + state.justified_epoch + }; + + let epoch_boundary_root = if is_previous_epoch { + *state + .get_block_root(previous_epoch.start_slot(spec.epoch_length), spec) + .unwrap() + } else { + *state + .get_block_root(current_epoch.start_slot(spec.epoch_length), spec) + .unwrap() + }; + + let justified_block_root = *state + .get_block_root(justified_epoch.start_slot(spec.epoch_length), &spec) + .unwrap(); + + PendingAttestation { + aggregation_bitfield, + data: AttestationData { + slot, + shard, + beacon_block_root: *state.get_block_root(slot, spec).unwrap(), + epoch_boundary_root, + shard_block_root: Hash256::zero(), + latest_crosslink: Crosslink { + epoch: slot.epoch(spec.epoch_length), + shard_block_root: Hash256::zero(), + }, + justified_epoch, + justified_block_root, + }, + custody_bitfield, + inclusion_slot: slot, + } +} diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs index a4a43a8ed..7f43ae9a1 100644 --- a/eth2/types/src/beacon_state/tests.rs +++ b/eth2/types/src/beacon_state/tests.rs @@ -2,73 +2,12 @@ use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; -use crate::{ - BeaconState, BeaconStateError, ChainSpec, Deposit, DepositData, DepositInput, Eth1Data, - Hash256, Keypair, -}; -use bls::create_proof_of_possession; +use crate::{BeaconState, ChainSpec}; use ssz::{ssz_encode, Decodable}; -struct BeaconStateTestBuilder { - pub genesis_time: u64, - pub initial_validator_deposits: Vec, - pub latest_eth1_data: Eth1Data, - pub spec: ChainSpec, - pub keypairs: Vec, -} - -impl BeaconStateTestBuilder { - pub fn with_random_validators(validator_count: usize) -> Self { - let genesis_time = 10_000_000; - let keypairs: Vec = (0..validator_count) - .collect::>() - .iter() - .map(|_| Keypair::random()) - .collect(); - let initial_validator_deposits = keypairs - .iter() - .map(|keypair| Deposit { - branch: vec![], // branch verification is not specified. - index: 0, // index verification is not specified. - deposit_data: DepositData { - amount: 32_000_000_000, // 32 ETH (in Gwei) - timestamp: genesis_time - 1, - deposit_input: DepositInput { - pubkey: keypair.pk.clone(), - withdrawal_credentials: Hash256::zero(), // Withdrawal not possible. - proof_of_possession: create_proof_of_possession(&keypair), - }, - }, - }) - .collect(); - let latest_eth1_data = Eth1Data { - deposit_root: Hash256::zero(), - block_hash: Hash256::zero(), - }; - let spec = ChainSpec::foundation(); - - Self { - genesis_time, - initial_validator_deposits, - latest_eth1_data, - spec, - keypairs, - } - } - - pub fn build(&self) -> Result { - BeaconState::genesis( - self.genesis_time, - self.initial_validator_deposits.clone(), - self.latest_eth1_data.clone(), - &self.spec, - ) - } -} - #[test] pub fn can_produce_genesis_block() { - let builder = BeaconStateTestBuilder::with_random_validators(2); + let builder = BeaconStateBuilder::with_random_validators(2); builder.build().unwrap(); } @@ -79,7 +18,7 @@ pub fn can_produce_genesis_block() { pub fn get_attestation_participants_consistency() { let mut rng = XorShiftRng::from_seed([42; 16]); - let mut builder = BeaconStateTestBuilder::with_random_validators(8); + let mut builder = BeaconStateBuilder::with_random_validators(8); builder.spec = ChainSpec::few_validators(); let mut state = builder.build().unwrap(); From 53663e54b5d23622776fc294a21b4263ec5cf602 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 16:26:06 +1300 Subject: [PATCH 023/132] Fix error with get_permutated_index. Error types were modified. Error involved shuffling with the _value_of active validator indicies, not an _index_. --- eth2/types/src/beacon_state.rs | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 956877a61..f8d71c324 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -7,7 +7,7 @@ use crate::{ }; use bls::verify_proof_of_possession; use honey_badger_split::SplitExt; -use log::{debug, trace}; +use log::{debug, trace, error}; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; @@ -387,12 +387,13 @@ impl BeaconState { seed: Hash256, epoch: Epoch, spec: &ChainSpec, - ) -> Option>> { + ) -> Result>, Error> { let active_validator_indices = get_active_validator_indices(&self.validator_registry, epoch); if active_validator_indices.is_empty() { - return None; + error!("get_shuffling: no validators."); + return Err(Error::InsufficientValidators); } let committees_per_epoch = @@ -405,22 +406,21 @@ impl BeaconState { ); let mut shuffled_active_validator_indices = vec![0; active_validator_indices.len()]; - for &i in &active_validator_indices { + for (i, _) in active_validator_indices.iter().enumerate() { let shuffled_i = get_permutated_index( i, active_validator_indices.len(), &seed[..], spec.shuffle_round_count, - )?; + ) + .ok_or_else(|| Error::UnableToShuffle)?; shuffled_active_validator_indices[i] = active_validator_indices[shuffled_i] } - Some( - shuffled_active_validator_indices - .honey_badger_split(committees_per_epoch as usize) - .map(|slice: &[usize]| slice.to_vec()) - .collect(), - ) + Ok(shuffled_active_validator_indices + .honey_badger_split(committees_per_epoch as usize) + .map(|slice: &[usize]| slice.to_vec()) + .collect()) } /// Return the number of committees in the previous epoch. @@ -525,7 +525,6 @@ impl BeaconState { self.get_committee_params_at_slot(slot, registry_change, spec)?; self.get_shuffling(seed, shuffling_epoch, spec) - .ok_or_else(|| Error::UnableToShuffle) } /// Returns the following params for the given slot: From 906131f8826bac9f338075cb0c045d3739dc2948 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 16:27:42 +1300 Subject: [PATCH 024/132] Add tests and benches for epoch processing --- eth2/state_processing/Cargo.toml | 9 ++++++ eth2/state_processing/benches/benches.rs | 28 +++++++++++++++++++ .../state_processing/src/epoch_processable.rs | 10 ++----- .../src/epoch_processable/tests.rs | 21 ++++++++++++++ 4 files changed, 60 insertions(+), 8 deletions(-) create mode 100644 eth2/state_processing/benches/benches.rs create mode 100644 eth2/state_processing/src/epoch_processable/tests.rs diff --git a/eth2/state_processing/Cargo.toml b/eth2/state_processing/Cargo.toml index b6b0ea57c..85ac8b42a 100644 --- a/eth2/state_processing/Cargo.toml +++ b/eth2/state_processing/Cargo.toml @@ -4,6 +4,15 @@ version = "0.1.0" authors = ["Paul Hauner "] edition = "2018" +[[bench]] +name = "benches" +harness = false + +[dev-dependencies] +criterion = "0.2" +test_harness = { path = "../../beacon_node/beacon_chain/test_harness" } +env_logger = "0.6.0" + [dependencies] hashing = { path = "../utils/hashing" } int_to_bytes = { path = "../utils/int_to_bytes" } diff --git a/eth2/state_processing/benches/benches.rs b/eth2/state_processing/benches/benches.rs new file mode 100644 index 000000000..29f9a1e86 --- /dev/null +++ b/eth2/state_processing/benches/benches.rs @@ -0,0 +1,28 @@ +use criterion::Criterion; +use criterion::{black_box, criterion_group, criterion_main}; +// use env_logger::{Builder, Env}; +use state_processing::SlotProcessable; +use types::{beacon_state::BeaconStateBuilder, ChainSpec, Hash256}; + +fn epoch_processing(c: &mut Criterion) { + // Builder::from_env(Env::default().default_filter_or("debug")).init(); + + let mut builder = BeaconStateBuilder::with_random_validators(8); + builder.spec = ChainSpec::few_validators(); + + builder.genesis().unwrap(); + builder.teleport_to_end_of_epoch(builder.spec.genesis_epoch + 4); + + let state = builder.build().unwrap(); + + c.bench_function("epoch processing", move |b| { + let spec = &builder.spec; + b.iter_with_setup( + || state.clone(), + |mut state| black_box(state.per_slot_processing(Hash256::zero(), spec).unwrap()), + ) + }); +} + +criterion_group!(benches, epoch_processing,); +criterion_main!(benches); diff --git a/eth2/state_processing/src/epoch_processable.rs b/eth2/state_processing/src/epoch_processable.rs index 9b6c98c86..2cee455a3 100644 --- a/eth2/state_processing/src/epoch_processable.rs +++ b/eth2/state_processing/src/epoch_processable.rs @@ -9,6 +9,8 @@ use types::{ Crosslink, Epoch, Hash256, InclusionError, PendingAttestation, RelativeEpoch, }; +mod tests; + macro_rules! safe_add_assign { ($a: expr, $b: expr) => { $a = $a.saturating_add($b); @@ -725,11 +727,3 @@ impl From for WinningRootError { WinningRootError::BeaconStateError(e) } } - -#[cfg(test)] -mod tests { - #[test] - fn it_works() { - assert_eq!(2 + 2, 4); - } -} diff --git a/eth2/state_processing/src/epoch_processable/tests.rs b/eth2/state_processing/src/epoch_processable/tests.rs new file mode 100644 index 000000000..353672fd1 --- /dev/null +++ b/eth2/state_processing/src/epoch_processable/tests.rs @@ -0,0 +1,21 @@ +#![cfg(test)] +use crate::EpochProcessable; +use env_logger::{Builder, Env}; +use types::beacon_state::BeaconStateBuilder; +use types::*; + +#[test] +fn runs_without_error() { + Builder::from_env(Env::default().default_filter_or("error")).init(); + + let mut builder = BeaconStateBuilder::with_random_validators(8); + builder.spec = ChainSpec::few_validators(); + + builder.genesis().unwrap(); + builder.teleport_to_end_of_epoch(builder.spec.genesis_epoch + 4); + + let mut state = builder.build().unwrap(); + + let spec = &builder.spec; + state.per_epoch_processing(spec).unwrap(); +} From 9f846eda2664e2c0e91b90507e84a16ab4ee5585 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 16:49:17 +1300 Subject: [PATCH 025/132] Fix clippy lint --- eth2/types/src/beacon_state/builder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth2/types/src/beacon_state/builder.rs b/eth2/types/src/beacon_state/builder.rs index 6258d819d..7273f3658 100644 --- a/eth2/types/src/beacon_state/builder.rs +++ b/eth2/types/src/beacon_state/builder.rs @@ -156,7 +156,7 @@ impl BeaconStateBuilder { /// Builds a valid PendingAttestation with full participation for some committee. fn committee_to_pending_attestation( state: &BeaconState, - committee: &Vec, + committee: &[usize], shard: u64, slot: Slot, spec: &ChainSpec, From e97b554485d12698cd8daae9ae6bba752115f891 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 16:50:04 +1300 Subject: [PATCH 026/132] Run rustfmt --- eth2/types/src/beacon_state.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index f8d71c324..f7df3b0ab 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -7,7 +7,7 @@ use crate::{ }; use bls::verify_proof_of_possession; use honey_badger_split::SplitExt; -use log::{debug, trace, error}; +use log::{debug, error, trace}; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; From 65d303c50090c717542e5d9da43f65df472783f1 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 17:20:43 +1300 Subject: [PATCH 027/132] Fix failing tests --- eth2/types/src/beacon_state/tests.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs index 7f43ae9a1..01b4be803 100644 --- a/eth2/types/src/beacon_state/tests.rs +++ b/eth2/types/src/beacon_state/tests.rs @@ -7,7 +7,8 @@ use ssz::{ssz_encode, Decodable}; #[test] pub fn can_produce_genesis_block() { - let builder = BeaconStateBuilder::with_random_validators(2); + let mut builder = BeaconStateBuilder::with_random_validators(2); + builder.genesis().unwrap(); builder.build().unwrap(); } @@ -21,6 +22,8 @@ pub fn get_attestation_participants_consistency() { let mut builder = BeaconStateBuilder::with_random_validators(8); builder.spec = ChainSpec::few_validators(); + builder.genesis().unwrap(); + let mut state = builder.build().unwrap(); let spec = builder.spec.clone(); From 0fe3a81c9ee1963128becfea31fb6734f52092fb Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 19:30:51 +1300 Subject: [PATCH 028/132] Add benches to `swap_or_not_shuffle` --- eth2/utils/swap_or_not_shuffle/Cargo.toml | 15 +++-- .../swap_or_not_shuffle/benches/benches.rs | 62 +++++++++++++++++++ 2 files changed, 72 insertions(+), 5 deletions(-) create mode 100644 eth2/utils/swap_or_not_shuffle/benches/benches.rs diff --git a/eth2/utils/swap_or_not_shuffle/Cargo.toml b/eth2/utils/swap_or_not_shuffle/Cargo.toml index 272abf608..3a866da92 100644 --- a/eth2/utils/swap_or_not_shuffle/Cargo.toml +++ b/eth2/utils/swap_or_not_shuffle/Cargo.toml @@ -4,12 +4,17 @@ version = "0.1.0" authors = ["Paul Hauner "] edition = "2018" +[[bench]] +name = "benches" +harness = false + +[dev-dependencies] +criterion = "0.2" +yaml-rust = "0.4.2" +hex = "0.3" +ethereum-types = "0.5" + [dependencies] bytes = "0.4" hashing = { path = "../hashing" } int_to_bytes = { path = "../int_to_bytes" } - -[dev-dependencies] -yaml-rust = "0.4.2" -hex = "0.3" -ethereum-types = "0.5" diff --git a/eth2/utils/swap_or_not_shuffle/benches/benches.rs b/eth2/utils/swap_or_not_shuffle/benches/benches.rs new file mode 100644 index 000000000..1d5b5476c --- /dev/null +++ b/eth2/utils/swap_or_not_shuffle/benches/benches.rs @@ -0,0 +1,62 @@ +use criterion::Criterion; +use criterion::{black_box, criterion_group, criterion_main, Benchmark}; +use swap_or_not_shuffle::get_permutated_index; + +const SHUFFLE_ROUND_COUNT: u8 = 90; + +fn shuffle_list(seed: &[u8], list_size: usize) -> Vec { + let mut output = Vec::with_capacity(list_size); + for i in 0..list_size { + output.push(get_permutated_index(i, list_size, seed, SHUFFLE_ROUND_COUNT).unwrap()); + } + output +} + +fn shuffles(c: &mut Criterion) { + c.bench_function("single swap", move |b| { + let seed = vec![42; 32]; + b.iter(|| black_box(get_permutated_index(0, 10, &seed, SHUFFLE_ROUND_COUNT))) + }); + + c.bench_function("whole list of size 8", move |b| { + let seed = vec![42; 32]; + b.iter(|| black_box(shuffle_list(&seed, 8))) + }); + + c.bench( + "whole list shuffle", + Benchmark::new("8 elements", move |b| { + let seed = vec![42; 32]; + b.iter(|| black_box(shuffle_list(&seed, 8))) + }), + ); + + c.bench( + "whole list shuffle", + Benchmark::new("16 elements", move |b| { + let seed = vec![42; 32]; + b.iter(|| black_box(shuffle_list(&seed, 16))) + }), + ); + + c.bench( + "whole list shuffle", + Benchmark::new("512 elements", move |b| { + let seed = vec![42; 32]; + b.iter(|| black_box(shuffle_list(&seed, 512))) + }) + .sample_size(10), + ); + + c.bench( + "whole list shuffle", + Benchmark::new("16384 elements", move |b| { + let seed = vec![42; 32]; + b.iter(|| black_box(shuffle_list(&seed, 16_384))) + }) + .sample_size(10), + ); +} + +criterion_group!(benches, shuffles,); +criterion_main!(benches); From a8ebc0e19c5c23d543df647a99a9d73197dbba28 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 19:31:10 +1300 Subject: [PATCH 029/132] Add benchmarks for `fisher_yates_shuffle` --- eth2/utils/fisher_yates_shuffle/Cargo.toml | 9 ++- .../fisher_yates_shuffle/benches/benches.rs | 55 +++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 eth2/utils/fisher_yates_shuffle/benches/benches.rs diff --git a/eth2/utils/fisher_yates_shuffle/Cargo.toml b/eth2/utils/fisher_yates_shuffle/Cargo.toml index 7d33c2e91..ff1f64608 100644 --- a/eth2/utils/fisher_yates_shuffle/Cargo.toml +++ b/eth2/utils/fisher_yates_shuffle/Cargo.toml @@ -4,8 +4,13 @@ version = "0.1.0" authors = ["Paul Hauner "] edition = "2018" -[dependencies] -hashing = { path = "../hashing" } +[[bench]] +name = "benches" +harness = false [dev-dependencies] +criterion = "0.2" yaml-rust = "0.4.2" + +[dependencies] +hashing = { path = "../hashing" } diff --git a/eth2/utils/fisher_yates_shuffle/benches/benches.rs b/eth2/utils/fisher_yates_shuffle/benches/benches.rs new file mode 100644 index 000000000..9aa1885ab --- /dev/null +++ b/eth2/utils/fisher_yates_shuffle/benches/benches.rs @@ -0,0 +1,55 @@ +use criterion::Criterion; +use criterion::{black_box, criterion_group, criterion_main, Benchmark}; +use fisher_yates_shuffle::shuffle; + +fn get_list(n: usize) -> Vec { + let mut list = Vec::with_capacity(n); + for i in 0..n { + list.push(i) + } + assert_eq!(list.len(), n); + list +} + +fn shuffles(c: &mut Criterion) { + c.bench( + "whole list shuffle", + Benchmark::new("8 elements", move |b| { + let seed = vec![42; 32]; + let list = get_list(8); + b.iter_with_setup(|| list.clone(), |list| black_box(shuffle(&seed, list))) + }), + ); + + c.bench( + "whole list shuffle", + Benchmark::new("16 elements", move |b| { + let seed = vec![42; 32]; + let list = get_list(16); + b.iter_with_setup(|| list.clone(), |list| black_box(shuffle(&seed, list))) + }), + ); + + c.bench( + "whole list shuffle", + Benchmark::new("512 elements", move |b| { + let seed = vec![42; 32]; + let list = get_list(512); + b.iter_with_setup(|| list.clone(), |list| black_box(shuffle(&seed, list))) + }) + .sample_size(10), + ); + + c.bench( + "whole list shuffle", + Benchmark::new("16384 elements", move |b| { + let seed = vec![42; 32]; + let list = get_list(16_384); + b.iter_with_setup(|| list.clone(), |list| black_box(shuffle(&seed, list))) + }) + .sample_size(10), + ); +} + +criterion_group!(benches, shuffles); +criterion_main!(benches); From f82c4268e29a8323f110c50ed0bfa0e7690f33f0 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 20:16:14 +1300 Subject: [PATCH 030/132] Split `BeaconState::genesis` into two functions This allows us to build a genesis state without inducting all the validators. This will be a signigicant speed up for tests as we can avoid generating keypairs and checking proof-of-possessions. --- eth2/types/src/beacon_state.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index f7df3b0ab..fad756708 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -117,19 +117,18 @@ pub struct BeaconState { impl BeaconState { /// Produce the first state of the Beacon Chain. - pub fn genesis( + pub fn genesis_without_validators( genesis_time: u64, - initial_validator_deposits: Vec, latest_eth1_data: Eth1Data, spec: &ChainSpec, ) -> Result { - debug!("Creating genesis state."); + debug!("Creating genesis state (without validator processing)."); let initial_crosslink = Crosslink { epoch: spec.genesis_epoch, shard_block_root: spec.zero_hash, }; - let mut genesis_state = BeaconState { + Ok(BeaconState { /* * Misc */ @@ -188,7 +187,19 @@ impl BeaconState { */ cache_index_offset: 0, caches: vec![EpochCache::empty(); CACHED_EPOCHS], - }; + }) + } + /// Produce the first state of the Beacon Chain. + pub fn genesis( + genesis_time: u64, + initial_validator_deposits: Vec, + latest_eth1_data: Eth1Data, + spec: &ChainSpec, + ) -> Result { + let mut genesis_state = + BeaconState::genesis_without_validators(genesis_time, latest_eth1_data, spec)?; + + trace!("Processing genesis deposits..."); let deposit_data = initial_validator_deposits .iter() From 931da13545d73ea02d4d43c687688f889d94e2ea Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 20:18:20 +1300 Subject: [PATCH 031/132] Add `drop_cache` fn to `BeaconState` Permits clearing an epoch cache. --- eth2/types/src/beacon_state.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index fad756708..3d94a8e3d 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -290,14 +290,18 @@ impl BeaconState { /// -- `Next` epoch is always _without_ a registry change. If you perform a registry update, /// you should rebuild the `Current` cache so it uses the new seed. pub fn advance_caches(&mut self) { - let previous_cache_index = self.cache_index(RelativeEpoch::Previous); - - self.caches[previous_cache_index] = EpochCache::empty(); + self.drop_cache(RelativeEpoch::Previous); self.cache_index_offset += 1; self.cache_index_offset %= CACHED_EPOCHS; } + /// Removes the specified cache and sets it to uninitialized. + pub fn drop_cache(&mut self, relative_epoch: RelativeEpoch) { + let previous_cache_index = self.cache_index(relative_epoch); + self.caches[previous_cache_index] = EpochCache::empty(); + } + /// Returns the index of `self.caches` for some `RelativeEpoch`. fn cache_index(&self, relative_epoch: RelativeEpoch) -> usize { let base_index = match relative_epoch { From 5cfc9cb21d166d4d145229240fd7488c3cea449b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 20:18:59 +1300 Subject: [PATCH 032/132] Test state processing with and w/o caches --- eth2/state_processing/benches/benches.rs | 36 ++++++++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/eth2/state_processing/benches/benches.rs b/eth2/state_processing/benches/benches.rs index 29f9a1e86..68a9735e1 100644 --- a/eth2/state_processing/benches/benches.rs +++ b/eth2/state_processing/benches/benches.rs @@ -2,7 +2,8 @@ use criterion::Criterion; use criterion::{black_box, criterion_group, criterion_main}; // use env_logger::{Builder, Env}; use state_processing::SlotProcessable; -use types::{beacon_state::BeaconStateBuilder, ChainSpec, Hash256}; +use types::*; +use types::beacon_state::BeaconStateBuilder; fn epoch_processing(c: &mut Criterion) { // Builder::from_env(Env::default().default_filter_or("debug")).init(); @@ -13,13 +14,36 @@ fn epoch_processing(c: &mut Criterion) { builder.genesis().unwrap(); builder.teleport_to_end_of_epoch(builder.spec.genesis_epoch + 4); - let state = builder.build().unwrap(); + let mut state = builder.build().unwrap(); - c.bench_function("epoch processing", move |b| { - let spec = &builder.spec; + // Build all the caches so the following state does _not_ include the cache-building time. + state.build_epoch_cache(RelativeEpoch::Previous, &builder.spec).unwrap(); + state.build_epoch_cache(RelativeEpoch::Current, &builder.spec).unwrap(); + state.build_epoch_cache(RelativeEpoch::Next, &builder.spec).unwrap(); + + let cached_state = state.clone(); + + // Drop all the caches so the following state includes the cache-building time. + state.drop_cache(RelativeEpoch::Previous); + state.drop_cache(RelativeEpoch::Current); + state.drop_cache(RelativeEpoch::Next); + + let cacheless_state = state; + + let spec_a = builder.spec.clone(); + let spec_b = builder.spec.clone(); + + c.bench_function("epoch processing with pre-built caches", move |b| { b.iter_with_setup( - || state.clone(), - |mut state| black_box(state.per_slot_processing(Hash256::zero(), spec).unwrap()), + || cached_state.clone(), + |mut state| black_box(state.per_slot_processing(Hash256::zero(), &spec_a).unwrap()), + ) + }); + + c.bench_function("epoch processing without pre-built caches", move |b| { + b.iter_with_setup( + || cacheless_state.clone(), + |mut state| black_box(state.per_slot_processing(Hash256::zero(), &spec_b).unwrap()), ) }); } From af17fb1d0301b96438c9028494bacaf4d7962965 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 22:00:07 +1300 Subject: [PATCH 033/132] Update `BeaconStateBuilder` API --- .../beacon_chain/test_harness/Cargo.toml | 1 + .../test_harness/benches/state_transition.rs | 1 + eth2/state_processing/benches/benches.rs | 20 ++-- .../src/epoch_processable/tests.rs | 6 +- eth2/types/src/beacon_state/builder.rs | 101 ++++++++++++------ eth2/types/src/beacon_state/tests.rs | 10 +- 6 files changed, 89 insertions(+), 50 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/Cargo.toml b/beacon_node/beacon_chain/test_harness/Cargo.toml index 77b52ccf6..657cc7955 100644 --- a/beacon_node/beacon_chain/test_harness/Cargo.toml +++ b/beacon_node/beacon_chain/test_harness/Cargo.toml @@ -10,6 +10,7 @@ harness = false [dev-dependencies] criterion = "0.2" +state_processing = { path = "../../../eth2/state_processing" } [dependencies] attester = { path = "../../../eth2/attester" } diff --git a/beacon_node/beacon_chain/test_harness/benches/state_transition.rs b/beacon_node/beacon_chain/test_harness/benches/state_transition.rs index 013ecfd1e..aa2a858fa 100644 --- a/beacon_node/beacon_chain/test_harness/benches/state_transition.rs +++ b/beacon_node/beacon_chain/test_harness/benches/state_transition.rs @@ -1,6 +1,7 @@ use criterion::Criterion; use criterion::{black_box, criterion_group, criterion_main, Benchmark}; // use env_logger::{Builder, Env}; +use state_processing::SlotProcessable; use test_harness::BeaconChainHarness; use types::{ChainSpec, Hash256}; diff --git a/eth2/state_processing/benches/benches.rs b/eth2/state_processing/benches/benches.rs index 68a9735e1..244e3c8a5 100644 --- a/eth2/state_processing/benches/benches.rs +++ b/eth2/state_processing/benches/benches.rs @@ -2,24 +2,30 @@ use criterion::Criterion; use criterion::{black_box, criterion_group, criterion_main}; // use env_logger::{Builder, Env}; use state_processing::SlotProcessable; -use types::*; use types::beacon_state::BeaconStateBuilder; +use types::*; fn epoch_processing(c: &mut Criterion) { // Builder::from_env(Env::default().default_filter_or("debug")).init(); - let mut builder = BeaconStateBuilder::with_random_validators(8); + let mut builder = BeaconStateBuilder::new(8); builder.spec = ChainSpec::few_validators(); - builder.genesis().unwrap(); + builder.build().unwrap(); builder.teleport_to_end_of_epoch(builder.spec.genesis_epoch + 4); - let mut state = builder.build().unwrap(); + let mut state = builder.cloned_state(); // Build all the caches so the following state does _not_ include the cache-building time. - state.build_epoch_cache(RelativeEpoch::Previous, &builder.spec).unwrap(); - state.build_epoch_cache(RelativeEpoch::Current, &builder.spec).unwrap(); - state.build_epoch_cache(RelativeEpoch::Next, &builder.spec).unwrap(); + state + .build_epoch_cache(RelativeEpoch::Previous, &builder.spec) + .unwrap(); + state + .build_epoch_cache(RelativeEpoch::Current, &builder.spec) + .unwrap(); + state + .build_epoch_cache(RelativeEpoch::Next, &builder.spec) + .unwrap(); let cached_state = state.clone(); diff --git a/eth2/state_processing/src/epoch_processable/tests.rs b/eth2/state_processing/src/epoch_processable/tests.rs index 353672fd1..d683d3971 100644 --- a/eth2/state_processing/src/epoch_processable/tests.rs +++ b/eth2/state_processing/src/epoch_processable/tests.rs @@ -8,13 +8,13 @@ use types::*; fn runs_without_error() { Builder::from_env(Env::default().default_filter_or("error")).init(); - let mut builder = BeaconStateBuilder::with_random_validators(8); + let mut builder = BeaconStateBuilder::new(8); builder.spec = ChainSpec::few_validators(); - builder.genesis().unwrap(); + builder.build().unwrap(); builder.teleport_to_end_of_epoch(builder.spec.genesis_epoch + 4); - let mut state = builder.build().unwrap(); + let mut state = builder.cloned_state(); let spec = &builder.spec; state.per_epoch_processing(spec).unwrap(); diff --git a/eth2/types/src/beacon_state/builder.rs b/eth2/types/src/beacon_state/builder.rs index 7273f3658..80e4d501f 100644 --- a/eth2/types/src/beacon_state/builder.rs +++ b/eth2/types/src/beacon_state/builder.rs @@ -6,20 +6,19 @@ use bls::create_proof_of_possession; /// Building the `BeaconState` is a three step processes: /// /// 1. Create a new `BeaconStateBuilder`. -/// 2. Run the `genesis` function to generate a new BeaconState. +/// 2. Call `Self::build()` or `Self::build_fast()` generate a `BeaconState`. /// 3. (Optional) Use builder functions to modify the `BeaconState`. -/// 4. Call `build()` to obtain a cloned `BeaconState`. +/// 4. Call `Self::cloned_state()` to obtain a `BeaconState` cloned from this struct. /// -/// Step (2) is necessary because step (3) requires an existing `BeaconState` object. (2) is not -/// included in (1) to allow for modifying params before generating the `BeaconState` (e.g., the -/// spec). +/// Step (2) happens prior to step (3) because some functionality requires an existing +/// `BeaconState`. /// /// Step (4) produces a clone of the BeaconState and doesn't consume the `BeaconStateBuilder` to -/// allow access to the `keypairs` and `spec`. +/// allow access to `self.keypairs` and `self.spec`. pub struct BeaconStateBuilder { + pub validator_count: usize, pub state: Option, pub genesis_time: u64, - pub initial_validator_deposits: Vec, pub latest_eth1_data: Eth1Data, pub spec: ChainSpec, pub keypairs: Vec, @@ -27,21 +26,41 @@ pub struct BeaconStateBuilder { impl BeaconStateBuilder { /// Create a new builder with the given number of validators. - pub fn with_random_validators(validator_count: usize) -> Self { + pub fn new(validator_count: usize) -> Self { let genesis_time = 10_000_000; - let keypairs: Vec = (0..validator_count) + + let latest_eth1_data = Eth1Data { + deposit_root: Hash256::zero(), + block_hash: Hash256::zero(), + }; + + let spec = ChainSpec::foundation(); + + Self { + validator_count, + state: None, + genesis_time, + latest_eth1_data, + spec, + keypairs: vec![], + } + } + + pub fn build(&mut self) -> Result<(), BeaconStateError> { + self.keypairs = (0..self.validator_count) .collect::>() .iter() .map(|_| Keypair::random()) .collect(); - let initial_validator_deposits = keypairs + + let initial_validator_deposits = self.keypairs .iter() .map(|keypair| Deposit { branch: vec![], // branch verification is not specified. index: 0, // index verification is not specified. deposit_data: DepositData { amount: 32_000_000_000, // 32 ETH (in Gwei) - timestamp: genesis_time - 1, + timestamp: self.genesis_time - 1, deposit_input: DepositInput { pubkey: keypair.pk.clone(), withdrawal_credentials: Hash256::zero(), // Withdrawal not possible. @@ -50,27 +69,10 @@ impl BeaconStateBuilder { }, }) .collect(); - let latest_eth1_data = Eth1Data { - deposit_root: Hash256::zero(), - block_hash: Hash256::zero(), - }; - let spec = ChainSpec::foundation(); - Self { - state: None, - genesis_time, - initial_validator_deposits, - latest_eth1_data, - spec, - keypairs, - } - } - - /// Runs the `BeaconState::genesis` function and produces a `BeaconState`. - pub fn genesis(&mut self) -> Result<(), BeaconStateError> { let state = BeaconState::genesis( self.genesis_time, - self.initial_validator_deposits.clone(), + initial_validator_deposits, self.latest_eth1_data.clone(), &self.spec, )?; @@ -80,6 +82,40 @@ impl BeaconStateBuilder { Ok(()) } + pub fn build_fast(&mut self) -> Result<(), BeaconStateError> { + let common_keypair = Keypair::random(); + + let mut validator_registry = Vec::with_capacity(self.validator_count); + let mut validator_balances = Vec::with_capacity(self.validator_count); + self.keypairs = Vec::with_capacity(self.validator_count); + + for _ in 0..self.validator_count { + self.keypairs.push(common_keypair.clone()); + validator_balances.push(32_000_000_000); + validator_registry.push(Validator { + pubkey: common_keypair.pk.clone(), + withdrawal_credentials: Hash256::zero(), + activation_epoch: self.spec.genesis_epoch, + ..Validator::default() + }) + } + + let state = BeaconState { + validator_registry, + validator_balances, + ..BeaconState::genesis( + self.genesis_time, + vec![], + self.latest_eth1_data.clone(), + &self.spec, + )? + }; + + self.state = Some(state); + + Ok(()) + } + /// Sets the `BeaconState` to be in the last slot of the given epoch. /// /// Sets all justification/finalization parameters to be be as "perfect" as possible (i.e., @@ -145,11 +181,8 @@ impl BeaconStateBuilder { } /// Returns a cloned `BeaconState`. - pub fn build(&self) -> Result { - match &self.state { - Some(state) => Ok(state.clone()), - None => panic!("Genesis required"), - } + pub fn cloned_state(&self) -> BeaconState { + self.state.as_ref().expect("Genesis required").clone() } } diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs index 01b4be803..bb8561511 100644 --- a/eth2/types/src/beacon_state/tests.rs +++ b/eth2/types/src/beacon_state/tests.rs @@ -7,9 +7,7 @@ use ssz::{ssz_encode, Decodable}; #[test] pub fn can_produce_genesis_block() { - let mut builder = BeaconStateBuilder::with_random_validators(2); - builder.genesis().unwrap(); - + let mut builder = BeaconStateBuilder::new(2); builder.build().unwrap(); } @@ -19,12 +17,12 @@ pub fn can_produce_genesis_block() { pub fn get_attestation_participants_consistency() { let mut rng = XorShiftRng::from_seed([42; 16]); - let mut builder = BeaconStateBuilder::with_random_validators(8); + let mut builder = BeaconStateBuilder::new(8); builder.spec = ChainSpec::few_validators(); - builder.genesis().unwrap(); + builder.build().unwrap(); - let mut state = builder.build().unwrap(); + let mut state = builder.cloned_state(); let spec = builder.spec.clone(); state From 3ff8f6ebb326cf429fe9a7a8038683c4b3b7cfad Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 22:25:29 +1300 Subject: [PATCH 034/132] Update epoch trans. tests to us 16,384 validators --- eth2/state_processing/benches/benches.rs | 39 ++++++++++++++---------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/eth2/state_processing/benches/benches.rs b/eth2/state_processing/benches/benches.rs index 244e3c8a5..682259eef 100644 --- a/eth2/state_processing/benches/benches.rs +++ b/eth2/state_processing/benches/benches.rs @@ -1,5 +1,5 @@ use criterion::Criterion; -use criterion::{black_box, criterion_group, criterion_main}; +use criterion::{black_box, criterion_group, criterion_main, Benchmark}; // use env_logger::{Builder, Env}; use state_processing::SlotProcessable; use types::beacon_state::BeaconStateBuilder; @@ -8,10 +8,9 @@ use types::*; fn epoch_processing(c: &mut Criterion) { // Builder::from_env(Env::default().default_filter_or("debug")).init(); - let mut builder = BeaconStateBuilder::new(8); - builder.spec = ChainSpec::few_validators(); + let mut builder = BeaconStateBuilder::new(16_384); - builder.build().unwrap(); + builder.build_fast().unwrap(); builder.teleport_to_end_of_epoch(builder.spec.genesis_epoch + 4); let mut state = builder.cloned_state(); @@ -39,19 +38,27 @@ fn epoch_processing(c: &mut Criterion) { let spec_a = builder.spec.clone(); let spec_b = builder.spec.clone(); - c.bench_function("epoch processing with pre-built caches", move |b| { - b.iter_with_setup( - || cached_state.clone(), - |mut state| black_box(state.per_slot_processing(Hash256::zero(), &spec_a).unwrap()), - ) - }); + c.bench( + "epoch processing", + Benchmark::new("with pre-built caches", move |b| { + b.iter_with_setup( + || cached_state.clone(), + |mut state| black_box(state.per_slot_processing(Hash256::zero(), &spec_a).unwrap()), + ) + }) + .sample_size(10), + ); - c.bench_function("epoch processing without pre-built caches", move |b| { - b.iter_with_setup( - || cacheless_state.clone(), - |mut state| black_box(state.per_slot_processing(Hash256::zero(), &spec_b).unwrap()), - ) - }); + c.bench( + "epoch processing", + Benchmark::new("without pre-built caches", move |b| { + b.iter_with_setup( + || cacheless_state.clone(), + |mut state| black_box(state.per_slot_processing(Hash256::zero(), &spec_b).unwrap()), + ) + }) + .sample_size(10), + ); } criterion_group!(benches, epoch_processing,); From c2fb09575947fab58289a2d0e6334da56f1b0458 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 22:34:41 +1300 Subject: [PATCH 035/132] Add docstrings to `BeaconStateBuilder` --- eth2/types/src/beacon_state/builder.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/eth2/types/src/beacon_state/builder.rs b/eth2/types/src/beacon_state/builder.rs index 80e4d501f..40bfe0d2c 100644 --- a/eth2/types/src/beacon_state/builder.rs +++ b/eth2/types/src/beacon_state/builder.rs @@ -46,6 +46,10 @@ impl BeaconStateBuilder { } } + /// Builds a `BeaconState` using the `BeaconState::genesis(..)` function. + /// + /// Each validator is assigned a unique, randomly-generated keypair and all + /// proof-of-possessions are verified during genesis. pub fn build(&mut self) -> Result<(), BeaconStateError> { self.keypairs = (0..self.validator_count) .collect::>() @@ -53,7 +57,8 @@ impl BeaconStateBuilder { .map(|_| Keypair::random()) .collect(); - let initial_validator_deposits = self.keypairs + let initial_validator_deposits = self + .keypairs .iter() .map(|keypair| Deposit { branch: vec![], // branch verification is not specified. @@ -82,6 +87,16 @@ impl BeaconStateBuilder { Ok(()) } + /// Builds a `BeaconState` using the `BeaconState::genesis(..)` function, without supplying any + /// validators. Instead validators are added to the state post-genesis. + /// + /// One keypair is randomly generated and all validators are assigned this same keypair. + /// Proof-of-possessions are not created (or validated). + /// + /// This function runs orders of magnitude faster than `Self::build()`, however it will be + /// erroneous for functions which use a validators public key as an identifier (e.g., + /// deposits). + /// proof-of-possessions are verified during genesis. pub fn build_fast(&mut self) -> Result<(), BeaconStateError> { let common_keypair = Keypair::random(); From 04f179243e4cd4bf0f6523efcf650b4f5d1125d5 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 26 Feb 2019 22:35:51 +1300 Subject: [PATCH 036/132] Fix type in `BeaconStateBuilder` comments --- eth2/types/src/beacon_state/builder.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/eth2/types/src/beacon_state/builder.rs b/eth2/types/src/beacon_state/builder.rs index 40bfe0d2c..02886a86e 100644 --- a/eth2/types/src/beacon_state/builder.rs +++ b/eth2/types/src/beacon_state/builder.rs @@ -96,7 +96,6 @@ impl BeaconStateBuilder { /// This function runs orders of magnitude faster than `Self::build()`, however it will be /// erroneous for functions which use a validators public key as an identifier (e.g., /// deposits). - /// proof-of-possessions are verified during genesis. pub fn build_fast(&mut self) -> Result<(), BeaconStateError> { let common_keypair = Keypair::random(); From 073be906da498171acd3a7b94d773cdded2d3b52 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 28 Feb 2019 16:37:12 +1100 Subject: [PATCH 037/132] Remove cache operations from epoch processing. - Don't build the next cache at all. - Call `advance_caches()` in per-slot processing. --- eth2/state_processing/src/epoch_processable.rs | 6 ------ eth2/state_processing/src/slot_processable.rs | 1 + 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/eth2/state_processing/src/epoch_processable.rs b/eth2/state_processing/src/epoch_processable.rs index 2cee455a3..0ecd1bbd1 100644 --- a/eth2/state_processing/src/epoch_processable.rs +++ b/eth2/state_processing/src/epoch_processable.rs @@ -619,12 +619,6 @@ impl EpochProcessable for BeaconState { .cloned() .collect(); - /* - * Manage the beacon state caches - */ - self.advance_caches(); - self.build_epoch_cache(RelativeEpoch::Next, spec)?; - debug!("Epoch transition complete."); Ok(()) diff --git a/eth2/state_processing/src/slot_processable.rs b/eth2/state_processing/src/slot_processable.rs index 0bbc79ab0..6017f4c0a 100644 --- a/eth2/state_processing/src/slot_processable.rs +++ b/eth2/state_processing/src/slot_processable.rs @@ -26,6 +26,7 @@ where ) -> Result<(), Error> { if (self.slot + 1) % spec.epoch_length == 0 { self.per_epoch_processing(spec)?; + self.advance_caches(); } self.slot += 1; From 7108d057fb8bf0d84326619098755305c41e1572 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 28 Feb 2019 23:09:21 +1100 Subject: [PATCH 038/132] Set `BeaconState` to process deposits in parallel Provides a large speed improvement. --- eth2/types/src/beacon_state.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 3d94a8e3d..61f270e15 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -9,6 +9,7 @@ use bls::verify_proof_of_possession; use honey_badger_split::SplitExt; use log::{debug, error, trace}; use rand::RngCore; +use rayon::prelude::*; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; use std::collections::HashMap; @@ -202,7 +203,7 @@ impl BeaconState { trace!("Processing genesis deposits..."); let deposit_data = initial_validator_deposits - .iter() + .par_iter() .map(|deposit| &deposit.deposit_data) .collect(); From e0926dcd8da65e8da0f83d06b4572aabe9ccf2de Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 28 Feb 2019 23:10:40 +1100 Subject: [PATCH 039/132] Change log msgs in `BeaconState` Make it louder when shuffling happens, also when deposits are being processed. --- eth2/types/src/beacon_state.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 61f270e15..5591fa301 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -200,7 +200,7 @@ impl BeaconState { let mut genesis_state = BeaconState::genesis_without_validators(genesis_time, latest_eth1_data, spec)?; - trace!("Processing genesis deposits..."); + debug!("Processing genesis deposits..."); let deposit_data = initial_validator_deposits .par_iter() @@ -412,6 +412,8 @@ impl BeaconState { return Err(Error::InsufficientValidators); } + debug!("Shuffling {} validators...", active_validator_indices.len()); + let committees_per_epoch = self.get_epoch_committee_count(active_validator_indices.len(), spec); From 8b06fa31da5e44c4a438d297d842695ab4dd315b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 28 Feb 2019 23:13:00 +1100 Subject: [PATCH 040/132] Add basic YAML test_harness tests Works, however ignores a lot of fields in the YAML. --- .../beacon_chain/test_harness/Cargo.toml | 10 ++ .../test_harness/examples/chain.yaml | 77 +++++++++ .../beacon_chain/test_harness/src/bin.rs | 148 ++++++++++++++++++ 3 files changed, 235 insertions(+) create mode 100644 beacon_node/beacon_chain/test_harness/examples/chain.yaml create mode 100644 beacon_node/beacon_chain/test_harness/src/bin.rs diff --git a/beacon_node/beacon_chain/test_harness/Cargo.toml b/beacon_node/beacon_chain/test_harness/Cargo.toml index 657cc7955..bd7a58270 100644 --- a/beacon_node/beacon_chain/test_harness/Cargo.toml +++ b/beacon_node/beacon_chain/test_harness/Cargo.toml @@ -4,6 +4,14 @@ version = "0.1.0" authors = ["Paul Hauner "] edition = "2018" +[[bin]] +name = "test_harness" +path = "src/bin.rs" + +[lib] +name = "test_harness" +path = "src/lib.rs" + [[bench]] name = "state_transition" harness = false @@ -18,6 +26,7 @@ beacon_chain = { path = "../../beacon_chain" } block_proposer = { path = "../../../eth2/block_proposer" } bls = { path = "../../../eth2/utils/bls" } boolean-bitfield = { path = "../../../eth2/utils/boolean-bitfield" } +clap = "2.32.0" db = { path = "../../db" } parking_lot = "0.7" failure = "0.1" @@ -33,3 +42,4 @@ serde_json = "1.0" slot_clock = { path = "../../../eth2/utils/slot_clock" } ssz = { path = "../../../eth2/utils/ssz" } types = { path = "../../../eth2/types" } +yaml-rust = "0.4.2" diff --git a/beacon_node/beacon_chain/test_harness/examples/chain.yaml b/beacon_node/beacon_chain/test_harness/examples/chain.yaml new file mode 100644 index 000000000..5d8e34795 --- /dev/null +++ b/beacon_node/beacon_chain/test_harness/examples/chain.yaml @@ -0,0 +1,77 @@ +title: Sample Ethereum Serenity State Transition Tests +summary: Testing full state transition block processing +test_suite: prysm +fork: sapphire +version: 1.0 +test_cases: + - config: + epoch_length: 64 + deposits_for_chain_start: 1000 + num_slots: 32 # Testing advancing state to slot < SlotsPerEpoch + results: + slot: 32 + num_validators: 1000 + - config: + epoch_length: 64 + deposits_for_chain_start: 16384 + num_slots: 64 + deposits: + - slot: 1 + amount: 32 + merkle_index: 0 + pubkey: !!binary | + SlAAbShSkUg7PLiPHZI/rTS1uAvKiieOrifPN6Moso0= + - slot: 15 + amount: 32 + merkle_index: 1 + pubkey: !!binary | + Oklajsjdkaklsdlkajsdjlajslkdjlkasjlkdjlajdsd + - slot: 55 + amount: 32 + merkle_index: 2 + pubkey: !!binary | + LkmqmqoodLKAslkjdkajsdljasdkajlksjdasldjasdd + proposer_slashings: + - slot: 16 # At slot 16, we trigger a proposal slashing occurring + proposer_index: 16385 # We penalize the proposer that was just added from slot 15 + proposal_1_shard: 0 + proposal_1_slot: 15 + proposal_1_root: !!binary | + LkmqmqoodLKAslkjdkajsdljasdkajlksjdasldjasdd + proposal_2_shard: 0 + proposal_2_slot: 15 + proposal_2_root: !!binary | + LkmqmqoodLKAslkjdkajsdljasdkajlksjdasldjasdd + attester_slashings: + - slot: 59 # At slot 59, we trigger a attester slashing + slashable_vote_data_1_slot: 55 + slashable_vote_data_2_slot: 55 + slashable_vote_data_1_justified_slot: 0 + slashable_vote_data_2_justified_slot: 1 + slashable_vote_data_1_custody_0_indices: [16386] + slashable_vote_data_1_custody_1_indices: [] + slashable_vote_data_2_custody_0_indices: [] + slashable_vote_data_2_custody_1_indices: [16386] + results: + slot: 64 + num_validators: 16387 + penalized_validators: [16385, 16386] # We test that the validators at indices 16385, 16386 were indeed penalized + - config: + skip_slots: [10, 20] + epoch_length: 64 + deposits_for_chain_start: 1000 + num_slots: 128 # Testing advancing state's slot == 2*SlotsPerEpoch + deposits: + - slot: 10 + amount: 32 + merkle_index: 0 + pubkey: !!binary | + SlAAbShSkUg7PLiPHZI/rTS1uAvKiieOrifPN6Moso0= + - slot: 20 + amount: 32 + merkle_index: 1 + pubkey: !!binary | + Oklajsjdkaklsdlkajsdjlajslkdjlkasjlkdjlajdsd + results: + slot: 128 + num_validators: 1000 # Validator registry should not have grown if slots 10 and 20 were skipped diff --git a/beacon_node/beacon_chain/test_harness/src/bin.rs b/beacon_node/beacon_chain/test_harness/src/bin.rs new file mode 100644 index 000000000..007ec3f60 --- /dev/null +++ b/beacon_node/beacon_chain/test_harness/src/bin.rs @@ -0,0 +1,148 @@ +use self::beacon_chain_harness::BeaconChainHarness; +use self::validator_harness::ValidatorHarness; +use clap::{App, Arg}; +use env_logger::{Builder, Env}; +use log::info; +use std::{fs::File, io::prelude::*}; +use types::*; +use yaml_rust::{Yaml, YamlLoader}; + +mod beacon_chain_harness; +mod validator_harness; + +fn main() { + let matches = App::new("Lighthouse Test Harness Runner") + .version("0.0.1") + .author("Sigma Prime ") + .about("Runs `test_harness` using a YAML manifest.") + .arg( + Arg::with_name("yaml") + .long("yaml") + .value_name("FILE") + .help("YAML file manifest.") + .required(true), + ) + .get_matches(); + + Builder::from_env(Env::default().default_filter_or("debug")).init(); + + if let Some(yaml_file) = matches.value_of("yaml") { + let docs = { + let mut file = File::open(yaml_file).unwrap(); + + let mut yaml_str = String::new(); + file.read_to_string(&mut yaml_str).unwrap(); + + YamlLoader::load_from_str(&yaml_str).unwrap() + }; + + for doc in &docs { + for test_case in doc["test_cases"].as_vec().unwrap() { + let manifest = Manifest::from_yaml(test_case); + manifest.execute(); + } + } + } +} + +struct Manifest { + pub results: Results, + pub config: Config, +} + +impl Manifest { + pub fn from_yaml(test_case: &Yaml) -> Self { + Self { + results: Results::from_yaml(&test_case["results"]), + config: Config::from_yaml(&test_case["config"]), + } + } + + fn spec(&self) -> ChainSpec { + let mut spec = ChainSpec::foundation(); + + if let Some(n) = self.config.epoch_length { + spec.epoch_length = n; + } + + spec + } + + pub fn execute(&self) { + let spec = self.spec(); + let validator_count = self.config.deposits_for_chain_start; + let slots = self.results.slot; + + info!( + "Building BeaconChainHarness with {} validators...", + validator_count + ); + + let mut harness = BeaconChainHarness::new(spec, validator_count); + + info!("Starting simulation across {} slots...", slots); + + for _ in 0..self.results.slot { + harness.advance_chain_with_block(); + } + + harness.run_fork_choice(); + + let dump = harness.chain_dump().expect("Chain dump failed."); + + assert_eq!(dump.len() as u64, slots + 1); // + 1 for genesis block. + + // harness.dump_to_file("/tmp/chaindump.json".to_string(), &dump); + } +} + +struct Results { + pub slot: u64, + pub num_validators: Option, + pub slashed_validators: Option>, + pub exited_validators: Option>, +} + +impl Results { + pub fn from_yaml(yaml: &Yaml) -> Self { + Self { + slot: as_u64(&yaml, "slot").expect("Must have end slot"), + num_validators: as_usize(&yaml, "num_validators"), + slashed_validators: as_vec_u64(&yaml, "slashed_validators"), + exited_validators: as_vec_u64(&yaml, "exited_validators"), + } + } +} + +struct Config { + pub deposits_for_chain_start: usize, + pub epoch_length: Option, +} + +impl Config { + pub fn from_yaml(yaml: &Yaml) -> Self { + Self { + deposits_for_chain_start: as_usize(&yaml, "deposits_for_chain_start") + .expect("Must specify validator count"), + epoch_length: as_u64(&yaml, "epoch_length"), + } + } +} + +fn as_usize(yaml: &Yaml, key: &str) -> Option { + yaml[key].as_i64().and_then(|n| Some(n as usize)) +} + +fn as_u64(yaml: &Yaml, key: &str) -> Option { + yaml[key].as_i64().and_then(|n| Some(n as u64)) +} + +fn as_vec_u64(yaml: &Yaml, key: &str) -> Option> { + yaml[key].clone().into_vec().and_then(|vec| { + Some( + vec.iter() + .map(|item| item.as_i64().unwrap() as u64) + .collect(), + ) + }) +} From 6fac35a7c51f3b16c4b9ac240b5ada19ecbb1298 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Mar 2019 00:18:53 +1100 Subject: [PATCH 041/132] Update main readme, add eth2/ readme --- README.md | 13 +++++++++++++ eth2/README.md | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 eth2/README.md diff --git a/README.md b/README.md index 7759c1166..e3df9c19c 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,19 @@ If you'd like some background on Sigma Prime, please see the [Lighthouse Update \#00](https://lighthouse.sigmaprime.io/update-00.html) blog post or the [company website](https://sigmaprime.io). +### Directory Structure + +- [`/beacon_node`](beacon_node/): the "Beacon Node" binary and crates exclusively + associated with it. +- [`/docs`](docs/): documentation related to the repository. This include contributor + guides, etc. Code documentation is produced with `cargo doc`. +- [`/eth2`](eth2/): Crates containing common logic across the Lighthouse project. For + example; Ethereum 2.0 types (`BeaconBlock`, `BeaconState`, etc) and + SimpleSerialize (SSZ). +- [`/protos`](protos/): protobuf/gRPC definitions common across the Lighthouse project. +- [`/validator_client`](validator_client/): the "Validator Client" binary and crates exclusively + associated with it. + ### Components The following list describes some of the components actively under development diff --git a/eth2/README.md b/eth2/README.md new file mode 100644 index 000000000..a96c1b9df --- /dev/null +++ b/eth2/README.md @@ -0,0 +1,40 @@ +# eth2 + +Rust crates containing logic common across the Lighthouse project. + +## Per-Crate Summary + +- [`attester/`](attester/): Core logic for attesting to beacon and shard blocks. +- [`block_proposer/`](block_proposer/): Core logic for proposing beacon blocks. +- [`fork_choice/`](fork_choice/): A collection of fork-choice algorithms for + the Beacon Chain. +- [`state_processing/`](state_processing/): Provides per-slot, per-block and + per-epoch state processing. +- [`types/`](types/): Defines base Ethereum 2.0 types (e.g., `BeaconBlock`, + `BeaconState`, etc). +- [`utils/`](utils/): + - [`bls`](utils/bls/): A wrapper around some external BLS encryption library. + - [`boolean-bitfield`](utils/boolean-bitfield/): Provides an expandable Vec + of bools, specifically for use in Eth2. + - [`fisher-yates-shuffle`](utils/fisher-yates-shuffle/): shuffles a list + pseudo-randomly. + - [`hashing`](utils/hashing/): Provides unified hashing methods, provided + be some external library. + - [`honey-badger-split`](utils/honey-badger-split/): Splits a list in `n` + parts without giving AF about the length of the list, `n` or anything + else. + - [`int-to-bytes`](utils/int-to-bytes/): Simple library which converts ints + into byte-strings of various lengths. + - [`slot_clock`](utils/slot_clock/): translates the system time into Beacon + - Chain "slots". Also + provides another slot clock that's useful during testing. + - [`ssz`](utils/ssz/): an implementation of the SimpleSerialize + - serialization/deserialization + protocol used by + Eth 2.0. + - [`ssz_derive`](utils/ssz_derive/): provides procedural macros for + deriving SSZ `Encodable`, `Decodable` and `TreeHash` methods. + - [`swap_or_not_shuffle`](utils/swap_or_not_shuffle/): a list-shuffling + method which is slow, but allows for shuffling a subset of indices. + - [`test_random_derive`](utils/test_random_derive/): provides procedural + macros for deriving the `TestRandom` trait defined in `types`. From 8842adb53b97c859eac1ea3bf80ee98ade976d3a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Mar 2019 00:25:11 +1100 Subject: [PATCH 042/132] Fix typos in READMEs --- README.md | 24 +++++++----------------- eth2/README.md | 17 +++++++---------- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index e3df9c19c..c26dae219 100644 --- a/README.md +++ b/README.md @@ -31,15 +31,15 @@ If you'd like some background on Sigma Prime, please see the [Lighthouse Update ### Directory Structure -- [`/beacon_node`](beacon_node/): the "Beacon Node" binary and crates exclusively +- [`beacon_node/`](beacon_node/): the "Beacon Node" binary and crates exclusively associated with it. -- [`/docs`](docs/): documentation related to the repository. This include contributor - guides, etc. Code documentation is produced with `cargo doc`. -- [`/eth2`](eth2/): Crates containing common logic across the Lighthouse project. For - example; Ethereum 2.0 types (`BeaconBlock`, `BeaconState`, etc) and +- [`docs/`](docs/): documentation related to the repository. This include contributor + guides, etc. (Code documentation is produced with `cargo doc`). +- [`eth2/`](eth2/): Crates containing common logic across the Lighthouse project. For + example; Ethereum 2.0 types ([`BeaconBlock`](eth2/types/src/beacon_block.rs), [`BeaconState`](eth2/types/src/beacon_state.rs), etc) and SimpleSerialize (SSZ). -- [`/protos`](protos/): protobuf/gRPC definitions common across the Lighthouse project. -- [`/validator_client`](validator_client/): the "Validator Client" binary and crates exclusively +- [`protos/`](protos/): protobuf/gRPC definitions common across the Lighthouse project. +- [`validator_client/`](validator_client/): the "Validator Client" binary and crates exclusively associated with it. ### Components @@ -92,16 +92,6 @@ In addition to these components we are also working on database schemas, RPC frameworks, specification development, database optimizations (e.g., bloom-filters), and tons of other interesting stuff (at least we think so). -### Directory Structure - -Here we provide an overview of the directory structure: - -- `beacon_chain/`: contains logic derived directly from the specification. - E.g., shuffling algorithms, state transition logic and structs, block -validation, BLS crypto, etc. -- `lighthouse/`: contains logic specific to this client implementation. E.g., - CLI parsing, RPC end-points, databases, etc. - ### Running **NOTE: The cryptography libraries used in this implementation are diff --git a/eth2/README.md b/eth2/README.md index a96c1b9df..e7b69635a 100644 --- a/eth2/README.md +++ b/eth2/README.md @@ -1,4 +1,4 @@ -# eth2 +# Ethereum 2.0 Common Crates Rust crates containing logic common across the Lighthouse project. @@ -13,25 +13,22 @@ Rust crates containing logic common across the Lighthouse project. - [`types/`](types/): Defines base Ethereum 2.0 types (e.g., `BeaconBlock`, `BeaconState`, etc). - [`utils/`](utils/): - - [`bls`](utils/bls/): A wrapper around some external BLS encryption library. + - [`bls`](utils/bls/): A wrapper around an external BLS encryption library. - [`boolean-bitfield`](utils/boolean-bitfield/): Provides an expandable Vec of bools, specifically for use in Eth2. - [`fisher-yates-shuffle`](utils/fisher-yates-shuffle/): shuffles a list pseudo-randomly. - - [`hashing`](utils/hashing/): Provides unified hashing methods, provided - be some external library. + - [`hashing`](utils/hashing/): A wrapper around external hashing libraries. - [`honey-badger-split`](utils/honey-badger-split/): Splits a list in `n` parts without giving AF about the length of the list, `n` or anything else. - [`int-to-bytes`](utils/int-to-bytes/): Simple library which converts ints into byte-strings of various lengths. - [`slot_clock`](utils/slot_clock/): translates the system time into Beacon - - Chain "slots". Also - provides another slot clock that's useful during testing. - - [`ssz`](utils/ssz/): an implementation of the SimpleSerialize - - serialization/deserialization - protocol used by - Eth 2.0. + Chain "slots". (Also provides another slot clock that's useful during + testing.) + - [`ssz`](utils/ssz/): an implementation of the SimpleSerialize + serialization/deserialization protocol used by Eth 2.0. - [`ssz_derive`](utils/ssz_derive/): provides procedural macros for deriving SSZ `Encodable`, `Decodable` and `TreeHash` methods. - [`swap_or_not_shuffle`](utils/swap_or_not_shuffle/): a list-shuffling From 8aa7f25bbcb3c2563f2d1596a5814c147135f199 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Mar 2019 01:47:40 +1100 Subject: [PATCH 043/132] Introduce faster swap-or-not whole-list shuffle --- eth2/types/src/beacon_state.rs | 22 ++++----- .../swap_or_not_shuffle/benches/benches.rs | 22 ++++++++- eth2/utils/swap_or_not_shuffle/src/lib.rs | 46 ++++++++++++++++--- 3 files changed, 71 insertions(+), 19 deletions(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 3d94a8e3d..2d4f463b7 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -12,7 +12,7 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; use std::collections::HashMap; -use swap_or_not_shuffle::get_permutated_index; +use swap_or_not_shuffle::get_permutated_list; pub use builder::BeaconStateBuilder; @@ -420,17 +420,15 @@ impl BeaconState { committees_per_epoch ); - let mut shuffled_active_validator_indices = vec![0; active_validator_indices.len()]; - for (i, _) in active_validator_indices.iter().enumerate() { - let shuffled_i = get_permutated_index( - i, - active_validator_indices.len(), - &seed[..], - spec.shuffle_round_count, - ) - .ok_or_else(|| Error::UnableToShuffle)?; - shuffled_active_validator_indices[i] = active_validator_indices[shuffled_i] - } + let active_validator_indices: Vec = + active_validator_indices.iter().cloned().collect(); + + let shuffled_active_validator_indices = get_permutated_list( + &active_validator_indices, + &seed[..], + spec.shuffle_round_count, + ) + .ok_or_else(|| Error::UnableToShuffle)?; Ok(shuffled_active_validator_indices .honey_badger_split(committees_per_epoch as usize) diff --git a/eth2/utils/swap_or_not_shuffle/benches/benches.rs b/eth2/utils/swap_or_not_shuffle/benches/benches.rs index 1d5b5476c..b1311b41e 100644 --- a/eth2/utils/swap_or_not_shuffle/benches/benches.rs +++ b/eth2/utils/swap_or_not_shuffle/benches/benches.rs @@ -1,6 +1,6 @@ use criterion::Criterion; use criterion::{black_box, criterion_group, criterion_main, Benchmark}; -use swap_or_not_shuffle::get_permutated_index; +use swap_or_not_shuffle::{get_permutated_index, get_permutated_list}; const SHUFFLE_ROUND_COUNT: u8 = 90; @@ -48,6 +48,16 @@ fn shuffles(c: &mut Criterion) { .sample_size(10), ); + c.bench( + "_fast_ whole list shuffle", + Benchmark::new("512 elements", move |b| { + let seed = vec![42; 32]; + let list: Vec = (0..512).collect(); + b.iter(|| black_box(get_permutated_list(&list, &seed, SHUFFLE_ROUND_COUNT))) + }) + .sample_size(10), + ); + c.bench( "whole list shuffle", Benchmark::new("16384 elements", move |b| { @@ -56,6 +66,16 @@ fn shuffles(c: &mut Criterion) { }) .sample_size(10), ); + + c.bench( + "_fast_ whole list shuffle", + Benchmark::new("16384 elements", move |b| { + let seed = vec![42; 32]; + let list: Vec = (0..16384).collect(); + b.iter(|| black_box(get_permutated_list(&list, &seed, SHUFFLE_ROUND_COUNT))) + }) + .sample_size(10), + ); } criterion_group!(benches, shuffles,); diff --git a/eth2/utils/swap_or_not_shuffle/src/lib.rs b/eth2/utils/swap_or_not_shuffle/src/lib.rs index 753265f3e..b9140b269 100644 --- a/eth2/utils/swap_or_not_shuffle/src/lib.rs +++ b/eth2/utils/swap_or_not_shuffle/src/lib.rs @@ -4,6 +4,36 @@ use int_to_bytes::{int_to_bytes1, int_to_bytes4}; use std::cmp::max; use std::io::Cursor; +pub fn get_permutated_list( + list: &[usize], + seed: &[u8], + shuffle_round_count: u8, +) -> Option> { + let list_size = list.len(); + + if list_size == 0 || list_size > usize::max_value() / 2 || list_size > 2_usize.pow(24) { + return None; + } + + let mut pivots = Vec::with_capacity(shuffle_round_count as usize); + for round in 0..shuffle_round_count { + pivots.push(bytes_to_int64(&hash_with_round(seed, round)[..]) as usize % list_size); + } + + let mut output = Vec::with_capacity(list_size); + + for i in 0..list_size { + let mut index = i; + for round in 0..shuffle_round_count { + let pivot = pivots[round as usize]; + index = do_round(seed, index, pivot, round, list_size)?; + } + output.push(list[index]) + } + + Some(output) +} + /// Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy. /// /// Utilizes 'swap or not' shuffling found in @@ -32,16 +62,20 @@ pub fn get_permutated_index( let mut index = index; for round in 0..shuffle_round_count { let pivot = bytes_to_int64(&hash_with_round(seed, round)[..]) as usize % list_size; - let flip = (pivot + list_size - index) % list_size; - let position = max(index, flip); - let source = hash_with_round_and_position(seed, round, position)?; - let byte = source[(position % 256) / 8]; - let bit = (byte >> (position % 8)) % 2; - index = if bit == 1 { flip } else { index } + index = do_round(seed, index, pivot, round, list_size)?; } Some(index) } +fn do_round(seed: &[u8], index: usize, pivot: usize, round: u8, list_size: usize) -> Option { + let flip = (pivot + list_size - index) % list_size; + let position = max(index, flip); + let source = hash_with_round_and_position(seed, round, position)?; + let byte = source[(position % 256) / 8]; + let bit = (byte >> (position % 8)) % 2; + Some(if bit == 1 { flip } else { index }) +} + fn hash_with_round_and_position(seed: &[u8], round: u8, position: usize) -> Option> { let mut seed = seed.to_vec(); seed.append(&mut int_to_bytes1(round)); From 056a263f0294acb17e74383910432e7a22ffc2ad Mon Sep 17 00:00:00 2001 From: Micah Dameron Date: Thu, 28 Feb 2019 18:05:11 -0700 Subject: [PATCH 044/132] modified: README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7759c1166..6523e9202 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ by the team: from the Ethereum Foundation to develop *simpleserialize* (SSZ), a purpose-built serialization format for sending information across a network. Check out the [SSZ -implementation](https://github.com/sigp/lighthouse/tree/master/beacon_chain/utils/ssz) +implementation](https://github.com/ethereum/eth2.0-specs/blob/00aa553fee95963b74fbec84dbd274d7247b8a0e/specs/simple-serialize.md) and this [research](https://github.com/sigp/serialization_sandbox/blob/report/report/serialization_report.md) on serialization formats for more information. From c3b2f802a78256f526353f4c6c5322b672df43e8 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Mar 2019 12:19:05 +1100 Subject: [PATCH 045/132] Add fast full-list shuffle for swap-or-not - Passes test vectors - Implemented in beacon state - Added more docs --- eth2/types/src/beacon_state.rs | 9 +- .../swap_or_not_shuffle/benches/benches.rs | 16 +- .../src/get_permutated_index.rs | 187 ++++++++++++++ eth2/utils/swap_or_not_shuffle/src/lib.rs | 229 ++---------------- .../swap_or_not_shuffle/src/shuffle_list.rs | 178 ++++++++++++++ 5 files changed, 402 insertions(+), 217 deletions(-) create mode 100644 eth2/utils/swap_or_not_shuffle/src/get_permutated_index.rs create mode 100644 eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 2d4f463b7..67cf2341a 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -12,7 +12,7 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; use std::collections::HashMap; -use swap_or_not_shuffle::get_permutated_list; +use swap_or_not_shuffle::shuffle_list; pub use builder::BeaconStateBuilder; @@ -423,10 +423,11 @@ impl BeaconState { let active_validator_indices: Vec = active_validator_indices.iter().cloned().collect(); - let shuffled_active_validator_indices = get_permutated_list( - &active_validator_indices, - &seed[..], + let shuffled_active_validator_indices = shuffle_list( + active_validator_indices, spec.shuffle_round_count, + &seed[..], + true, ) .ok_or_else(|| Error::UnableToShuffle)?; diff --git a/eth2/utils/swap_or_not_shuffle/benches/benches.rs b/eth2/utils/swap_or_not_shuffle/benches/benches.rs index b1311b41e..0502e6fc4 100644 --- a/eth2/utils/swap_or_not_shuffle/benches/benches.rs +++ b/eth2/utils/swap_or_not_shuffle/benches/benches.rs @@ -1,6 +1,6 @@ use criterion::Criterion; use criterion::{black_box, criterion_group, criterion_main, Benchmark}; -use swap_or_not_shuffle::{get_permutated_index, get_permutated_list}; +use swap_or_not_shuffle::{get_permutated_index, shuffle_list as fast_shuffle}; const SHUFFLE_ROUND_COUNT: u8 = 90; @@ -53,7 +53,7 @@ fn shuffles(c: &mut Criterion) { Benchmark::new("512 elements", move |b| { let seed = vec![42; 32]; let list: Vec = (0..512).collect(); - b.iter(|| black_box(get_permutated_list(&list, &seed, SHUFFLE_ROUND_COUNT))) + b.iter(|| black_box(fast_shuffle(list.clone(), SHUFFLE_ROUND_COUNT, &seed, true))) }) .sample_size(10), ); @@ -72,7 +72,17 @@ fn shuffles(c: &mut Criterion) { Benchmark::new("16384 elements", move |b| { let seed = vec![42; 32]; let list: Vec = (0..16384).collect(); - b.iter(|| black_box(get_permutated_list(&list, &seed, SHUFFLE_ROUND_COUNT))) + b.iter(|| black_box(fast_shuffle(list.clone(), SHUFFLE_ROUND_COUNT, &seed, true))) + }) + .sample_size(10), + ); + + c.bench( + "_fast_ whole list shuffle", + Benchmark::new("4m elements", move |b| { + let seed = vec![42; 32]; + let list: Vec = (0..4_000_000).collect(); + b.iter(|| black_box(fast_shuffle(list.clone(), SHUFFLE_ROUND_COUNT, &seed, true))) }) .sample_size(10), ); diff --git a/eth2/utils/swap_or_not_shuffle/src/get_permutated_index.rs b/eth2/utils/swap_or_not_shuffle/src/get_permutated_index.rs new file mode 100644 index 000000000..37a82341e --- /dev/null +++ b/eth2/utils/swap_or_not_shuffle/src/get_permutated_index.rs @@ -0,0 +1,187 @@ +use bytes::Buf; +use hashing::hash; +use int_to_bytes::{int_to_bytes1, int_to_bytes4}; +use std::cmp::max; +use std::io::Cursor; + +/// Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy. +/// +/// Utilizes 'swap or not' shuffling found in +/// https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf +/// See the 'generalized domain' algorithm on page 3. +/// +/// Note: this function is significantly slower than the `shuffle_list` function in this crate. +/// Using `get_permutated_list` to shuffle an entire list, index by index, has been observed to be +/// 250x slower than `shuffle_list`. Therefore, this function is only useful when shuffling a small +/// portion of a much larger list. +/// +/// Returns `None` under any of the following conditions: +/// - `list_size == 0` +/// - `index >= list_size` +/// - `list_size > 2**24` +/// - `list_size > usize::max_value() / 2` +pub fn get_permutated_index( + index: usize, + list_size: usize, + seed: &[u8], + shuffle_round_count: u8, +) -> Option { + if list_size == 0 + || index >= list_size + || list_size > usize::max_value() / 2 + || list_size > 2_usize.pow(24) + { + return None; + } + + let mut index = index; + for round in 0..shuffle_round_count { + let pivot = bytes_to_int64(&hash_with_round(seed, round)[..]) as usize % list_size; + index = do_round(seed, index, pivot, round, list_size)?; + } + Some(index) +} + +fn do_round(seed: &[u8], index: usize, pivot: usize, round: u8, list_size: usize) -> Option { + let flip = (pivot + list_size - index) % list_size; + let position = max(index, flip); + let source = hash_with_round_and_position(seed, round, position)?; + let byte = source[(position % 256) / 8]; + let bit = (byte >> (position % 8)) % 2; + Some(if bit == 1 { flip } else { index }) +} + +fn hash_with_round_and_position(seed: &[u8], round: u8, position: usize) -> Option> { + let mut seed = seed.to_vec(); + seed.append(&mut int_to_bytes1(round)); + /* + * Note: the specification has an implicit assertion in `int_to_bytes4` that `position / 256 < + * 2**24`. For efficiency, we do not check for that here as it is checked in `get_permutated_index`. + */ + seed.append(&mut int_to_bytes4((position / 256) as u32)); + Some(hash(&seed[..])) +} + +fn hash_with_round(seed: &[u8], round: u8) -> Vec { + let mut seed = seed.to_vec(); + seed.append(&mut int_to_bytes1(round)); + hash(&seed[..]) +} + +fn bytes_to_int64(bytes: &[u8]) -> u64 { + let mut cursor = Cursor::new(bytes); + cursor.get_u64_le() +} + +#[cfg(test)] +mod tests { + use super::*; + use ethereum_types::H256 as Hash256; + use hex; + use std::{fs::File, io::prelude::*, path::PathBuf}; + use yaml_rust::yaml; + + #[test] + #[ignore] + fn fuzz_test() { + let max_list_size = 2_usize.pow(24); + let test_runs = 1000; + + // Test at max list_size with the end index. + for _ in 0..test_runs { + let index = max_list_size - 1; + let list_size = max_list_size; + let seed = Hash256::random(); + let shuffle_rounds = 90; + + assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some()); + } + + // Test at max list_size low indices. + for i in 0..test_runs { + let index = i; + let list_size = max_list_size; + let seed = Hash256::random(); + let shuffle_rounds = 90; + + assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some()); + } + + // Test at max list_size high indices. + for i in 0..test_runs { + let index = max_list_size - 1 - i; + let list_size = max_list_size; + let seed = Hash256::random(); + let shuffle_rounds = 90; + + assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some()); + } + } + + #[test] + fn returns_none_for_zero_length_list() { + assert_eq!(None, get_permutated_index(100, 0, &[42, 42], 90)); + } + + #[test] + fn returns_none_for_out_of_bounds_index() { + assert_eq!(None, get_permutated_index(100, 100, &[42, 42], 90)); + } + + #[test] + fn returns_none_for_too_large_list() { + assert_eq!( + None, + get_permutated_index(100, usize::max_value() / 2, &[42, 42], 90) + ); + } + + #[test] + fn test_vectors() { + /* + * Test vectors are generated here: + * + * https://github.com/ethereum/eth2.0-test-generators + */ + let mut file = { + let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + file_path_buf.push("src/specs/test_vector_permutated_index.yml"); + + File::open(file_path_buf).unwrap() + }; + + let mut yaml_str = String::new(); + + file.read_to_string(&mut yaml_str).unwrap(); + + let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap(); + let doc = &docs[0]; + let test_cases = doc["test_cases"].as_vec().unwrap(); + + for (i, test_case) in test_cases.iter().enumerate() { + let index = test_case["index"].as_i64().unwrap() as usize; + let list_size = test_case["list_size"].as_i64().unwrap() as usize; + let permutated_index = test_case["permutated_index"].as_i64().unwrap() as usize; + let shuffle_round_count = test_case["shuffle_round_count"].as_i64().unwrap(); + let seed_string = test_case["seed"].clone().into_string().unwrap(); + let seed = hex::decode(seed_string.replace("0x", "")).unwrap(); + + let shuffle_round_count = if shuffle_round_count < (u8::max_value() as i64) { + shuffle_round_count as u8 + } else { + panic!("shuffle_round_count must be a u8") + }; + + assert_eq!( + Some(permutated_index), + get_permutated_index(index, list_size, &seed[..], shuffle_round_count), + "Failure on case #{} index: {}, list_size: {}, round_count: {}, seed: {}", + i, + index, + list_size, + shuffle_round_count, + seed_string, + ); + } + } +} diff --git a/eth2/utils/swap_or_not_shuffle/src/lib.rs b/eth2/utils/swap_or_not_shuffle/src/lib.rs index b9140b269..57049fbdf 100644 --- a/eth2/utils/swap_or_not_shuffle/src/lib.rs +++ b/eth2/utils/swap_or_not_shuffle/src/lib.rs @@ -1,212 +1,21 @@ -use bytes::Buf; -use hashing::hash; -use int_to_bytes::{int_to_bytes1, int_to_bytes4}; -use std::cmp::max; -use std::io::Cursor; +//! Provides list-shuffling functions matching the Ethereum 2.0 specification. +//! +//! See +//! [get_permutated_index](https://github.com/ethereum/eth2.0-specs/blob/0.4.0/specs/core/0_beacon-chain.md#get_permuted_index) +//! for specifications. +//! +//! There are two functions exported by this crate: +//! +//! - `get_permutated_index`: given a single index, computes the index resulting from a shuffle. +//! Runs in less time than it takes to run `shuffle_list`. +//! - `shuffle_list`: shuffles an entire list in-place. Runs in less time than it takes to run +//! `get_permutated_index` on each index. +//! +//! In general, use `get_permutated_list` to calculate the shuffling of a small subset of a much +//! larger list (~250x larger is a good guide, but solid figures yet to be calculated). -pub fn get_permutated_list( - list: &[usize], - seed: &[u8], - shuffle_round_count: u8, -) -> Option> { - let list_size = list.len(); +mod get_permutated_index; +mod shuffle_list; - if list_size == 0 || list_size > usize::max_value() / 2 || list_size > 2_usize.pow(24) { - return None; - } - - let mut pivots = Vec::with_capacity(shuffle_round_count as usize); - for round in 0..shuffle_round_count { - pivots.push(bytes_to_int64(&hash_with_round(seed, round)[..]) as usize % list_size); - } - - let mut output = Vec::with_capacity(list_size); - - for i in 0..list_size { - let mut index = i; - for round in 0..shuffle_round_count { - let pivot = pivots[round as usize]; - index = do_round(seed, index, pivot, round, list_size)?; - } - output.push(list[index]) - } - - Some(output) -} - -/// Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy. -/// -/// Utilizes 'swap or not' shuffling found in -/// https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf -/// See the 'generalized domain' algorithm on page 3. -/// -/// Returns `None` under any of the following conditions: -/// - `list_size == 0` -/// - `index >= list_size` -/// - `list_size > 2**24` -/// - `list_size > usize::max_value() / 2` -pub fn get_permutated_index( - index: usize, - list_size: usize, - seed: &[u8], - shuffle_round_count: u8, -) -> Option { - if list_size == 0 - || index >= list_size - || list_size > usize::max_value() / 2 - || list_size > 2_usize.pow(24) - { - return None; - } - - let mut index = index; - for round in 0..shuffle_round_count { - let pivot = bytes_to_int64(&hash_with_round(seed, round)[..]) as usize % list_size; - index = do_round(seed, index, pivot, round, list_size)?; - } - Some(index) -} - -fn do_round(seed: &[u8], index: usize, pivot: usize, round: u8, list_size: usize) -> Option { - let flip = (pivot + list_size - index) % list_size; - let position = max(index, flip); - let source = hash_with_round_and_position(seed, round, position)?; - let byte = source[(position % 256) / 8]; - let bit = (byte >> (position % 8)) % 2; - Some(if bit == 1 { flip } else { index }) -} - -fn hash_with_round_and_position(seed: &[u8], round: u8, position: usize) -> Option> { - let mut seed = seed.to_vec(); - seed.append(&mut int_to_bytes1(round)); - /* - * Note: the specification has an implicit assertion in `int_to_bytes4` that `position / 256 < - * 2**24`. For efficiency, we do not check for that here as it is checked in `get_permutated_index`. - */ - seed.append(&mut int_to_bytes4((position / 256) as u32)); - Some(hash(&seed[..])) -} - -fn hash_with_round(seed: &[u8], round: u8) -> Vec { - let mut seed = seed.to_vec(); - seed.append(&mut int_to_bytes1(round)); - hash(&seed[..]) -} - -fn bytes_to_int64(bytes: &[u8]) -> u64 { - let mut cursor = Cursor::new(bytes); - cursor.get_u64_le() -} - -#[cfg(test)] -mod tests { - use super::*; - use ethereum_types::H256 as Hash256; - use hex; - use std::{fs::File, io::prelude::*, path::PathBuf}; - use yaml_rust::yaml; - - #[test] - #[ignore] - fn fuzz_test() { - let max_list_size = 2_usize.pow(24); - let test_runs = 1000; - - // Test at max list_size with the end index. - for _ in 0..test_runs { - let index = max_list_size - 1; - let list_size = max_list_size; - let seed = Hash256::random(); - let shuffle_rounds = 90; - - assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some()); - } - - // Test at max list_size low indices. - for i in 0..test_runs { - let index = i; - let list_size = max_list_size; - let seed = Hash256::random(); - let shuffle_rounds = 90; - - assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some()); - } - - // Test at max list_size high indices. - for i in 0..test_runs { - let index = max_list_size - 1 - i; - let list_size = max_list_size; - let seed = Hash256::random(); - let shuffle_rounds = 90; - - assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some()); - } - } - - #[test] - fn returns_none_for_zero_length_list() { - assert_eq!(None, get_permutated_index(100, 0, &[42, 42], 90)); - } - - #[test] - fn returns_none_for_out_of_bounds_index() { - assert_eq!(None, get_permutated_index(100, 100, &[42, 42], 90)); - } - - #[test] - fn returns_none_for_too_large_list() { - assert_eq!( - None, - get_permutated_index(100, usize::max_value() / 2, &[42, 42], 90) - ); - } - - #[test] - fn test_vectors() { - /* - * Test vectors are generated here: - * - * https://github.com/ethereum/eth2.0-test-generators - */ - let mut file = { - let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - file_path_buf.push("src/specs/test_vector_permutated_index.yml"); - - File::open(file_path_buf).unwrap() - }; - - let mut yaml_str = String::new(); - - file.read_to_string(&mut yaml_str).unwrap(); - - let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap(); - let doc = &docs[0]; - let test_cases = doc["test_cases"].as_vec().unwrap(); - - for (i, test_case) in test_cases.iter().enumerate() { - let index = test_case["index"].as_i64().unwrap() as usize; - let list_size = test_case["list_size"].as_i64().unwrap() as usize; - let permutated_index = test_case["permutated_index"].as_i64().unwrap() as usize; - let shuffle_round_count = test_case["shuffle_round_count"].as_i64().unwrap(); - let seed_string = test_case["seed"].clone().into_string().unwrap(); - let seed = hex::decode(seed_string.replace("0x", "")).unwrap(); - - let shuffle_round_count = if shuffle_round_count < (u8::max_value() as i64) { - shuffle_round_count as u8 - } else { - panic!("shuffle_round_count must be a u8") - }; - - assert_eq!( - Some(permutated_index), - get_permutated_index(index, list_size, &seed[..], shuffle_round_count), - "Failure on case #{} index: {}, list_size: {}, round_count: {}, seed: {}", - i, - index, - list_size, - shuffle_round_count, - seed_string, - ); - } - } -} +pub use get_permutated_index::get_permutated_index; +pub use shuffle_list::shuffle_list; diff --git a/eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs b/eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs new file mode 100644 index 000000000..e7e1e18e6 --- /dev/null +++ b/eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs @@ -0,0 +1,178 @@ +use bytes::Buf; +use hashing::hash; +use int_to_bytes::int_to_bytes4; +use std::io::Cursor; + +const SEED_SIZE: usize = 32; +const ROUND_SIZE: usize = 1; +const POSITION_WINDOW_SIZE: usize = 4; +const PIVOT_VIEW_SIZE: usize = SEED_SIZE + ROUND_SIZE; +const TOTAL_SIZE: usize = SEED_SIZE + ROUND_SIZE + POSITION_WINDOW_SIZE; + +/// Shuffles an entire list in-place. +/// +/// Note: this is equivalent to the `get_permutated_index` function, except it shuffles an entire +/// list not just a single index. With large lists this function has been observed to be 250x +/// faster than running `get_permutated_index` across an entire list. +/// +/// Credits to [@protolambda](https://github.com/protolambda) for defining this algorithm. +/// +/// Shuffles if `forwards == true`, otherwise un-shuffles. +/// +/// Returns `None` under any of the following conditions: +/// - `list_size == 0` +/// - `list_size > 2**24` +/// - `list_size > usize::max_value() / 2` +pub fn shuffle_list( + mut input: Vec, + rounds: u8, + seed: &[u8], + forwards: bool, +) -> Option> { + let list_size = input.len(); + + if input.is_empty() + || list_size > usize::max_value() / 2 + || list_size > 2_usize.pow(24) + || rounds == 0 + { + return None; + } + + let mut buf: Vec = Vec::with_capacity(TOTAL_SIZE); + + let mut r = if forwards { 0 } else { rounds - 1 }; + + buf.extend_from_slice(seed); + + loop { + buf.splice(SEED_SIZE.., vec![r]); + + let pivot = bytes_to_int64(&hash(&buf[0..PIVOT_VIEW_SIZE])[0..8]) as usize % list_size; + + let mirror = (pivot + 1) >> 1; + + buf.splice(PIVOT_VIEW_SIZE.., int_to_bytes4((pivot >> 8) as u32)); + let mut source = hash(&buf[..]); + let mut byte_v = source[(pivot & 0xff) >> 3]; + + for i in 0..mirror { + let j = pivot - i; + + if j & 0xff == 0xff { + buf.splice(PIVOT_VIEW_SIZE.., int_to_bytes4((j >> 8) as u32)); + source = hash(&buf[..]); + } + + if j & 0x07 == 0x07 { + byte_v = source[(j & 0xff) >> 3]; + } + let bit_v = (byte_v >> (j & 0x07)) & 0x01; + + if bit_v == 1 { + input.swap(i, j); + } + } + + let mirror = (pivot + list_size + 1) >> 1; + let end = list_size - 1; + + buf.splice(PIVOT_VIEW_SIZE.., int_to_bytes4((end >> 8) as u32)); + let mut source = hash(&buf[..]); + let mut byte_v = source[(end & 0xff) >> 3]; + + for (loop_iter, i) in ((pivot + 1)..mirror).enumerate() { + let j = end - loop_iter; + + if j & 0xff == 0xff { + buf.splice(PIVOT_VIEW_SIZE.., int_to_bytes4((j >> 8) as u32)); + source = hash(&buf[..]); + } + + if j & 0x07 == 0x07 { + byte_v = source[(j & 0xff) >> 3]; + } + let bit_v = (byte_v >> (j & 0x07)) & 0x01; + + if bit_v == 1 { + input.swap(i, j); + } + } + + if forwards { + r += 1; + if r == rounds { + break; + } + } else { + if r == 0 { + break; + } + r -= 1; + } + } + + Some(input) +} + +fn bytes_to_int64(bytes: &[u8]) -> u64 { + let mut cursor = Cursor::new(bytes); + cursor.get_u64_le() +} + +#[cfg(test)] +mod tests { + use super::*; + use hex; + use std::{fs::File, io::prelude::*, path::PathBuf}; + use yaml_rust::yaml; + + #[test] + fn returns_none_for_zero_length_list() { + assert_eq!(None, shuffle_list(vec![], 90, &[42, 42], true)); + } + + #[test] + fn test_vectors() { + let mut file = { + let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + file_path_buf.push("src/specs/test_vector_permutated_index.yml"); + + File::open(file_path_buf).unwrap() + }; + + let mut yaml_str = String::new(); + + file.read_to_string(&mut yaml_str).unwrap(); + + let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap(); + let doc = &docs[0]; + let test_cases = doc["test_cases"].as_vec().unwrap(); + + for (i, test_case) in test_cases.iter().enumerate() { + let index = test_case["index"].as_i64().unwrap() as usize; + let list_size = test_case["list_size"].as_i64().unwrap() as usize; + let permutated_index = test_case["permutated_index"].as_i64().unwrap() as usize; + let shuffle_round_count = test_case["shuffle_round_count"].as_i64().unwrap(); + let seed_string = test_case["seed"].clone().into_string().unwrap(); + let seed = hex::decode(seed_string.replace("0x", "")).unwrap(); + + let shuffle_round_count = if shuffle_round_count < (u8::max_value() as i64) { + shuffle_round_count as u8 + } else { + panic!("shuffle_round_count must be a u8") + }; + + let list: Vec = (0..list_size).collect(); + + let shuffled = + shuffle_list(list.clone(), shuffle_round_count, &seed[..], true).unwrap(); + + assert_eq!( + list[index], shuffled[permutated_index], + "Failure on case #{} index: {}, list_size: {}, round_count: {}, seed: {}", + i, index, list_size, shuffle_round_count, seed_string + ); + } + } +} From 3753782c40187d06df310730f773acfef916d5fa Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Fri, 1 Mar 2019 13:10:50 +1100 Subject: [PATCH 046/132] Just some small gramatical improvements on READMEs --- README.md | 8 ++++---- eth2/README.md | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index c26dae219..8552d64ee 100644 --- a/README.md +++ b/README.md @@ -33,12 +33,12 @@ If you'd like some background on Sigma Prime, please see the [Lighthouse Update - [`beacon_node/`](beacon_node/): the "Beacon Node" binary and crates exclusively associated with it. -- [`docs/`](docs/): documentation related to the repository. This include contributor - guides, etc. (Code documentation is produced with `cargo doc`). +- [`docs/`](docs/): documentation related to the repository. This includes contributor + guides, etc. (It does not include code documentation, which can be produced with `cargo doc`). - [`eth2/`](eth2/): Crates containing common logic across the Lighthouse project. For - example; Ethereum 2.0 types ([`BeaconBlock`](eth2/types/src/beacon_block.rs), [`BeaconState`](eth2/types/src/beacon_state.rs), etc) and + example: Ethereum 2.0 types ([`BeaconBlock`](eth2/types/src/beacon_block.rs), [`BeaconState`](eth2/types/src/beacon_state.rs), etc) and SimpleSerialize (SSZ). -- [`protos/`](protos/): protobuf/gRPC definitions common across the Lighthouse project. +- [`protos/`](protos/): protobuf/gRPC definitions that are common across the Lighthouse project. - [`validator_client/`](validator_client/): the "Validator Client" binary and crates exclusively associated with it. diff --git a/eth2/README.md b/eth2/README.md index e7b69635a..cf041e987 100644 --- a/eth2/README.md +++ b/eth2/README.md @@ -8,19 +8,19 @@ Rust crates containing logic common across the Lighthouse project. - [`block_proposer/`](block_proposer/): Core logic for proposing beacon blocks. - [`fork_choice/`](fork_choice/): A collection of fork-choice algorithms for the Beacon Chain. -- [`state_processing/`](state_processing/): Provides per-slot, per-block and +- [`state_processing/`](state_processing/): Provides per-slot, per-block, and per-epoch state processing. - [`types/`](types/): Defines base Ethereum 2.0 types (e.g., `BeaconBlock`, `BeaconState`, etc). - [`utils/`](utils/): - - [`bls`](utils/bls/): A wrapper around an external BLS encryption library. - - [`boolean-bitfield`](utils/boolean-bitfield/): Provides an expandable Vec + - [`bls`](utils/bls/): A wrapper for an external BLS encryption library. + - [`boolean-bitfield`](utils/boolean-bitfield/): Provides an expandable vector of bools, specifically for use in Eth2. - [`fisher-yates-shuffle`](utils/fisher-yates-shuffle/): shuffles a list pseudo-randomly. - - [`hashing`](utils/hashing/): A wrapper around external hashing libraries. + - [`hashing`](utils/hashing/): A wrapper for external hashing libraries. - [`honey-badger-split`](utils/honey-badger-split/): Splits a list in `n` - parts without giving AF about the length of the list, `n` or anything + parts without giving AF about the length of the list, `n`, or anything else. - [`int-to-bytes`](utils/int-to-bytes/): Simple library which converts ints into byte-strings of various lengths. @@ -30,8 +30,8 @@ Rust crates containing logic common across the Lighthouse project. - [`ssz`](utils/ssz/): an implementation of the SimpleSerialize serialization/deserialization protocol used by Eth 2.0. - [`ssz_derive`](utils/ssz_derive/): provides procedural macros for - deriving SSZ `Encodable`, `Decodable` and `TreeHash` methods. + deriving SSZ `Encodable`, `Decodable`, and `TreeHash` methods. - [`swap_or_not_shuffle`](utils/swap_or_not_shuffle/): a list-shuffling - method which is slow, but allows for shuffling a subset of indices. + method which is slow, but allows for a subset of indices to be shuffled. - [`test_random_derive`](utils/test_random_derive/): provides procedural macros for deriving the `TestRandom` trait defined in `types`. From 1479013bd01bdd913aa05ad4f9edf360d4f0e41c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Mar 2019 13:28:07 +1100 Subject: [PATCH 047/132] Add skip_slots to test_harness yaml processor --- .../test_harness/examples/chain.yaml | 1 + .../beacon_chain/test_harness/src/bin.rs | 51 +++++++++++++++---- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/examples/chain.yaml b/beacon_node/beacon_chain/test_harness/examples/chain.yaml index 5d8e34795..b170a0b2e 100644 --- a/beacon_node/beacon_chain/test_harness/examples/chain.yaml +++ b/beacon_node/beacon_chain/test_harness/examples/chain.yaml @@ -8,6 +8,7 @@ test_cases: epoch_length: 64 deposits_for_chain_start: 1000 num_slots: 32 # Testing advancing state to slot < SlotsPerEpoch + skip_slots: [2, 3] results: slot: 32 num_validators: 1000 diff --git a/beacon_node/beacon_chain/test_harness/src/bin.rs b/beacon_node/beacon_chain/test_harness/src/bin.rs index 007ec3f60..48f349d4a 100644 --- a/beacon_node/beacon_chain/test_harness/src/bin.rs +++ b/beacon_node/beacon_chain/test_harness/src/bin.rs @@ -1,8 +1,10 @@ use self::beacon_chain_harness::BeaconChainHarness; use self::validator_harness::ValidatorHarness; +use beacon_chain::CheckPoint; use clap::{App, Arg}; use env_logger::{Builder, Env}; -use log::info; +use log::{info, warn}; +use std::collections::HashMap; use std::{fs::File, io::prelude::*}; use types::*; use yaml_rust::{Yaml, YamlLoader}; @@ -39,7 +41,7 @@ fn main() { for doc in &docs { for test_case in doc["test_cases"].as_vec().unwrap() { let manifest = Manifest::from_yaml(test_case); - manifest.execute(); + manifest.assert_result_valid(manifest.execute()) } } } @@ -68,7 +70,7 @@ impl Manifest { spec } - pub fn execute(&self) { + pub fn execute(&self) -> ExecutionResult { let spec = self.spec(); let validator_count = self.config.deposits_for_chain_start; let slots = self.results.slot; @@ -82,18 +84,47 @@ impl Manifest { info!("Starting simulation across {} slots...", slots); - for _ in 0..self.results.slot { - harness.advance_chain_with_block(); + for slot_height in 0..self.results.slot { + match self.config.skip_slots { + Some(ref skip_slots) if skip_slots.contains(&slot_height) => { + warn!("Skipping slot at height {}.", slot_height); + harness.increment_beacon_chain_slot(); + } + _ => { + info!("Producing block at slot height {}.", slot_height); + harness.advance_chain_with_block(); + } + } } harness.run_fork_choice(); - let dump = harness.chain_dump().expect("Chain dump failed."); + info!("Test execution complete!"); - assert_eq!(dump.len() as u64, slots + 1); // + 1 for genesis block. - - // harness.dump_to_file("/tmp/chaindump.json".to_string(), &dump); + ExecutionResult { + chain: harness.chain_dump().expect("Chain dump failed."), + } } + + pub fn assert_result_valid(&self, result: ExecutionResult) { + info!("Verifying test results..."); + + if let Some(ref skip_slots) = self.config.skip_slots { + for checkpoint in result.chain { + let block_slot = checkpoint.beacon_block.slot.as_u64(); + assert!( + !skip_slots.contains(&block_slot), + "Slot {} was not skipped.", + block_slot + ); + } + } + info!("OK: Skipped slots not present in chain."); + } +} + +struct ExecutionResult { + pub chain: Vec, } struct Results { @@ -117,6 +148,7 @@ impl Results { struct Config { pub deposits_for_chain_start: usize, pub epoch_length: Option, + pub skip_slots: Option>, } impl Config { @@ -125,6 +157,7 @@ impl Config { deposits_for_chain_start: as_usize(&yaml, "deposits_for_chain_start") .expect("Must specify validator count"), epoch_length: as_u64(&yaml, "epoch_length"), + skip_slots: as_vec_u64(yaml, "skip_slots"), } } } From 13c957bef7623fbac689330b0aa221b404443e14 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Fri, 1 Mar 2019 14:38:07 +1100 Subject: [PATCH 048/132] Correct bitwise fork-choice rule. --- eth2/fork_choice/src/bitwise_lmd_ghost.rs | 9 ++++-- .../tests/bitwise_lmd_ghost_test_vectors.yaml | 28 +++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/eth2/fork_choice/src/bitwise_lmd_ghost.rs b/eth2/fork_choice/src/bitwise_lmd_ghost.rs index e1d246e92..1e66de079 100644 --- a/eth2/fork_choice/src/bitwise_lmd_ghost.rs +++ b/eth2/fork_choice/src/bitwise_lmd_ghost.rs @@ -192,7 +192,7 @@ where } // Check if there is a clear block winner at this height. If so return it. for (hash, votes) in current_votes.iter() { - if *votes >= total_vote_count / 2 { + if *votes > total_vote_count / 2 { // we have a clear winner, return it return Some(*hash); } @@ -371,7 +371,10 @@ impl ForkChoice for BitwiseLMDGhost { // if there are no children, we are done, return the current_head let children = match self.children.get(¤t_head) { Some(children) => children.clone(), - None => return Ok(current_head), + None => { + debug!("Head found: {}", current_head); + return Ok(current_head); + } }; // logarithmic lookup blocks to see if there are obvious winners, if so, @@ -391,7 +394,7 @@ impl ForkChoice for BitwiseLMDGhost { step /= 2; } if step > 0 { - trace!("Found clear winner in log lookup"); + trace!("Found clear winner: {}", current_head); } // if our skip lookup failed and we only have one child, progress to that child else if children.len() == 1 { diff --git a/eth2/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml b/eth2/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml index 1578673cd..3233137ab 100644 --- a/eth2/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml +++ b/eth2/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml @@ -35,3 +35,31 @@ test_cases: - b3: 3 heads: - id: 'b2' +- blocks: + - id: 'b0' + parent: 'b0' + - id: 'b1' + parent: 'b0' + - id: 'b2' + parent: 'b0' + - id: 'b3' + parent: 'b1' + - id: 'b4' + parent: 'b1' + - id: 'b5' + parent: 'b1' + - id: 'b6' + parent: 'b2' + - id: 'b7' + parent: 'b6' + weights: + - b0: 0 + - b1: 3 + - b2: 2 + - b3: 1 + - b4: 1 + - b5: 1 + - b6: 2 + - b7: 2 + heads: + - id: 'b4' From 1097c8089b5baa6da8280cb722098d90228e9fcd Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Mar 2019 16:54:59 +1100 Subject: [PATCH 049/132] Add naive deposit-handling to BeaconChain --- beacon_node/beacon_chain/src/beacon_chain.rs | 39 +++++++++++++++++-- .../state_processing/src/block_processable.rs | 28 ++++++++++--- 2 files changed, 58 insertions(+), 9 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 1065f661d..80cc79305 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -15,9 +15,7 @@ use state_processing::{ use std::sync::Arc; use types::{ readers::{BeaconBlockReader, BeaconStateReader}, - AttestationData, BeaconBlock, BeaconBlockBody, BeaconState, BeaconStateError, ChainSpec, - Crosslink, Deposit, Epoch, Eth1Data, FreeAttestation, Hash256, PublicKey, RelativeEpoch, - Signature, Slot, + *, }; #[derive(Debug, PartialEq)] @@ -66,6 +64,7 @@ pub struct BeaconChain { pub state_store: Arc>, pub slot_clock: U, pub attestation_aggregator: RwLock, + pub deposits_for_inclusion: RwLock>, canonical_head: RwLock, finalized_head: RwLock, pub state: RwLock, @@ -132,6 +131,7 @@ where state_store, slot_clock, attestation_aggregator, + deposits_for_inclusion: RwLock::new(vec![]), state: RwLock::new(genesis_state), finalized_head, canonical_head, @@ -364,6 +364,34 @@ where Ok(aggregation_outcome) } + pub fn receive_deposit_for_inclusion(&self, deposit: Deposit) { + // TODO: deposits are not check for validity; check them. + self.deposits_for_inclusion.write().push(deposit); + } + + pub fn get_deposits_for_block(&self) -> Vec { + // TODO: deposits are indiscriminately included; check them for validity. + self.deposits_for_inclusion.read().clone() + } + + pub fn mark_deposits_as_included(&self, included_deposits: &[Deposit]) { + // TODO: method does not take forks into account; consider this. + let mut indices_to_delete = vec![]; + + for included in included_deposits { + for (i, for_inclusion) in self.deposits_for_inclusion.read().iter().enumerate() { + if included == for_inclusion { + indices_to_delete.push(i); + } + } + } + + let deposits_for_inclusion = &mut self.deposits_for_inclusion.write(); + for i in indices_to_delete { + deposits_for_inclusion.remove(i); + } + } + /// Dumps the entire canonical chain, from the head to genesis to a vector for analysis. /// /// This could be a very expensive operation and should only be done in testing/analysis @@ -488,6 +516,9 @@ where self.block_store.put(&block_root, &ssz_encode(&block)[..])?; self.state_store.put(&state_root, &ssz_encode(&state)[..])?; + // Remove any included deposits from the for-inclusion queue + self.mark_deposits_as_included(&block.body.deposits[..]); + // run the fork_choice add_block logic self.fork_choice .write() @@ -544,7 +575,7 @@ where proposer_slashings: vec![], attester_slashings: vec![], attestations, - deposits: vec![], + deposits: self.get_deposits_for_block(), exits: vec![], }, }; diff --git a/eth2/state_processing/src/block_processable.rs b/eth2/state_processing/src/block_processable.rs index effaa6507..aab87c3ad 100644 --- a/eth2/state_processing/src/block_processable.rs +++ b/eth2/state_processing/src/block_processable.rs @@ -3,10 +3,7 @@ use hashing::hash; use int_to_bytes::int_to_bytes32; use log::{debug, trace}; use ssz::{ssz_encode, TreeHash}; -use types::{ - AggregatePublicKey, Attestation, BeaconBlock, BeaconState, BeaconStateError, ChainSpec, - Crosslink, Epoch, Exit, Fork, Hash256, PendingAttestation, PublicKey, RelativeEpoch, Signature, -}; +use types::*; // TODO: define elsehwere. const DOMAIN_PROPOSAL: u64 = 2; @@ -35,6 +32,7 @@ pub enum Error { InvalidAttestation(AttestationValidationError), NoBlockRoot, MaxDepositsExceeded, + BadDeposit, MaxExitsExceeded, BadExit, BadCustodyReseeds, @@ -242,7 +240,27 @@ fn per_block_processing_signature_optional( Error::MaxDepositsExceeded ); - // TODO: process deposits. + // TODO: verify deposit merkle branches. + for deposit in &block.body.deposits { + debug!( + "Processing deposit for pubkey {:?}", + deposit.deposit_data.deposit_input.pubkey + ); + state + .process_deposit( + deposit.deposit_data.deposit_input.pubkey.clone(), + deposit.deposit_data.amount, + deposit + .deposit_data + .deposit_input + .proof_of_possession + .clone(), + deposit.deposit_data.deposit_input.withdrawal_credentials, + None, + spec, + ) + .map_err(|_| Error::BadDeposit)?; + } /* * Exits From eeeff9ef02f9d07ebfdbdf8b2c1334fa450301c8 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Mar 2019 16:56:52 +1100 Subject: [PATCH 050/132] Ensure chain-dumps come with earliest block first Previously dump.first() was the latest block. IMO, this is counter-intuitive --- beacon_node/beacon_chain/src/beacon_chain.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 80cc79305..805ccc9fd 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -440,6 +440,8 @@ where last_slot = slot; } + dump.reverse(); + Ok(dump) } From c278c08e34eefd6be05a1c64d164807238166922 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Mar 2019 16:57:24 +1100 Subject: [PATCH 051/132] Remove unnecessary clone. --- beacon_node/beacon_chain/src/beacon_chain.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 805ccc9fd..34e1a5183 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -533,7 +533,7 @@ where if self.head().beacon_block_root == parent_block_root { self.update_canonical_head(block.clone(), block_root, state.clone(), state_root); // Update the local state variable. - *self.state.write() = state.clone(); + *self.state.write() = state; } Ok(BlockProcessingOutcome::ValidBlock(ValidBlock::Processed)) From b0403707eb0f4c9cf1f8c6819f2fdd9684c6c987 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Mar 2019 16:59:55 +1100 Subject: [PATCH 052/132] Add support for deposits to test_harness --- .../test_harness/examples/chain.yaml | 13 ++- .../test_harness/src/beacon_chain_harness.rs | 16 +++- .../beacon_chain/test_harness/src/bin.rs | 85 +++++++++++++++++-- 3 files changed, 101 insertions(+), 13 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/examples/chain.yaml b/beacon_node/beacon_chain/test_harness/examples/chain.yaml index b170a0b2e..919753afa 100644 --- a/beacon_node/beacon_chain/test_harness/examples/chain.yaml +++ b/beacon_node/beacon_chain/test_harness/examples/chain.yaml @@ -7,10 +7,19 @@ test_cases: - config: epoch_length: 64 deposits_for_chain_start: 1000 - num_slots: 32 # Testing advancing state to slot < SlotsPerEpoch + num_slots: 65 skip_slots: [2, 3] + deposits: + - slot: 1 + amount: 32 + merkle_index: 0 + - slot: 3 + amount: 32 + merkle_index: 1 + - slot: 5 + amount: 32 + merkle_index: 2 results: - slot: 32 num_validators: 1000 - config: epoch_length: 64 diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index d3bd444d1..b60454b57 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -15,10 +15,7 @@ use std::fs::File; use std::io::prelude::*; use std::iter::FromIterator; use std::sync::Arc; -use types::{ - BeaconBlock, ChainSpec, Deposit, DepositData, DepositInput, Eth1Data, FreeAttestation, Hash256, - Keypair, Slot, -}; +use types::*; /// The beacon chain harness simulates a single beacon node with `validator_count` validators connected /// to it. Each validator is provided a borrow to the beacon chain, where it may read @@ -245,6 +242,17 @@ impl BeaconChainHarness { debug!("Free attestations processed."); } + pub fn process_deposit(&mut self, deposit: Deposit, keypair: Option) { + self.beacon_chain.receive_deposit_for_inclusion(deposit); + + // If a keypair is present, add a new `ValidatorHarness` to the rig. + if let Some(keypair) = keypair { + let validator = + ValidatorHarness::new(keypair, self.beacon_chain.clone(), self.spec.clone()); + self.validators.push(validator); + } + } + pub fn run_fork_choice(&mut self) { self.beacon_chain.fork_choice().unwrap() } diff --git a/beacon_node/beacon_chain/test_harness/src/bin.rs b/beacon_node/beacon_chain/test_harness/src/bin.rs index 48f349d4a..f747be60a 100644 --- a/beacon_node/beacon_chain/test_harness/src/bin.rs +++ b/beacon_node/beacon_chain/test_harness/src/bin.rs @@ -1,10 +1,10 @@ use self::beacon_chain_harness::BeaconChainHarness; use self::validator_harness::ValidatorHarness; use beacon_chain::CheckPoint; +use bls::create_proof_of_possession; use clap::{App, Arg}; use env_logger::{Builder, Env}; use log::{info, warn}; -use std::collections::HashMap; use std::{fs::File, io::prelude::*}; use types::*; use yaml_rust::{Yaml, YamlLoader}; @@ -73,7 +73,7 @@ impl Manifest { pub fn execute(&self) -> ExecutionResult { let spec = self.spec(); let validator_count = self.config.deposits_for_chain_start; - let slots = self.results.slot; + let slots = self.config.num_slots; info!( "Building BeaconChainHarness with {} validators...", @@ -84,7 +84,18 @@ impl Manifest { info!("Starting simulation across {} slots...", slots); - for slot_height in 0..self.results.slot { + for slot_height in 0..slots { + // Include deposits + if let Some(ref deposits) = self.config.deposits { + for (slot, deposit, keypair) in deposits { + if *slot == slot_height { + info!("Including deposit at slot height {}.", slot_height); + harness.process_deposit(deposit.clone(), Some(keypair.clone())); + } + } + } + + // Build a block or skip a slot. match self.config.skip_slots { Some(ref skip_slots) if skip_slots.contains(&slot_height) => { warn!("Skipping slot at height {}.", slot_height); @@ -109,8 +120,24 @@ impl Manifest { pub fn assert_result_valid(&self, result: ExecutionResult) { info!("Verifying test results..."); + let skipped_slots = self + .config + .skip_slots + .clone() + .and_then(|slots| Some(slots.len())) + .unwrap_or_else(|| 0); + let expected_blocks = self.config.num_slots as usize + 1 - skipped_slots; + + assert_eq!(result.chain.len(), expected_blocks); + + info!( + "OK: Chain length is {} ({} skipped slots).", + result.chain.len(), + skipped_slots + ); + if let Some(ref skip_slots) = self.config.skip_slots { - for checkpoint in result.chain { + for checkpoint in &result.chain { let block_slot = checkpoint.beacon_block.slot.as_u64(); assert!( !skip_slots.contains(&block_slot), @@ -118,17 +145,30 @@ impl Manifest { block_slot ); } + info!("OK: Skipped slots not present in chain."); + } + + if let Some(ref deposits) = self.config.deposits { + let latest_state = &result.chain.last().expect("Empty chain.").beacon_state; + assert_eq!( + latest_state.validator_registry.len(), + self.config.deposits_for_chain_start + deposits.len() + ); + info!( + "OK: Validator registry has {} more validators.", + deposits.len() + ); } - info!("OK: Skipped slots not present in chain."); } } +pub type DepositTuple = (u64, Deposit, Keypair); + struct ExecutionResult { pub chain: Vec, } struct Results { - pub slot: u64, pub num_validators: Option, pub slashed_validators: Option>, pub exited_validators: Option>, @@ -137,7 +177,6 @@ struct Results { impl Results { pub fn from_yaml(yaml: &Yaml) -> Self { Self { - slot: as_u64(&yaml, "slot").expect("Must have end slot"), num_validators: as_usize(&yaml, "num_validators"), slashed_validators: as_vec_u64(&yaml, "slashed_validators"), exited_validators: as_vec_u64(&yaml, "exited_validators"), @@ -148,7 +187,9 @@ impl Results { struct Config { pub deposits_for_chain_start: usize, pub epoch_length: Option, + pub num_slots: u64, pub skip_slots: Option>, + pub deposits: Option>, } impl Config { @@ -157,11 +198,41 @@ impl Config { deposits_for_chain_start: as_usize(&yaml, "deposits_for_chain_start") .expect("Must specify validator count"), epoch_length: as_u64(&yaml, "epoch_length"), + num_slots: as_u64(&yaml, "num_slots").expect("Must specify `config.num_slots`"), skip_slots: as_vec_u64(yaml, "skip_slots"), + deposits: process_deposits(&yaml), } } } +fn process_deposits(yaml: &Yaml) -> Option> { + let mut deposits = vec![]; + + for deposit in yaml["deposits"].as_vec()? { + let keypair = Keypair::random(); + let proof_of_possession = create_proof_of_possession(&keypair); + + let slot = as_u64(deposit, "slot").expect("Incomplete deposit"); + let deposit = Deposit { + branch: vec![], + index: as_u64(deposit, "merkle_index").unwrap(), + deposit_data: DepositData { + amount: 32_000_000_000, + timestamp: 1, + deposit_input: DepositInput { + pubkey: keypair.pk.clone(), + withdrawal_credentials: Hash256::zero(), + proof_of_possession, + }, + }, + }; + + deposits.push((slot, deposit, keypair)); + } + + Some(deposits) +} + fn as_usize(yaml: &Yaml, key: &str) -> Option { yaml[key].as_i64().and_then(|n| Some(n as usize)) } From f0ea69120813638277d12117a2fbddffd2877d35 Mon Sep 17 00:00:00 2001 From: thojest Date: Fri, 1 Mar 2019 18:19:08 +0100 Subject: [PATCH 053/132] now possible to select ChainSpec by using CLI flag (lighthouse-252) --- validator_client/src/config.rs | 9 ++++++++- validator_client/src/main.rs | 27 ++++++++++++++++++++++----- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index 104a4bbe6..68405ed2f 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -1,11 +1,13 @@ use std::fs; use std::path::PathBuf; +use types::ChainSpec; /// Stores the core configuration for this validator instance. #[derive(Clone)] pub struct ClientConfig { pub data_dir: PathBuf, pub server: String, + pub spec: ChainSpec, } const DEFAULT_LIGHTHOUSE_DIR: &str = ".lighthouse-validators"; @@ -20,6 +22,11 @@ impl ClientConfig { fs::create_dir_all(&data_dir) .unwrap_or_else(|_| panic!("Unable to create {:?}", &data_dir)); let server = "localhost:50051".to_string(); - Self { data_dir, server } + let spec = ChainSpec::foundation(); + Self { + data_dir, + server, + spec, + } } } diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index c835300b5..e9a7b15a3 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -43,6 +43,15 @@ fn main() { .help("Address to connect to BeaconNode.") .takes_value(true), ) + .arg( + Arg::with_name("spec") + .long("spec") + .value_name("spec") + .short("s") + .help("Configuration of Beacon Chain") + .takes_value(true) + .possible_values(&["foundation", "few_validators"]), + ) .get_matches(); let mut config = ClientConfig::default(); @@ -62,6 +71,17 @@ fn main() { } } + // TODO: Permit loading a custom spec from file. + // Custom spec + if let Some(spec_str) = matches.value_of("spec") { + match spec_str { + "foundation" => config.spec = ChainSpec::foundation(), + "few_validators" => config.spec = ChainSpec::few_validators(), + // Should be impossible + _ => error!(log, "Invalid spec defined"; "spec" => format!("{:?}", config.spec)), + }; + } + // Log configuration info!(log, ""; "data_dir" => &config.data_dir.to_str(), @@ -81,11 +101,8 @@ fn main() { Arc::new(ValidatorServiceClient::new(ch)) }; - // Ethereum - // - // TODO: Permit loading a custom spec from file. - // https://github.com/sigp/lighthouse/issues/160 - let spec = Arc::new(ChainSpec::foundation()); + // Spec + let spec = Arc::new(config.spec.clone()); // Clock for determining the present slot. // TODO: this shouldn't be a static time, instead it should be pulled from the beacon node. From ed7a0810080475fabffdd7cd01e35edfabddb049 Mon Sep 17 00:00:00 2001 From: thojest Date: Fri, 1 Mar 2019 18:23:25 +0100 Subject: [PATCH 054/132] slightly adapted impossible error for validator_client (lighthouse-252) --- validator_client/src/main.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index e9a7b15a3..f81c3e40c 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -78,7 +78,10 @@ fn main() { "foundation" => config.spec = ChainSpec::foundation(), "few_validators" => config.spec = ChainSpec::few_validators(), // Should be impossible - _ => error!(log, "Invalid spec defined"; "spec" => format!("{:?}", config.spec)), + _ => { + error!(log, "Invalid ChainSpec defined"; "spec" => spec_str); + return; + } }; } From 1de723b2752057f7d94b517700b3fd32c3f3d67c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 2 Mar 2019 11:23:37 +1100 Subject: [PATCH 055/132] Add proposer/attester slash queues to BeaconChain Allows for storing and including AttesterSlashing and ProposerSlashing objects in blocks. --- beacon_node/beacon_chain/src/beacon_chain.rs | 92 ++++++++++++++++++-- 1 file changed, 87 insertions(+), 5 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 34e1a5183..a16fc6472 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -65,6 +65,8 @@ pub struct BeaconChain { pub slot_clock: U, pub attestation_aggregator: RwLock, pub deposits_for_inclusion: RwLock>, + pub proposer_slashings_for_inclusion: RwLock>, + pub attester_slashings_for_inclusion: RwLock>, canonical_head: RwLock, finalized_head: RwLock, pub state: RwLock, @@ -132,6 +134,8 @@ where slot_clock, attestation_aggregator, deposits_for_inclusion: RwLock::new(vec![]), + proposer_slashings_for_inclusion: RwLock::new(vec![]), + attester_slashings_for_inclusion: RwLock::new(vec![]), state: RwLock::new(genesis_state), finalized_head, canonical_head, @@ -374,7 +378,7 @@ where self.deposits_for_inclusion.read().clone() } - pub fn mark_deposits_as_included(&self, included_deposits: &[Deposit]) { + pub fn set_deposits_as_included(&self, included_deposits: &[Deposit]) { // TODO: method does not take forks into account; consider this. let mut indices_to_delete = vec![]; @@ -392,6 +396,82 @@ where } } + pub fn receive_proposer_slashing_for_inclusion(&self, proposer_slashing: ProposerSlashing) { + // TODO: proposer_slashings are not check for validity; check them. + self.proposer_slashings_for_inclusion + .write() + .push(proposer_slashing); + } + + pub fn get_proposer_slashings_for_block(&self) -> Vec { + // TODO: proposer_slashings are indiscriminately included; check them for validity. + self.proposer_slashings_for_inclusion.read().clone() + } + + pub fn set_proposer_slashings_as_included( + &self, + included_proposer_slashings: &[ProposerSlashing], + ) { + // TODO: method does not take forks into account; consider this. + let mut indices_to_delete = vec![]; + + for included in included_proposer_slashings { + for (i, for_inclusion) in self + .proposer_slashings_for_inclusion + .read() + .iter() + .enumerate() + { + if included == for_inclusion { + indices_to_delete.push(i); + } + } + } + + let proposer_slashings_for_inclusion = &mut self.proposer_slashings_for_inclusion.write(); + for i in indices_to_delete { + proposer_slashings_for_inclusion.remove(i); + } + } + + pub fn receive_attester_slashing_for_inclusion(&self, attester_slashing: AttesterSlashing) { + // TODO: attester_slashings are not check for validity; check them. + self.attester_slashings_for_inclusion + .write() + .push(attester_slashing); + } + + pub fn get_attester_slashings_for_block(&self) -> Vec { + // TODO: attester_slashings are indiscriminately included; check them for validity. + self.attester_slashings_for_inclusion.read().clone() + } + + pub fn set_attester_slashings_as_included( + &self, + included_attester_slashings: &[AttesterSlashing], + ) { + // TODO: method does not take forks into account; consider this. + let mut indices_to_delete = vec![]; + + for included in included_attester_slashings { + for (i, for_inclusion) in self + .attester_slashings_for_inclusion + .read() + .iter() + .enumerate() + { + if included == for_inclusion { + indices_to_delete.push(i); + } + } + } + + let attester_slashings_for_inclusion = &mut self.attester_slashings_for_inclusion.write(); + for i in indices_to_delete { + attester_slashings_for_inclusion.remove(i); + } + } + /// Dumps the entire canonical chain, from the head to genesis to a vector for analysis. /// /// This could be a very expensive operation and should only be done in testing/analysis @@ -518,8 +598,10 @@ where self.block_store.put(&block_root, &ssz_encode(&block)[..])?; self.state_store.put(&state_root, &ssz_encode(&state)[..])?; - // Remove any included deposits from the for-inclusion queue - self.mark_deposits_as_included(&block.body.deposits[..]); + // Update the inclusion queues so they aren't re-submitted. + self.set_deposits_as_included(&block.body.deposits[..]); + self.set_proposer_slashings_as_included(&block.body.proposer_slashings[..]); + self.set_attester_slashings_as_included(&block.body.attester_slashings[..]); // run the fork_choice add_block logic self.fork_choice @@ -574,8 +656,8 @@ where }, signature: self.spec.empty_signature.clone(), // To be completed by a validator. body: BeaconBlockBody { - proposer_slashings: vec![], - attester_slashings: vec![], + proposer_slashings: self.get_proposer_slashings_for_block(), + attester_slashings: self.get_attester_slashings_for_block(), attestations, deposits: self.get_deposits_for_block(), exits: vec![], From fd819fb7caaa24996daf8069f0d7a4e561408dd0 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 2 Mar 2019 11:24:41 +1100 Subject: [PATCH 056/132] Set BeaconChain block propose failure log to warn It think it's more suitable to a warn --- beacon_node/beacon_chain/src/beacon_chain.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index a16fc6472..625a89197 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -5,7 +5,7 @@ use db::{ ClientDB, DBError, }; use fork_choice::{ForkChoice, ForkChoiceError}; -use log::{debug, trace}; +use log::{debug, trace, warn}; use parking_lot::{RwLock, RwLockReadGuard}; use slot_clock::SlotClock; use ssz::ssz_encode; @@ -668,7 +668,7 @@ where let result = state.per_block_processing_without_verifying_block_signature(&block, &self.spec); - trace!( + warn!( "BeaconNode::produce_block: state processing result: {:?}", result ); From 7f1e40a8c640304196b0cfde5d392fb0e3aa423d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 2 Mar 2019 11:25:55 +1100 Subject: [PATCH 057/132] Add proposer slashing support to test_harness Test harness will now add signatures to a ProposerSlashing and submit it to the BeaconChain --- .../test_harness/examples/chain.yaml | 11 +++ .../test_harness/src/beacon_chain_harness.rs | 33 ++++++++- .../beacon_chain/test_harness/src/bin.rs | 69 +++++++++++++++++-- 3 files changed, 108 insertions(+), 5 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/examples/chain.yaml b/beacon_node/beacon_chain/test_harness/examples/chain.yaml index 919753afa..036871aeb 100644 --- a/beacon_node/beacon_chain/test_harness/examples/chain.yaml +++ b/beacon_node/beacon_chain/test_harness/examples/chain.yaml @@ -19,6 +19,17 @@ test_cases: - slot: 5 amount: 32 merkle_index: 2 + proposer_slashings: + - slot: 8 # At slot 8, we trigger a proposal slashing occurring + proposer_index: 42 # We penalize the validator at position 42 + proposal_1_shard: 0 + proposal_1_slot: 15 + proposal_1_root: !!binary | + LkmqmqoodLKAslkjdkajsdljasdkajlksjdasldjasdd + proposal_2_shard: 0 + proposal_2_slot: 15 + proposal_2_root: !!binary | + DIFFERENTKAslkjdkajsdljasdkajlksjdasldjasdd results: num_validators: 1000 - config: diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index b60454b57..43b60d506 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -10,6 +10,7 @@ use fork_choice::BitwiseLMDGhost; use log::debug; use rayon::prelude::*; use slot_clock::TestingSlotClock; +use ssz::TreeHash; use std::collections::HashSet; use std::fs::File; use std::io::prelude::*; @@ -242,7 +243,7 @@ impl BeaconChainHarness { debug!("Free attestations processed."); } - pub fn process_deposit(&mut self, deposit: Deposit, keypair: Option) { + pub fn add_deposit(&mut self, deposit: Deposit, keypair: Option) { self.beacon_chain.receive_deposit_for_inclusion(deposit); // If a keypair is present, add a new `ValidatorHarness` to the rig. @@ -253,6 +254,36 @@ impl BeaconChainHarness { } } + pub fn add_proposer_slashing(&mut self, mut proposer_slashing: ProposerSlashing) { + let validator = &self.validators[proposer_slashing.proposer_index as usize]; + + // This following code is a little awkward, but managing the data_1 and data_1 was getting + // rather confusing. I think this is better + let proposals = vec![ + &proposer_slashing.proposal_data_1, + &proposer_slashing.proposal_data_2, + ]; + let signatures: Vec = proposals + .iter() + .map(|proposal_data| { + let message = proposal_data.hash_tree_root(); + let epoch = proposal_data.slot.epoch(self.spec.epoch_length); + let domain = self + .beacon_chain + .state + .read() + .fork + .get_domain(epoch, self.spec.domain_proposal); + Signature::new(&message[..], domain, &validator.keypair.sk) + }) + .collect(); + proposer_slashing.proposal_signature_1 = signatures[0].clone(); + proposer_slashing.proposal_signature_2 = signatures[1].clone(); + + self.beacon_chain + .receive_proposer_slashing_for_inclusion(proposer_slashing); + } + pub fn run_fork_choice(&mut self) { self.beacon_chain.fork_choice().unwrap() } diff --git a/beacon_node/beacon_chain/test_harness/src/bin.rs b/beacon_node/beacon_chain/test_harness/src/bin.rs index f747be60a..e905e9342 100644 --- a/beacon_node/beacon_chain/test_harness/src/bin.rs +++ b/beacon_node/beacon_chain/test_harness/src/bin.rs @@ -85,12 +85,25 @@ impl Manifest { info!("Starting simulation across {} slots...", slots); for slot_height in 0..slots { - // Include deposits + // Feed deposits to the BeaconChain. if let Some(ref deposits) = self.config.deposits { for (slot, deposit, keypair) in deposits { if *slot == slot_height { info!("Including deposit at slot height {}.", slot_height); - harness.process_deposit(deposit.clone(), Some(keypair.clone())); + harness.add_deposit(deposit.clone(), Some(keypair.clone())); + } + } + } + + // Feed proposer slashings to the BeaconChain. + if let Some(ref slashings) = self.config.proposer_slashings { + for (slot, slashing) in slashings { + if *slot == slot_height { + info!( + "Including proposer slashing at slot height {}.", + slot_height + ); + harness.add_proposer_slashing(slashing.clone()); } } } @@ -163,6 +176,7 @@ impl Manifest { } pub type DepositTuple = (u64, Deposit, Keypair); +pub type ProposerSlashingTuple = (u64, ProposerSlashing); struct ExecutionResult { pub chain: Vec, @@ -190,6 +204,7 @@ struct Config { pub num_slots: u64, pub skip_slots: Option>, pub deposits: Option>, + pub proposer_slashings: Option>, } impl Config { @@ -200,12 +215,52 @@ impl Config { epoch_length: as_u64(&yaml, "epoch_length"), num_slots: as_u64(&yaml, "num_slots").expect("Must specify `config.num_slots`"), skip_slots: as_vec_u64(yaml, "skip_slots"), - deposits: process_deposits(&yaml), + deposits: parse_deposits(&yaml), + proposer_slashings: parse_proposer_slashings(&yaml), } } } -fn process_deposits(yaml: &Yaml) -> Option> { +fn parse_proposer_slashings(yaml: &Yaml) -> Option> { + let mut slashings = vec![]; + + for slashing in yaml["proposer_slashings"].as_vec()? { + let slot = as_u64(slashing, "slot").expect("Incomplete slashing"); + + let slashing = ProposerSlashing { + proposer_index: as_u64(slashing, "proposer_index") + .expect("Incomplete slashing (proposer_index)"), + proposal_data_1: ProposalSignedData { + slot: Slot::from( + as_u64(slashing, "proposal_1_slot") + .expect("Incomplete slashing (proposal_1_slot)."), + ), + shard: as_u64(slashing, "proposal_1_shard") + .expect("Incomplete slashing (proposal_1_shard)."), + block_root: as_hash256(slashing, "proposal_1_root") + .expect("Incomplete slashing (proposal_1_root)."), + }, + proposal_signature_1: Signature::empty_signature(), // Will be replaced with real signature at runtime. + proposal_data_2: ProposalSignedData { + slot: Slot::from( + as_u64(slashing, "proposal_2_slot") + .expect("Incomplete slashing (proposal_2_slot)."), + ), + shard: as_u64(slashing, "proposal_2_shard") + .expect("Incomplete slashing (proposal_2_shard)."), + block_root: as_hash256(slashing, "proposal_2_root") + .expect("Incomplete slashing (proposal_2_root)."), + }, + proposal_signature_2: Signature::empty_signature(), // Will be replaced with real signature at runtime. + }; + + slashings.push((slot, slashing)); + } + + Some(slashings) +} + +fn parse_deposits(yaml: &Yaml) -> Option> { let mut deposits = vec![]; for deposit in yaml["deposits"].as_vec()? { @@ -241,6 +296,12 @@ fn as_u64(yaml: &Yaml, key: &str) -> Option { yaml[key].as_i64().and_then(|n| Some(n as u64)) } +fn as_hash256(yaml: &Yaml, key: &str) -> Option { + yaml[key] + .as_str() + .and_then(|s| Some(Hash256::from(s.as_bytes()))) +} + fn as_vec_u64(yaml: &Yaml, key: &str) -> Option> { yaml[key].clone().into_vec().and_then(|vec| { Some( From 867dce34cdd6b7d32c805fd81dfa1cb35d7ed54d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 2 Mar 2019 11:26:39 +1100 Subject: [PATCH 058/132] Remove domain constants from block_processable The information is gather from the `spec` object, not the constants. --- eth2/state_processing/src/block_processable.rs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/eth2/state_processing/src/block_processable.rs b/eth2/state_processing/src/block_processable.rs index aab87c3ad..4083d17d8 100644 --- a/eth2/state_processing/src/block_processable.rs +++ b/eth2/state_processing/src/block_processable.rs @@ -5,12 +5,7 @@ use log::{debug, trace}; use ssz::{ssz_encode, TreeHash}; use types::*; -// TODO: define elsehwere. -const DOMAIN_PROPOSAL: u64 = 2; -const DOMAIN_EXIT: u64 = 3; -const DOMAIN_RANDAO: u64 = 4; const PHASE_0_CUSTODY_BIT: bool = false; -const DOMAIN_ATTESTATION: u64 = 1; #[derive(Debug, PartialEq)] pub enum Error { @@ -111,7 +106,7 @@ fn per_block_processing_signature_optional( &block_proposer.pubkey, &block.proposal_root(spec)[..], &block.signature, - get_domain(&state.fork, state.current_epoch(spec), DOMAIN_PROPOSAL) + get_domain(&state.fork, state.current_epoch(spec), spec.domain_proposal) ), Error::BadBlockSignature ); @@ -125,7 +120,7 @@ fn per_block_processing_signature_optional( &block_proposer.pubkey, &int_to_bytes32(state.current_epoch(spec).as_u64()), &block.randao_reveal, - get_domain(&state.fork, state.current_epoch(spec), DOMAIN_RANDAO) + get_domain(&state.fork, state.current_epoch(spec), spec.domain_randao) ), Error::BadRandaoSignature ); @@ -186,7 +181,7 @@ fn per_block_processing_signature_optional( .proposal_data_1 .slot .epoch(spec.epoch_length), - DOMAIN_PROPOSAL + spec.domain_proposal ) ), Error::BadProposerSlashing @@ -202,7 +197,7 @@ fn per_block_processing_signature_optional( .proposal_data_2 .slot .epoch(spec.epoch_length), - DOMAIN_PROPOSAL + spec.domain_proposal ) ), Error::BadProposerSlashing @@ -294,7 +289,7 @@ fn per_block_processing_signature_optional( &validator.pubkey, &exit_message, &exit.signature, - get_domain(&state.fork, exit.epoch, DOMAIN_EXIT) + get_domain(&state.fork, exit.epoch, spec.domain_exit) ), Error::BadProposerSlashing ); @@ -401,7 +396,7 @@ fn validate_attestation_signature_optional( get_domain( &state.fork, attestation.data.slot.epoch(spec.epoch_length), - DOMAIN_ATTESTATION, + spec.domain_attestation, ) ), AttestationValidationError::BadSignature From 22d59a70cc4d5224ef5a0f5b096693e2142b215f Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 2 Mar 2019 11:27:12 +1100 Subject: [PATCH 059/132] Add debug message when validator is penalized --- eth2/types/src/beacon_state.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 5591fa301..1c486e9b6 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -1004,6 +1004,10 @@ impl BeaconState { whistleblower_reward ); self.validator_registry[validator_index].penalized_epoch = current_epoch; + debug!( + "Whistleblower {} penalized validator {}.", + whistleblower_index, validator_index + ); Ok(()) } From ec5581ce1dd33931c679f6d60e38cc891bcc5439 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 2 Mar 2019 15:28:01 +1100 Subject: [PATCH 060/132] Shorten test_harness YAML to single test --- .../test_harness/examples/chain.yaml | 72 ++----------------- 1 file changed, 6 insertions(+), 66 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/examples/chain.yaml b/beacon_node/beacon_chain/test_harness/examples/chain.yaml index 036871aeb..4c4946c5d 100644 --- a/beacon_node/beacon_chain/test_harness/examples/chain.yaml +++ b/beacon_node/beacon_chain/test_harness/examples/chain.yaml @@ -20,8 +20,8 @@ test_cases: amount: 32 merkle_index: 2 proposer_slashings: - - slot: 8 # At slot 8, we trigger a proposal slashing occurring - proposer_index: 42 # We penalize the validator at position 42 + - slot: 8 # At slot 8, trigger a proposal slashing + proposer_index: 42 # Penalize the validator at position 42 proposal_1_shard: 0 proposal_1_slot: 15 proposal_1_root: !!binary | @@ -30,69 +30,9 @@ test_cases: proposal_2_slot: 15 proposal_2_root: !!binary | DIFFERENTKAslkjdkajsdljasdkajlksjdasldjasdd + attester_slashings: + # At slot 2, trigger an attester slashing for validators #11 and #12 + - slot: 2 + validator_indices: [11, 12] results: num_validators: 1000 - - config: - epoch_length: 64 - deposits_for_chain_start: 16384 - num_slots: 64 - deposits: - - slot: 1 - amount: 32 - merkle_index: 0 - pubkey: !!binary | - SlAAbShSkUg7PLiPHZI/rTS1uAvKiieOrifPN6Moso0= - - slot: 15 - amount: 32 - merkle_index: 1 - pubkey: !!binary | - Oklajsjdkaklsdlkajsdjlajslkdjlkasjlkdjlajdsd - - slot: 55 - amount: 32 - merkle_index: 2 - pubkey: !!binary | - LkmqmqoodLKAslkjdkajsdljasdkajlksjdasldjasdd - proposer_slashings: - - slot: 16 # At slot 16, we trigger a proposal slashing occurring - proposer_index: 16385 # We penalize the proposer that was just added from slot 15 - proposal_1_shard: 0 - proposal_1_slot: 15 - proposal_1_root: !!binary | - LkmqmqoodLKAslkjdkajsdljasdkajlksjdasldjasdd - proposal_2_shard: 0 - proposal_2_slot: 15 - proposal_2_root: !!binary | - LkmqmqoodLKAslkjdkajsdljasdkajlksjdasldjasdd - attester_slashings: - - slot: 59 # At slot 59, we trigger a attester slashing - slashable_vote_data_1_slot: 55 - slashable_vote_data_2_slot: 55 - slashable_vote_data_1_justified_slot: 0 - slashable_vote_data_2_justified_slot: 1 - slashable_vote_data_1_custody_0_indices: [16386] - slashable_vote_data_1_custody_1_indices: [] - slashable_vote_data_2_custody_0_indices: [] - slashable_vote_data_2_custody_1_indices: [16386] - results: - slot: 64 - num_validators: 16387 - penalized_validators: [16385, 16386] # We test that the validators at indices 16385, 16386 were indeed penalized - - config: - skip_slots: [10, 20] - epoch_length: 64 - deposits_for_chain_start: 1000 - num_slots: 128 # Testing advancing state's slot == 2*SlotsPerEpoch - deposits: - - slot: 10 - amount: 32 - merkle_index: 0 - pubkey: !!binary | - SlAAbShSkUg7PLiPHZI/rTS1uAvKiieOrifPN6Moso0= - - slot: 20 - amount: 32 - merkle_index: 1 - pubkey: !!binary | - Oklajsjdkaklsdlkajsdjlajslkdjlkasjlkdjlajdsd - results: - slot: 128 - num_validators: 1000 # Validator registry should not have grown if slots 10 and 20 were skipped From c885e36a939bcb92a1b943337ad3c2c8153c4f11 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 2 Mar 2019 15:30:50 +1100 Subject: [PATCH 061/132] Add fn to BeaconChainHarness validator signing Signs some message using the priv key of some validator --- .../test_harness/src/beacon_chain_harness.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index 43b60d506..f0fcad5cc 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -243,6 +243,25 @@ impl BeaconChainHarness { debug!("Free attestations processed."); } + pub fn validator_sign( + &self, + validator_index: usize, + message: &[u8], + epoch: Epoch, + domain_type: u64, + ) -> Option { + let validator = self.validators.get(validator_index)?; + + let domain = self + .beacon_chain + .state + .read() + .fork + .get_domain(epoch, domain_type); + + Some(Signature::new(message, domain, &validator.keypair.sk)) + } + pub fn add_deposit(&mut self, deposit: Deposit, keypair: Option) { self.beacon_chain.receive_deposit_for_inclusion(deposit); From f3a3cfcc45357cd691b9bbb663482056704ebf40 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 2 Mar 2019 15:33:52 +1100 Subject: [PATCH 062/132] Add surround/dbl vote fns to SlashableAttestation Copied from `SlashableVoteData` --- eth2/types/src/attester_slashing.rs | 2 +- eth2/types/src/slashable_attestation.rs | 23 ++++++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/eth2/types/src/attester_slashing.rs b/eth2/types/src/attester_slashing.rs index 2a1df9e0c..9cb212324 100644 --- a/eth2/types/src/attester_slashing.rs +++ b/eth2/types/src/attester_slashing.rs @@ -1,4 +1,4 @@ -use crate::{test_utils::TestRandom, SlashableAttestation}; +use crate::{test_utils::TestRandom, ChainSpec, SlashableAttestation}; use rand::RngCore; use serde_derive::Serialize; use ssz_derive::{Decode, Encode, TreeHash}; diff --git a/eth2/types/src/slashable_attestation.rs b/eth2/types/src/slashable_attestation.rs index c4a12338a..d24c5dde4 100644 --- a/eth2/types/src/slashable_attestation.rs +++ b/eth2/types/src/slashable_attestation.rs @@ -1,4 +1,4 @@ -use crate::{test_utils::TestRandom, AggregateSignature, AttestationData, Bitfield}; +use crate::{test_utils::TestRandom, AggregateSignature, AttestationData, Bitfield, ChainSpec}; use rand::RngCore; use serde_derive::Serialize; use ssz_derive::{Decode, Encode, TreeHash}; @@ -12,6 +12,27 @@ pub struct SlashableAttestation { pub aggregate_signature: AggregateSignature, } +impl SlashableAttestation { + /// Check if ``attestation_data_1`` and ``attestation_data_2`` have the same target. + /// + /// Spec v0.3.0 + pub fn is_double_vote(&self, other: &SlashableAttestation, spec: &ChainSpec) -> bool { + self.data.slot.epoch(spec.epoch_length) == other.data.slot.epoch(spec.epoch_length) + } + + /// Check if ``attestation_data_1`` surrounds ``attestation_data_2``. + /// + /// Spec v0.3.0 + pub fn is_surround_vote(&self, other: &SlashableAttestation, spec: &ChainSpec) -> bool { + let source_epoch_1 = self.data.justified_epoch; + let source_epoch_2 = other.data.justified_epoch; + let target_epoch_1 = self.data.slot.epoch(spec.epoch_length); + let target_epoch_2 = other.data.slot.epoch(spec.epoch_length); + + (source_epoch_1 < source_epoch_2) && (target_epoch_2 < target_epoch_1) + } +} + #[cfg(test)] mod tests { use super::*; From ff2783a1cb6773e8c392b555973e96b911786593 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 2 Mar 2019 15:35:02 +1100 Subject: [PATCH 063/132] Add AttesterSlashing to test_harness - Adds methods to BeaconChainHarness - Adds YAML parsing --- .../test_harness/src/beacon_chain_harness.rs | 5 + .../beacon_chain/test_harness/src/bin.rs | 115 ++++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index f0fcad5cc..862e037e2 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -303,6 +303,11 @@ impl BeaconChainHarness { .receive_proposer_slashing_for_inclusion(proposer_slashing); } + pub fn add_attester_slashing(&mut self, attester_slashing: AttesterSlashing) { + self.beacon_chain + .receive_attester_slashing_for_inclusion(attester_slashing); + } + pub fn run_fork_choice(&mut self) { self.beacon_chain.fork_choice().unwrap() } diff --git a/beacon_node/beacon_chain/test_harness/src/bin.rs b/beacon_node/beacon_chain/test_harness/src/bin.rs index e905e9342..216e03d9f 100644 --- a/beacon_node/beacon_chain/test_harness/src/bin.rs +++ b/beacon_node/beacon_chain/test_harness/src/bin.rs @@ -5,6 +5,7 @@ use bls::create_proof_of_possession; use clap::{App, Arg}; use env_logger::{Builder, Env}; use log::{info, warn}; +use ssz::TreeHash; use std::{fs::File, io::prelude::*}; use types::*; use yaml_rust::{Yaml, YamlLoader}; @@ -108,6 +109,21 @@ impl Manifest { } } + // Feed attester slashings to the BeaconChain. + if let Some(ref slashings) = self.config.attester_slashings { + for (slot, validator_indices) in slashings { + if *slot == slot_height { + info!( + "Including attester slashing at slot height {}.", + slot_height + ); + let slashing = + build_double_vote_attester_slashing(&harness, &validator_indices[..]); + harness.add_attester_slashing(slashing); + } + } + } + // Build a block or skip a slot. match self.config.skip_slots { Some(ref skip_slots) if skip_slots.contains(&slot_height) => { @@ -175,8 +191,87 @@ impl Manifest { } } +fn build_double_vote_attester_slashing( + harness: &BeaconChainHarness, + validator_indices: &[u64], +) -> AttesterSlashing { + let double_voted_slot = Slot::new(0); + let shard = 0; + let justified_epoch = Epoch::new(0); + let epoch = Epoch::new(0); + let hash_1 = Hash256::from("1".as_bytes()); + let hash_2 = Hash256::from("2".as_bytes()); + + let mut slashable_attestation_1 = SlashableAttestation { + validator_indices: validator_indices.to_vec(), + data: AttestationData { + slot: double_voted_slot, + shard, + beacon_block_root: hash_1, + epoch_boundary_root: hash_1, + shard_block_root: hash_1, + latest_crosslink: Crosslink { + epoch, + shard_block_root: hash_1, + }, + justified_epoch, + justified_block_root: hash_1, + }, + custody_bitfield: Bitfield::new(), + aggregate_signature: AggregateSignature::new(), + }; + + let mut slashable_attestation_2 = SlashableAttestation { + validator_indices: validator_indices.to_vec(), + data: AttestationData { + slot: double_voted_slot, + shard, + beacon_block_root: hash_2, + epoch_boundary_root: hash_2, + shard_block_root: hash_2, + latest_crosslink: Crosslink { + epoch, + shard_block_root: hash_2, + }, + justified_epoch, + justified_block_root: hash_2, + }, + custody_bitfield: Bitfield::new(), + aggregate_signature: AggregateSignature::new(), + }; + + let add_signatures = |attestation: &mut SlashableAttestation| { + for (i, validator_index) in validator_indices.iter().enumerate() { + let attestation_data_and_custody_bit = AttestationDataAndCustodyBit { + data: attestation.data.clone(), + custody_bit: attestation.custody_bitfield.get(i).unwrap(), + }; + let message = attestation_data_and_custody_bit.hash_tree_root(); + let signature = harness + .validator_sign( + *validator_index as usize, + &message[..], + epoch, + harness.spec.domain_attestation, + ) + .expect("Unable to sign attestation with unknown validator index."); + attestation.aggregate_signature.add(&signature); + } + }; + + add_signatures(&mut slashable_attestation_1); + add_signatures(&mut slashable_attestation_2); + + AttesterSlashing { + slashable_attestation_1, + slashable_attestation_2, + } +} + pub type DepositTuple = (u64, Deposit, Keypair); pub type ProposerSlashingTuple = (u64, ProposerSlashing); +// (slot, validator_indices) +pub type AttesterSlashingTuple = (u64, Vec); struct ExecutionResult { pub chain: Vec, @@ -205,6 +300,7 @@ struct Config { pub skip_slots: Option>, pub deposits: Option>, pub proposer_slashings: Option>, + pub attester_slashings: Option>, } impl Config { @@ -217,16 +313,35 @@ impl Config { skip_slots: as_vec_u64(yaml, "skip_slots"), deposits: parse_deposits(&yaml), proposer_slashings: parse_proposer_slashings(&yaml), + attester_slashings: parse_attester_slashings(&yaml), } } } +fn parse_attester_slashings(yaml: &Yaml) -> Option> { + let mut slashings = vec![]; + + for slashing in yaml["attester_slashings"].as_vec()? { + let slot = as_u64(slashing, "slot").expect("Incomplete attester_slashing (slot)"); + let validator_indices = as_vec_u64(slashing, "validator_indices") + .expect("Incomplete attester_slashing (validator_indices)"); + + slashings.push((slot, validator_indices)); + } + + Some(slashings) +} + fn parse_proposer_slashings(yaml: &Yaml) -> Option> { let mut slashings = vec![]; for slashing in yaml["proposer_slashings"].as_vec()? { let slot = as_u64(slashing, "slot").expect("Incomplete slashing"); + // Builds a ProposerSlashing object from YAML fields. + // + // Rustfmt make this look rather ugly, however it is just a simple struct + // instantiation. let slashing = ProposerSlashing { proposer_index: as_u64(slashing, "proposer_index") .expect("Incomplete slashing (proposer_index)"), From 6795aa42b233f10fae6d00b5e339fef509f5cee3 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Sat, 2 Mar 2019 15:52:33 +1100 Subject: [PATCH 064/132] Fix log_int implementation, removing floats The cast from f32::MAX to u32 was undefined behaviour, and the use of floating point logarithms would yield incorrect results due to rounding and truncation, e.g. for the integer 16777206 --- eth2/fork_choice/Cargo.toml | 1 - eth2/fork_choice/src/bitwise_lmd_ghost.rs | 29 ++++++++++++----------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/eth2/fork_choice/Cargo.toml b/eth2/fork_choice/Cargo.toml index 210f3c235..819b84055 100644 --- a/eth2/fork_choice/Cargo.toml +++ b/eth2/fork_choice/Cargo.toml @@ -8,7 +8,6 @@ edition = "2018" db = { path = "../../beacon_node/db" } ssz = { path = "../utils/ssz" } types = { path = "../types" } -fast-math = "0.1.1" log = "0.4.6" bit-vec = "0.5.0" diff --git a/eth2/fork_choice/src/bitwise_lmd_ghost.rs b/eth2/fork_choice/src/bitwise_lmd_ghost.rs index 1e66de079..58cd2b5a3 100644 --- a/eth2/fork_choice/src/bitwise_lmd_ghost.rs +++ b/eth2/fork_choice/src/bitwise_lmd_ghost.rs @@ -1,5 +1,5 @@ +//! The optimised bitwise LMD-GHOST fork choice rule. extern crate bit_vec; -extern crate fast_math; use crate::{ForkChoice, ForkChoiceError}; use bit_vec::BitVec; @@ -7,7 +7,6 @@ use db::{ stores::{BeaconBlockStore, BeaconStateStore}, ClientDB, }; -use fast_math::log2_raw; use log::{debug, trace}; use std::collections::HashMap; use std::sync::Arc; @@ -19,22 +18,17 @@ use types::{ //TODO: Pruning - Children //TODO: Handle Syncing -/// The optimised bitwise LMD-GHOST fork choice rule. -/// NOTE: This uses u32 to represent difference between block heights. Thus this is only -/// applicable for block height differences in the range of a u32. -/// This can potentially be parallelized in some parts. -// we use fast log2, a log2 lookup table is implemented in Vitaliks code, potentially do -// the comparison. Log2_raw takes 2ns according to the documentation. +// NOTE: This uses u32 to represent difference between block heights. Thus this is only +// applicable for block height differences in the range of a u32. +// This can potentially be parallelized in some parts. + +/// Compute the base-2 logarithm of an integer, floored (rounded down) #[inline] fn log2_int(x: u32) -> u32 { if x == 0 { return 0; } - assert!( - x <= std::f32::MAX as u32, - "Height too large for fast log in bitwise fork choice" - ); - log2_raw(x as f32) as u32 + 31 - x.leading_zeros() } fn power_of_2_below(x: u32) -> u32 { @@ -469,7 +463,6 @@ mod tests { #[test] pub fn test_power_of_2_below() { - println!("{:?}", std::f32::MAX); assert_eq!(power_of_2_below(4), 4); assert_eq!(power_of_2_below(5), 4); assert_eq!(power_of_2_below(7), 4); @@ -478,4 +471,12 @@ mod tests { assert_eq!(power_of_2_below(33), 32); assert_eq!(power_of_2_below(63), 32); } + + #[test] + pub fn test_power_of_2_below_large() { + let pow: u32 = 1 << 24; + for x in (pow - 20)..(pow + 20) { + assert!(power_of_2_below(x) <= x, "{}", x); + } + } } From bb4d392a98029664f04e114df207f48c6b8069f9 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 2 Mar 2019 16:05:25 +1100 Subject: [PATCH 065/132] Add AttestationSlashingBuilder --- .../beacon_chain/test_harness/src/bin.rs | 76 ++--------------- eth2/types/src/attester_slashing.rs | 6 +- eth2/types/src/attester_slashing/builder.rs | 85 +++++++++++++++++++ 3 files changed, 96 insertions(+), 71 deletions(-) create mode 100644 eth2/types/src/attester_slashing/builder.rs diff --git a/beacon_node/beacon_chain/test_harness/src/bin.rs b/beacon_node/beacon_chain/test_harness/src/bin.rs index 216e03d9f..fa5aa45a2 100644 --- a/beacon_node/beacon_chain/test_harness/src/bin.rs +++ b/beacon_node/beacon_chain/test_harness/src/bin.rs @@ -7,6 +7,7 @@ use env_logger::{Builder, Env}; use log::{info, warn}; use ssz::TreeHash; use std::{fs::File, io::prelude::*}; +use types::attester_slashing::AttesterSlashingBuilder; use types::*; use yaml_rust::{Yaml, YamlLoader}; @@ -195,82 +196,17 @@ fn build_double_vote_attester_slashing( harness: &BeaconChainHarness, validator_indices: &[u64], ) -> AttesterSlashing { - let double_voted_slot = Slot::new(0); - let shard = 0; - let justified_epoch = Epoch::new(0); - let epoch = Epoch::new(0); - let hash_1 = Hash256::from("1".as_bytes()); - let hash_2 = Hash256::from("2".as_bytes()); - - let mut slashable_attestation_1 = SlashableAttestation { - validator_indices: validator_indices.to_vec(), - data: AttestationData { - slot: double_voted_slot, - shard, - beacon_block_root: hash_1, - epoch_boundary_root: hash_1, - shard_block_root: hash_1, - latest_crosslink: Crosslink { - epoch, - shard_block_root: hash_1, - }, - justified_epoch, - justified_block_root: hash_1, - }, - custody_bitfield: Bitfield::new(), - aggregate_signature: AggregateSignature::new(), + let signer = |validator_index: u64, message: &[u8], epoch: Epoch, domain: u64| { + harness + .validator_sign(validator_index as usize, message, epoch, domain) + .expect("Unable to sign AttesterSlashing") }; - let mut slashable_attestation_2 = SlashableAttestation { - validator_indices: validator_indices.to_vec(), - data: AttestationData { - slot: double_voted_slot, - shard, - beacon_block_root: hash_2, - epoch_boundary_root: hash_2, - shard_block_root: hash_2, - latest_crosslink: Crosslink { - epoch, - shard_block_root: hash_2, - }, - justified_epoch, - justified_block_root: hash_2, - }, - custody_bitfield: Bitfield::new(), - aggregate_signature: AggregateSignature::new(), - }; - - let add_signatures = |attestation: &mut SlashableAttestation| { - for (i, validator_index) in validator_indices.iter().enumerate() { - let attestation_data_and_custody_bit = AttestationDataAndCustodyBit { - data: attestation.data.clone(), - custody_bit: attestation.custody_bitfield.get(i).unwrap(), - }; - let message = attestation_data_and_custody_bit.hash_tree_root(); - let signature = harness - .validator_sign( - *validator_index as usize, - &message[..], - epoch, - harness.spec.domain_attestation, - ) - .expect("Unable to sign attestation with unknown validator index."); - attestation.aggregate_signature.add(&signature); - } - }; - - add_signatures(&mut slashable_attestation_1); - add_signatures(&mut slashable_attestation_2); - - AttesterSlashing { - slashable_attestation_1, - slashable_attestation_2, - } + AttesterSlashingBuilder::double_vote(validator_indices, signer, &harness.spec) } pub type DepositTuple = (u64, Deposit, Keypair); pub type ProposerSlashingTuple = (u64, ProposerSlashing); -// (slot, validator_indices) pub type AttesterSlashingTuple = (u64, Vec); struct ExecutionResult { diff --git a/eth2/types/src/attester_slashing.rs b/eth2/types/src/attester_slashing.rs index 9cb212324..ac75a2562 100644 --- a/eth2/types/src/attester_slashing.rs +++ b/eth2/types/src/attester_slashing.rs @@ -1,9 +1,13 @@ -use crate::{test_utils::TestRandom, ChainSpec, SlashableAttestation}; +use crate::{test_utils::TestRandom, SlashableAttestation}; use rand::RngCore; use serde_derive::Serialize; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; +mod builder; + +pub use builder::AttesterSlashingBuilder; + #[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] pub struct AttesterSlashing { pub slashable_attestation_1: SlashableAttestation, diff --git a/eth2/types/src/attester_slashing/builder.rs b/eth2/types/src/attester_slashing/builder.rs new file mode 100644 index 000000000..d382cb64c --- /dev/null +++ b/eth2/types/src/attester_slashing/builder.rs @@ -0,0 +1,85 @@ +use crate::*; +use ssz::TreeHash; + +pub struct AttesterSlashingBuilder(); + +impl AttesterSlashingBuilder { + pub fn double_vote( + validator_indices: &[u64], + signer: F, + spec: &ChainSpec, + ) -> AttesterSlashing + where + F: Fn(u64, &[u8], Epoch, u64) -> Signature, + { + let double_voted_slot = Slot::new(0); + let shard = 0; + let justified_epoch = Epoch::new(0); + let epoch = Epoch::new(0); + let hash_1 = Hash256::from("1".as_bytes()); + let hash_2 = Hash256::from("2".as_bytes()); + + let mut slashable_attestation_1 = SlashableAttestation { + validator_indices: validator_indices.to_vec(), + data: AttestationData { + slot: double_voted_slot, + shard, + beacon_block_root: hash_1, + epoch_boundary_root: hash_1, + shard_block_root: hash_1, + latest_crosslink: Crosslink { + epoch, + shard_block_root: hash_1, + }, + justified_epoch, + justified_block_root: hash_1, + }, + custody_bitfield: Bitfield::new(), + aggregate_signature: AggregateSignature::new(), + }; + + let mut slashable_attestation_2 = SlashableAttestation { + validator_indices: validator_indices.to_vec(), + data: AttestationData { + slot: double_voted_slot, + shard, + beacon_block_root: hash_2, + epoch_boundary_root: hash_2, + shard_block_root: hash_2, + latest_crosslink: Crosslink { + epoch, + shard_block_root: hash_2, + }, + justified_epoch, + justified_block_root: hash_2, + }, + custody_bitfield: Bitfield::new(), + aggregate_signature: AggregateSignature::new(), + }; + + let add_signatures = |attestation: &mut SlashableAttestation| { + for (i, validator_index) in validator_indices.iter().enumerate() { + let attestation_data_and_custody_bit = AttestationDataAndCustodyBit { + data: attestation.data.clone(), + custody_bit: attestation.custody_bitfield.get(i).unwrap(), + }; + let message = attestation_data_and_custody_bit.hash_tree_root(); + let signature = signer( + *validator_index, + &message[..], + epoch, + spec.domain_attestation, + ); + attestation.aggregate_signature.add(&signature); + } + }; + + add_signatures(&mut slashable_attestation_1); + add_signatures(&mut slashable_attestation_2); + + AttesterSlashing { + slashable_attestation_1, + slashable_attestation_2, + } + } +} From 8e1380d7c4ad9afe10db13735ce52c2d6ab380cd Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 2 Mar 2019 18:36:44 +1100 Subject: [PATCH 066/132] Add ProposerSlashingBuilder It is capable of producing double votes --- eth2/types/src/proposer_slashing.rs | 4 ++ eth2/types/src/proposer_slashing/builder.rs | 48 +++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 eth2/types/src/proposer_slashing/builder.rs diff --git a/eth2/types/src/proposer_slashing.rs b/eth2/types/src/proposer_slashing.rs index 610017c0c..ea30d46ec 100644 --- a/eth2/types/src/proposer_slashing.rs +++ b/eth2/types/src/proposer_slashing.rs @@ -6,6 +6,10 @@ use serde_derive::Serialize; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; +mod builder; + +pub use builder::ProposerSlashingBuilder; + #[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] pub struct ProposerSlashing { pub proposer_index: u64, diff --git a/eth2/types/src/proposer_slashing/builder.rs b/eth2/types/src/proposer_slashing/builder.rs new file mode 100644 index 000000000..a43a73ff0 --- /dev/null +++ b/eth2/types/src/proposer_slashing/builder.rs @@ -0,0 +1,48 @@ +use crate::*; +use ssz::TreeHash; + +pub struct ProposerSlashingBuilder(); + +impl ProposerSlashingBuilder { + pub fn double_vote(proposer_index: u64, signer: F, spec: &ChainSpec) -> ProposerSlashing + where + F: Fn(u64, &[u8], Epoch, u64) -> Signature, + { + let slot = Slot::new(0); + let shard = 0; + + let proposal_data_1 = ProposalSignedData { + slot, + shard, + block_root: Hash256::from("one".as_bytes()), + }; + + let proposal_data_2 = ProposalSignedData { + slot, + shard, + block_root: Hash256::from("two".as_bytes()), + }; + + let proposal_signature_1 = { + let message = proposal_data_1.hash_tree_root(); + let epoch = slot.epoch(spec.epoch_length); + let domain = spec.domain_proposal; + signer(proposer_index, &message[..], epoch, domain) + }; + + let proposal_signature_2 = { + let message = proposal_data_2.hash_tree_root(); + let epoch = slot.epoch(spec.epoch_length); + let domain = spec.domain_proposal; + signer(proposer_index, &message[..], epoch, domain) + }; + + ProposerSlashing { + proposer_index, + proposal_data_1, + proposal_signature_1, + proposal_data_2, + proposal_signature_2, + } + } +} From e59404f463bc0e8325decff349000cc086c969ed Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 2 Mar 2019 18:37:21 +1100 Subject: [PATCH 067/132] Change test_harness proposer_slashings YAML Removes a lot of the detail from the `proposer_slashings` field -- IMO this is not necessary in the test spec, the details of how a proposer_slashing is created should be held in the program and the spec should only define that one happens. --- .../test_harness/examples/chain.yaml | 21 +++--- .../test_harness/src/beacon_chain_harness.rs | 27 +------- .../beacon_chain/test_harness/src/bin.rs | 66 +++++++------------ 3 files changed, 36 insertions(+), 78 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/examples/chain.yaml b/beacon_node/beacon_chain/test_harness/examples/chain.yaml index 4c4946c5d..9e00f4569 100644 --- a/beacon_node/beacon_chain/test_harness/examples/chain.yaml +++ b/beacon_node/beacon_chain/test_harness/examples/chain.yaml @@ -20,19 +20,18 @@ test_cases: amount: 32 merkle_index: 2 proposer_slashings: - - slot: 8 # At slot 8, trigger a proposal slashing - proposer_index: 42 # Penalize the validator at position 42 - proposal_1_shard: 0 - proposal_1_slot: 15 - proposal_1_root: !!binary | - LkmqmqoodLKAslkjdkajsdljasdkajlksjdasldjasdd - proposal_2_shard: 0 - proposal_2_slot: 15 - proposal_2_root: !!binary | - DIFFERENTKAslkjdkajsdljasdkajlksjdasldjasdd + # At slot 2, trigger a proposer slashing for validator #42. + - slot: 2 + validator_index: 42 + # At slot 8, trigger a proposer slashing for validator #13. + - slot: 8 + validator_index: 13 attester_slashings: - # At slot 2, trigger an attester slashing for validators #11 and #12 + # At slot 2, trigger an attester slashing for validators #11 and #12. - slot: 2 validator_indices: [11, 12] + # At slot 5, trigger an attester slashing for validator #14. + - slot: 5 + validator_indices: [14] results: num_validators: 1000 diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index 862e037e2..9060c4ec1 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -273,32 +273,7 @@ impl BeaconChainHarness { } } - pub fn add_proposer_slashing(&mut self, mut proposer_slashing: ProposerSlashing) { - let validator = &self.validators[proposer_slashing.proposer_index as usize]; - - // This following code is a little awkward, but managing the data_1 and data_1 was getting - // rather confusing. I think this is better - let proposals = vec![ - &proposer_slashing.proposal_data_1, - &proposer_slashing.proposal_data_2, - ]; - let signatures: Vec = proposals - .iter() - .map(|proposal_data| { - let message = proposal_data.hash_tree_root(); - let epoch = proposal_data.slot.epoch(self.spec.epoch_length); - let domain = self - .beacon_chain - .state - .read() - .fork - .get_domain(epoch, self.spec.domain_proposal); - Signature::new(&message[..], domain, &validator.keypair.sk) - }) - .collect(); - proposer_slashing.proposal_signature_1 = signatures[0].clone(); - proposer_slashing.proposal_signature_2 = signatures[1].clone(); - + pub fn add_proposer_slashing(&mut self, proposer_slashing: ProposerSlashing) { self.beacon_chain .receive_proposer_slashing_for_inclusion(proposer_slashing); } diff --git a/beacon_node/beacon_chain/test_harness/src/bin.rs b/beacon_node/beacon_chain/test_harness/src/bin.rs index fa5aa45a2..7c525c768 100644 --- a/beacon_node/beacon_chain/test_harness/src/bin.rs +++ b/beacon_node/beacon_chain/test_harness/src/bin.rs @@ -7,8 +7,10 @@ use env_logger::{Builder, Env}; use log::{info, warn}; use ssz::TreeHash; use std::{fs::File, io::prelude::*}; -use types::attester_slashing::AttesterSlashingBuilder; use types::*; +use types::{ + attester_slashing::AttesterSlashingBuilder, proposer_slashing::ProposerSlashingBuilder, +}; use yaml_rust::{Yaml, YamlLoader}; mod beacon_chain_harness; @@ -99,13 +101,14 @@ impl Manifest { // Feed proposer slashings to the BeaconChain. if let Some(ref slashings) = self.config.proposer_slashings { - for (slot, slashing) in slashings { + for (slot, validator_index) in slashings { if *slot == slot_height { info!( - "Including proposer slashing at slot height {}.", - slot_height + "Including proposer slashing at slot height {} for validator #{}.", + slot_height, validator_index ); - harness.add_proposer_slashing(slashing.clone()); + let slashing = build_proposer_slashing(&harness, *validator_index); + harness.add_proposer_slashing(slashing); } } } @@ -115,8 +118,8 @@ impl Manifest { for (slot, validator_indices) in slashings { if *slot == slot_height { info!( - "Including attester slashing at slot height {}.", - slot_height + "Including attester slashing at slot height {} for validators {:?}.", + slot_height, validator_indices ); let slashing = build_double_vote_attester_slashing(&harness, &validator_indices[..]); @@ -205,8 +208,18 @@ fn build_double_vote_attester_slashing( AttesterSlashingBuilder::double_vote(validator_indices, signer, &harness.spec) } +fn build_proposer_slashing(harness: &BeaconChainHarness, validator_index: u64) -> ProposerSlashing { + let signer = |validator_index: u64, message: &[u8], epoch: Epoch, domain: u64| { + harness + .validator_sign(validator_index as usize, message, epoch, domain) + .expect("Unable to sign AttesterSlashing") + }; + + ProposerSlashingBuilder::double_vote(validator_index, signer, &harness.spec) +} + pub type DepositTuple = (u64, Deposit, Keypair); -pub type ProposerSlashingTuple = (u64, ProposerSlashing); +pub type ProposerSlashingTuple = (u64, u64); pub type AttesterSlashingTuple = (u64, Vec); struct ExecutionResult { @@ -272,40 +285,11 @@ fn parse_proposer_slashings(yaml: &Yaml) -> Option> { let mut slashings = vec![]; for slashing in yaml["proposer_slashings"].as_vec()? { - let slot = as_u64(slashing, "slot").expect("Incomplete slashing"); + let slot = as_u64(slashing, "slot").expect("Incomplete proposer slashing (slot)_"); + let validator_index = as_u64(slashing, "validator_index") + .expect("Incomplete proposer slashing (validator_index)"); - // Builds a ProposerSlashing object from YAML fields. - // - // Rustfmt make this look rather ugly, however it is just a simple struct - // instantiation. - let slashing = ProposerSlashing { - proposer_index: as_u64(slashing, "proposer_index") - .expect("Incomplete slashing (proposer_index)"), - proposal_data_1: ProposalSignedData { - slot: Slot::from( - as_u64(slashing, "proposal_1_slot") - .expect("Incomplete slashing (proposal_1_slot)."), - ), - shard: as_u64(slashing, "proposal_1_shard") - .expect("Incomplete slashing (proposal_1_shard)."), - block_root: as_hash256(slashing, "proposal_1_root") - .expect("Incomplete slashing (proposal_1_root)."), - }, - proposal_signature_1: Signature::empty_signature(), // Will be replaced with real signature at runtime. - proposal_data_2: ProposalSignedData { - slot: Slot::from( - as_u64(slashing, "proposal_2_slot") - .expect("Incomplete slashing (proposal_2_slot)."), - ), - shard: as_u64(slashing, "proposal_2_shard") - .expect("Incomplete slashing (proposal_2_shard)."), - block_root: as_hash256(slashing, "proposal_2_root") - .expect("Incomplete slashing (proposal_2_root)."), - }, - proposal_signature_2: Signature::empty_signature(), // Will be replaced with real signature at runtime. - }; - - slashings.push((slot, slashing)); + slashings.push((slot, validator_index)); } Some(slashings) From c975d49ead55a9535dc15e778161d0f8b690869f Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 2 Mar 2019 18:39:52 +1100 Subject: [PATCH 068/132] Copy SlashableVote.. tests to SlashableAttestation SlashableVoteData tests were just copied directly across --- eth2/types/src/slashable_attestation.rs | 87 +++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/eth2/types/src/slashable_attestation.rs b/eth2/types/src/slashable_attestation.rs index d24c5dde4..8ad582ce6 100644 --- a/eth2/types/src/slashable_attestation.rs +++ b/eth2/types/src/slashable_attestation.rs @@ -36,9 +36,83 @@ impl SlashableAttestation { #[cfg(test)] mod tests { use super::*; + use crate::chain_spec::ChainSpec; + use crate::slot_epoch::{Epoch, Slot}; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; use ssz::{ssz_encode, Decodable, TreeHash}; + #[test] + pub fn test_is_double_vote_true() { + let spec = ChainSpec::foundation(); + let slashable_vote_first = create_slashable_attestation(1, 1, &spec); + let slashable_vote_second = create_slashable_attestation(1, 1, &spec); + + assert_eq!( + slashable_vote_first.is_double_vote(&slashable_vote_second, &spec), + true + ) + } + + #[test] + pub fn test_is_double_vote_false() { + let spec = ChainSpec::foundation(); + let slashable_vote_first = create_slashable_attestation(1, 1, &spec); + let slashable_vote_second = create_slashable_attestation(2, 1, &spec); + + assert_eq!( + slashable_vote_first.is_double_vote(&slashable_vote_second, &spec), + false + ); + } + + #[test] + pub fn test_is_surround_vote_true() { + let spec = ChainSpec::foundation(); + let slashable_vote_first = create_slashable_attestation(2, 1, &spec); + let slashable_vote_second = create_slashable_attestation(1, 2, &spec); + + assert_eq!( + slashable_vote_first.is_surround_vote(&slashable_vote_second, &spec), + true + ); + } + + #[test] + pub fn test_is_surround_vote_true_realistic() { + let spec = ChainSpec::foundation(); + let slashable_vote_first = create_slashable_attestation(4, 1, &spec); + let slashable_vote_second = create_slashable_attestation(3, 2, &spec); + + assert_eq!( + slashable_vote_first.is_surround_vote(&slashable_vote_second, &spec), + true + ); + } + + #[test] + pub fn test_is_surround_vote_false_source_epoch_fails() { + let spec = ChainSpec::foundation(); + let slashable_vote_first = create_slashable_attestation(2, 2, &spec); + let slashable_vote_second = create_slashable_attestation(1, 1, &spec); + + assert_eq!( + slashable_vote_first.is_surround_vote(&slashable_vote_second, &spec), + false + ); + } + + #[test] + pub fn test_is_surround_vote_false_target_epoch_fails() { + let spec = ChainSpec::foundation(); + let slashable_vote_first = create_slashable_attestation(1, 1, &spec); + let slashable_vote_second = create_slashable_attestation(2, 2, &spec); + + assert_eq!( + slashable_vote_first.is_surround_vote(&slashable_vote_second, &spec), + false + ); + } + #[test] pub fn test_ssz_round_trip() { let mut rng = XorShiftRng::from_seed([42; 16]); @@ -61,4 +135,17 @@ mod tests { // TODO: Add further tests // https://github.com/sigp/lighthouse/issues/170 } + + fn create_slashable_attestation( + slot_factor: u64, + justified_epoch: u64, + spec: &ChainSpec, + ) -> SlashableAttestation { + let mut rng = XorShiftRng::from_seed([42; 16]); + let mut slashable_vote = SlashableAttestation::random_for_test(&mut rng); + + slashable_vote.data.slot = Slot::new(slot_factor * spec.epoch_length); + slashable_vote.data.justified_epoch = Epoch::new(justified_epoch); + slashable_vote + } } From db28cc1b9270895580c651869dfe2f4a8d5d7ce6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 2 Mar 2019 18:43:27 +1100 Subject: [PATCH 069/132] Fix warnings in test_harness/src/bin.rs --- beacon_node/beacon_chain/test_harness/src/bin.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/src/bin.rs b/beacon_node/beacon_chain/test_harness/src/bin.rs index 7c525c768..182021440 100644 --- a/beacon_node/beacon_chain/test_harness/src/bin.rs +++ b/beacon_node/beacon_chain/test_harness/src/bin.rs @@ -5,7 +5,6 @@ use bls::create_proof_of_possession; use clap::{App, Arg}; use env_logger::{Builder, Env}; use log::{info, warn}; -use ssz::TreeHash; use std::{fs::File, io::prelude::*}; use types::*; use types::{ @@ -331,12 +330,6 @@ fn as_u64(yaml: &Yaml, key: &str) -> Option { yaml[key].as_i64().and_then(|n| Some(n as u64)) } -fn as_hash256(yaml: &Yaml, key: &str) -> Option { - yaml[key] - .as_str() - .and_then(|s| Some(Hash256::from(s.as_bytes()))) -} - fn as_vec_u64(yaml: &Yaml, key: &str) -> Option> { yaml[key].clone().into_vec().and_then(|vec| { Some( From f5614381e122900d21ee4bfae3f3b69a9736a998 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 2 Mar 2019 18:59:47 +1100 Subject: [PATCH 070/132] Re-organise test_harness binary Moves manifest and components into separate files. --- .../beacon_chain/test_harness/src/bin.rs | 305 +----------------- .../test_harness/src/manifest/config.rs | 89 +++++ .../test_harness/src/manifest/mod.rs | 185 +++++++++++ .../test_harness/src/manifest/results.rs | 18 ++ .../test_harness/src/manifest/yaml_helpers.rs | 19 ++ 5 files changed, 316 insertions(+), 300 deletions(-) create mode 100644 beacon_node/beacon_chain/test_harness/src/manifest/config.rs create mode 100644 beacon_node/beacon_chain/test_harness/src/manifest/mod.rs create mode 100644 beacon_node/beacon_chain/test_harness/src/manifest/results.rs create mode 100644 beacon_node/beacon_chain/test_harness/src/manifest/yaml_helpers.rs diff --git a/beacon_node/beacon_chain/test_harness/src/bin.rs b/beacon_node/beacon_chain/test_harness/src/bin.rs index 182021440..b6a0529c7 100644 --- a/beacon_node/beacon_chain/test_harness/src/bin.rs +++ b/beacon_node/beacon_chain/test_harness/src/bin.rs @@ -1,20 +1,15 @@ -use self::beacon_chain_harness::BeaconChainHarness; -use self::validator_harness::ValidatorHarness; -use beacon_chain::CheckPoint; -use bls::create_proof_of_possession; use clap::{App, Arg}; use env_logger::{Builder, Env}; -use log::{info, warn}; +use manifest::Manifest; use std::{fs::File, io::prelude::*}; -use types::*; -use types::{ - attester_slashing::AttesterSlashingBuilder, proposer_slashing::ProposerSlashingBuilder, -}; -use yaml_rust::{Yaml, YamlLoader}; +use yaml_rust::YamlLoader; mod beacon_chain_harness; +mod manifest; mod validator_harness; +use validator_harness::ValidatorHarness; + fn main() { let matches = App::new("Lighthouse Test Harness Runner") .version("0.0.1") @@ -49,293 +44,3 @@ fn main() { } } } - -struct Manifest { - pub results: Results, - pub config: Config, -} - -impl Manifest { - pub fn from_yaml(test_case: &Yaml) -> Self { - Self { - results: Results::from_yaml(&test_case["results"]), - config: Config::from_yaml(&test_case["config"]), - } - } - - fn spec(&self) -> ChainSpec { - let mut spec = ChainSpec::foundation(); - - if let Some(n) = self.config.epoch_length { - spec.epoch_length = n; - } - - spec - } - - pub fn execute(&self) -> ExecutionResult { - let spec = self.spec(); - let validator_count = self.config.deposits_for_chain_start; - let slots = self.config.num_slots; - - info!( - "Building BeaconChainHarness with {} validators...", - validator_count - ); - - let mut harness = BeaconChainHarness::new(spec, validator_count); - - info!("Starting simulation across {} slots...", slots); - - for slot_height in 0..slots { - // Feed deposits to the BeaconChain. - if let Some(ref deposits) = self.config.deposits { - for (slot, deposit, keypair) in deposits { - if *slot == slot_height { - info!("Including deposit at slot height {}.", slot_height); - harness.add_deposit(deposit.clone(), Some(keypair.clone())); - } - } - } - - // Feed proposer slashings to the BeaconChain. - if let Some(ref slashings) = self.config.proposer_slashings { - for (slot, validator_index) in slashings { - if *slot == slot_height { - info!( - "Including proposer slashing at slot height {} for validator #{}.", - slot_height, validator_index - ); - let slashing = build_proposer_slashing(&harness, *validator_index); - harness.add_proposer_slashing(slashing); - } - } - } - - // Feed attester slashings to the BeaconChain. - if let Some(ref slashings) = self.config.attester_slashings { - for (slot, validator_indices) in slashings { - if *slot == slot_height { - info!( - "Including attester slashing at slot height {} for validators {:?}.", - slot_height, validator_indices - ); - let slashing = - build_double_vote_attester_slashing(&harness, &validator_indices[..]); - harness.add_attester_slashing(slashing); - } - } - } - - // Build a block or skip a slot. - match self.config.skip_slots { - Some(ref skip_slots) if skip_slots.contains(&slot_height) => { - warn!("Skipping slot at height {}.", slot_height); - harness.increment_beacon_chain_slot(); - } - _ => { - info!("Producing block at slot height {}.", slot_height); - harness.advance_chain_with_block(); - } - } - } - - harness.run_fork_choice(); - - info!("Test execution complete!"); - - ExecutionResult { - chain: harness.chain_dump().expect("Chain dump failed."), - } - } - - pub fn assert_result_valid(&self, result: ExecutionResult) { - info!("Verifying test results..."); - - let skipped_slots = self - .config - .skip_slots - .clone() - .and_then(|slots| Some(slots.len())) - .unwrap_or_else(|| 0); - let expected_blocks = self.config.num_slots as usize + 1 - skipped_slots; - - assert_eq!(result.chain.len(), expected_blocks); - - info!( - "OK: Chain length is {} ({} skipped slots).", - result.chain.len(), - skipped_slots - ); - - if let Some(ref skip_slots) = self.config.skip_slots { - for checkpoint in &result.chain { - let block_slot = checkpoint.beacon_block.slot.as_u64(); - assert!( - !skip_slots.contains(&block_slot), - "Slot {} was not skipped.", - block_slot - ); - } - info!("OK: Skipped slots not present in chain."); - } - - if let Some(ref deposits) = self.config.deposits { - let latest_state = &result.chain.last().expect("Empty chain.").beacon_state; - assert_eq!( - latest_state.validator_registry.len(), - self.config.deposits_for_chain_start + deposits.len() - ); - info!( - "OK: Validator registry has {} more validators.", - deposits.len() - ); - } - } -} - -fn build_double_vote_attester_slashing( - harness: &BeaconChainHarness, - validator_indices: &[u64], -) -> AttesterSlashing { - let signer = |validator_index: u64, message: &[u8], epoch: Epoch, domain: u64| { - harness - .validator_sign(validator_index as usize, message, epoch, domain) - .expect("Unable to sign AttesterSlashing") - }; - - AttesterSlashingBuilder::double_vote(validator_indices, signer, &harness.spec) -} - -fn build_proposer_slashing(harness: &BeaconChainHarness, validator_index: u64) -> ProposerSlashing { - let signer = |validator_index: u64, message: &[u8], epoch: Epoch, domain: u64| { - harness - .validator_sign(validator_index as usize, message, epoch, domain) - .expect("Unable to sign AttesterSlashing") - }; - - ProposerSlashingBuilder::double_vote(validator_index, signer, &harness.spec) -} - -pub type DepositTuple = (u64, Deposit, Keypair); -pub type ProposerSlashingTuple = (u64, u64); -pub type AttesterSlashingTuple = (u64, Vec); - -struct ExecutionResult { - pub chain: Vec, -} - -struct Results { - pub num_validators: Option, - pub slashed_validators: Option>, - pub exited_validators: Option>, -} - -impl Results { - pub fn from_yaml(yaml: &Yaml) -> Self { - Self { - num_validators: as_usize(&yaml, "num_validators"), - slashed_validators: as_vec_u64(&yaml, "slashed_validators"), - exited_validators: as_vec_u64(&yaml, "exited_validators"), - } - } -} - -struct Config { - pub deposits_for_chain_start: usize, - pub epoch_length: Option, - pub num_slots: u64, - pub skip_slots: Option>, - pub deposits: Option>, - pub proposer_slashings: Option>, - pub attester_slashings: Option>, -} - -impl Config { - pub fn from_yaml(yaml: &Yaml) -> Self { - Self { - deposits_for_chain_start: as_usize(&yaml, "deposits_for_chain_start") - .expect("Must specify validator count"), - epoch_length: as_u64(&yaml, "epoch_length"), - num_slots: as_u64(&yaml, "num_slots").expect("Must specify `config.num_slots`"), - skip_slots: as_vec_u64(yaml, "skip_slots"), - deposits: parse_deposits(&yaml), - proposer_slashings: parse_proposer_slashings(&yaml), - attester_slashings: parse_attester_slashings(&yaml), - } - } -} - -fn parse_attester_slashings(yaml: &Yaml) -> Option> { - let mut slashings = vec![]; - - for slashing in yaml["attester_slashings"].as_vec()? { - let slot = as_u64(slashing, "slot").expect("Incomplete attester_slashing (slot)"); - let validator_indices = as_vec_u64(slashing, "validator_indices") - .expect("Incomplete attester_slashing (validator_indices)"); - - slashings.push((slot, validator_indices)); - } - - Some(slashings) -} - -fn parse_proposer_slashings(yaml: &Yaml) -> Option> { - let mut slashings = vec![]; - - for slashing in yaml["proposer_slashings"].as_vec()? { - let slot = as_u64(slashing, "slot").expect("Incomplete proposer slashing (slot)_"); - let validator_index = as_u64(slashing, "validator_index") - .expect("Incomplete proposer slashing (validator_index)"); - - slashings.push((slot, validator_index)); - } - - Some(slashings) -} - -fn parse_deposits(yaml: &Yaml) -> Option> { - let mut deposits = vec![]; - - for deposit in yaml["deposits"].as_vec()? { - let keypair = Keypair::random(); - let proof_of_possession = create_proof_of_possession(&keypair); - - let slot = as_u64(deposit, "slot").expect("Incomplete deposit"); - let deposit = Deposit { - branch: vec![], - index: as_u64(deposit, "merkle_index").unwrap(), - deposit_data: DepositData { - amount: 32_000_000_000, - timestamp: 1, - deposit_input: DepositInput { - pubkey: keypair.pk.clone(), - withdrawal_credentials: Hash256::zero(), - proof_of_possession, - }, - }, - }; - - deposits.push((slot, deposit, keypair)); - } - - Some(deposits) -} - -fn as_usize(yaml: &Yaml, key: &str) -> Option { - yaml[key].as_i64().and_then(|n| Some(n as usize)) -} - -fn as_u64(yaml: &Yaml, key: &str) -> Option { - yaml[key].as_i64().and_then(|n| Some(n as u64)) -} - -fn as_vec_u64(yaml: &Yaml, key: &str) -> Option> { - yaml[key].clone().into_vec().and_then(|vec| { - Some( - vec.iter() - .map(|item| item.as_i64().unwrap() as u64) - .collect(), - ) - }) -} diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/config.rs b/beacon_node/beacon_chain/test_harness/src/manifest/config.rs new file mode 100644 index 000000000..f960a0bb8 --- /dev/null +++ b/beacon_node/beacon_chain/test_harness/src/manifest/config.rs @@ -0,0 +1,89 @@ +use super::yaml_helpers::{as_u64, as_usize, as_vec_u64}; +use bls::create_proof_of_possession; +use types::*; +use yaml_rust::Yaml; + +pub type DepositTuple = (u64, Deposit, Keypair); +pub type ProposerSlashingTuple = (u64, u64); +pub type AttesterSlashingTuple = (u64, Vec); + +pub struct Config { + pub deposits_for_chain_start: usize, + pub epoch_length: Option, + pub num_slots: u64, + pub skip_slots: Option>, + pub deposits: Option>, + pub proposer_slashings: Option>, + pub attester_slashings: Option>, +} + +impl Config { + pub fn from_yaml(yaml: &Yaml) -> Self { + Self { + deposits_for_chain_start: as_usize(&yaml, "deposits_for_chain_start") + .expect("Must specify validator count"), + epoch_length: as_u64(&yaml, "epoch_length"), + num_slots: as_u64(&yaml, "num_slots").expect("Must specify `config.num_slots`"), + skip_slots: as_vec_u64(yaml, "skip_slots"), + deposits: parse_deposits(&yaml), + proposer_slashings: parse_proposer_slashings(&yaml), + attester_slashings: parse_attester_slashings(&yaml), + } + } +} + +fn parse_attester_slashings(yaml: &Yaml) -> Option> { + let mut slashings = vec![]; + + for slashing in yaml["attester_slashings"].as_vec()? { + let slot = as_u64(slashing, "slot").expect("Incomplete attester_slashing (slot)"); + let validator_indices = as_vec_u64(slashing, "validator_indices") + .expect("Incomplete attester_slashing (validator_indices)"); + + slashings.push((slot, validator_indices)); + } + + Some(slashings) +} + +fn parse_proposer_slashings(yaml: &Yaml) -> Option> { + let mut slashings = vec![]; + + for slashing in yaml["proposer_slashings"].as_vec()? { + let slot = as_u64(slashing, "slot").expect("Incomplete proposer slashing (slot)_"); + let validator_index = as_u64(slashing, "validator_index") + .expect("Incomplete proposer slashing (validator_index)"); + + slashings.push((slot, validator_index)); + } + + Some(slashings) +} + +fn parse_deposits(yaml: &Yaml) -> Option> { + let mut deposits = vec![]; + + for deposit in yaml["deposits"].as_vec()? { + let keypair = Keypair::random(); + let proof_of_possession = create_proof_of_possession(&keypair); + + let slot = as_u64(deposit, "slot").expect("Incomplete deposit"); + let deposit = Deposit { + branch: vec![], + index: as_u64(deposit, "merkle_index").unwrap(), + deposit_data: DepositData { + amount: 32_000_000_000, + timestamp: 1, + deposit_input: DepositInput { + pubkey: keypair.pk.clone(), + withdrawal_credentials: Hash256::zero(), + proof_of_possession, + }, + }, + }; + + deposits.push((slot, deposit, keypair)); + } + + Some(deposits) +} diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs b/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs new file mode 100644 index 000000000..3149a7d16 --- /dev/null +++ b/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs @@ -0,0 +1,185 @@ +use self::config::Config; +use self::results::Results; +use crate::beacon_chain_harness::BeaconChainHarness; +use beacon_chain::CheckPoint; +use log::{info, warn}; +use types::*; +use types::{ + attester_slashing::AttesterSlashingBuilder, proposer_slashing::ProposerSlashingBuilder, +}; +use yaml_rust::Yaml; + +mod config; +mod results; +mod yaml_helpers; + +pub struct Manifest { + pub results: Results, + pub config: Config, +} + +pub struct ExecutionResult { + pub chain: Vec, +} + +impl Manifest { + pub fn from_yaml(test_case: &Yaml) -> Self { + Self { + results: Results::from_yaml(&test_case["results"]), + config: Config::from_yaml(&test_case["config"]), + } + } + + fn spec(&self) -> ChainSpec { + let mut spec = ChainSpec::foundation(); + + if let Some(n) = self.config.epoch_length { + spec.epoch_length = n; + } + + spec + } + + pub fn execute(&self) -> ExecutionResult { + let spec = self.spec(); + let validator_count = self.config.deposits_for_chain_start; + let slots = self.config.num_slots; + + info!( + "Building BeaconChainHarness with {} validators...", + validator_count + ); + + let mut harness = BeaconChainHarness::new(spec, validator_count); + + info!("Starting simulation across {} slots...", slots); + + for slot_height in 0..slots { + // Feed deposits to the BeaconChain. + if let Some(ref deposits) = self.config.deposits { + for (slot, deposit, keypair) in deposits { + if *slot == slot_height { + info!("Including deposit at slot height {}.", slot_height); + harness.add_deposit(deposit.clone(), Some(keypair.clone())); + } + } + } + + // Feed proposer slashings to the BeaconChain. + if let Some(ref slashings) = self.config.proposer_slashings { + for (slot, validator_index) in slashings { + if *slot == slot_height { + info!( + "Including proposer slashing at slot height {} for validator #{}.", + slot_height, validator_index + ); + let slashing = build_proposer_slashing(&harness, *validator_index); + harness.add_proposer_slashing(slashing); + } + } + } + + // Feed attester slashings to the BeaconChain. + if let Some(ref slashings) = self.config.attester_slashings { + for (slot, validator_indices) in slashings { + if *slot == slot_height { + info!( + "Including attester slashing at slot height {} for validators {:?}.", + slot_height, validator_indices + ); + let slashing = + build_double_vote_attester_slashing(&harness, &validator_indices[..]); + harness.add_attester_slashing(slashing); + } + } + } + + // Build a block or skip a slot. + match self.config.skip_slots { + Some(ref skip_slots) if skip_slots.contains(&slot_height) => { + warn!("Skipping slot at height {}.", slot_height); + harness.increment_beacon_chain_slot(); + } + _ => { + info!("Producing block at slot height {}.", slot_height); + harness.advance_chain_with_block(); + } + } + } + + harness.run_fork_choice(); + + info!("Test execution complete!"); + + ExecutionResult { + chain: harness.chain_dump().expect("Chain dump failed."), + } + } + + pub fn assert_result_valid(&self, result: ExecutionResult) { + info!("Verifying test results..."); + + let skipped_slots = self + .config + .skip_slots + .clone() + .and_then(|slots| Some(slots.len())) + .unwrap_or_else(|| 0); + let expected_blocks = self.config.num_slots as usize + 1 - skipped_slots; + + assert_eq!(result.chain.len(), expected_blocks); + + info!( + "OK: Chain length is {} ({} skipped slots).", + result.chain.len(), + skipped_slots + ); + + if let Some(ref skip_slots) = self.config.skip_slots { + for checkpoint in &result.chain { + let block_slot = checkpoint.beacon_block.slot.as_u64(); + assert!( + !skip_slots.contains(&block_slot), + "Slot {} was not skipped.", + block_slot + ); + } + info!("OK: Skipped slots not present in chain."); + } + + if let Some(ref deposits) = self.config.deposits { + let latest_state = &result.chain.last().expect("Empty chain.").beacon_state; + assert_eq!( + latest_state.validator_registry.len(), + self.config.deposits_for_chain_start + deposits.len() + ); + info!( + "OK: Validator registry has {} more validators.", + deposits.len() + ); + } + } +} + +fn build_double_vote_attester_slashing( + harness: &BeaconChainHarness, + validator_indices: &[u64], +) -> AttesterSlashing { + let signer = |validator_index: u64, message: &[u8], epoch: Epoch, domain: u64| { + harness + .validator_sign(validator_index as usize, message, epoch, domain) + .expect("Unable to sign AttesterSlashing") + }; + + AttesterSlashingBuilder::double_vote(validator_indices, signer, &harness.spec) +} + +fn build_proposer_slashing(harness: &BeaconChainHarness, validator_index: u64) -> ProposerSlashing { + let signer = |validator_index: u64, message: &[u8], epoch: Epoch, domain: u64| { + harness + .validator_sign(validator_index as usize, message, epoch, domain) + .expect("Unable to sign AttesterSlashing") + }; + + ProposerSlashingBuilder::double_vote(validator_index, signer, &harness.spec) +} diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/results.rs b/beacon_node/beacon_chain/test_harness/src/manifest/results.rs new file mode 100644 index 000000000..2d84e12dc --- /dev/null +++ b/beacon_node/beacon_chain/test_harness/src/manifest/results.rs @@ -0,0 +1,18 @@ +use super::yaml_helpers::{as_usize, as_vec_u64}; +use yaml_rust::Yaml; + +pub struct Results { + pub num_validators: Option, + pub slashed_validators: Option>, + pub exited_validators: Option>, +} + +impl Results { + pub fn from_yaml(yaml: &Yaml) -> Self { + Self { + num_validators: as_usize(&yaml, "num_validators"), + slashed_validators: as_vec_u64(&yaml, "slashed_validators"), + exited_validators: as_vec_u64(&yaml, "exited_validators"), + } + } +} diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/yaml_helpers.rs b/beacon_node/beacon_chain/test_harness/src/manifest/yaml_helpers.rs new file mode 100644 index 000000000..c499b3c0f --- /dev/null +++ b/beacon_node/beacon_chain/test_harness/src/manifest/yaml_helpers.rs @@ -0,0 +1,19 @@ +use yaml_rust::Yaml; + +pub fn as_usize(yaml: &Yaml, key: &str) -> Option { + yaml[key].as_i64().and_then(|n| Some(n as usize)) +} + +pub fn as_u64(yaml: &Yaml, key: &str) -> Option { + yaml[key].as_i64().and_then(|n| Some(n as u64)) +} + +pub fn as_vec_u64(yaml: &Yaml, key: &str) -> Option> { + yaml[key].clone().into_vec().and_then(|vec| { + Some( + vec.iter() + .map(|item| item.as_i64().unwrap() as u64) + .collect(), + ) + }) +} From 4db2f082e133697493794194bf29750a371d429d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 2 Mar 2019 20:17:14 +1100 Subject: [PATCH 071/132] Add state-checks to test_harness YAML Runs tests against a state at some slot --- .../test_harness/examples/chain.yaml | 11 ++- .../test_harness/src/manifest/config.rs | 1 + .../test_harness/src/manifest/mod.rs | 66 +++++++-------- .../test_harness/src/manifest/results.rs | 24 ++++-- .../test_harness/src/manifest/state_check.rs | 84 +++++++++++++++++++ eth2/types/src/validator.rs | 12 ++- 6 files changed, 151 insertions(+), 47 deletions(-) create mode 100644 beacon_node/beacon_chain/test_harness/src/manifest/state_check.rs diff --git a/beacon_node/beacon_chain/test_harness/examples/chain.yaml b/beacon_node/beacon_chain/test_harness/examples/chain.yaml index 9e00f4569..f22d0874e 100644 --- a/beacon_node/beacon_chain/test_harness/examples/chain.yaml +++ b/beacon_node/beacon_chain/test_harness/examples/chain.yaml @@ -7,7 +7,7 @@ test_cases: - config: epoch_length: 64 deposits_for_chain_start: 1000 - num_slots: 65 + num_slots: 64 skip_slots: [2, 3] deposits: - slot: 1 @@ -34,4 +34,11 @@ test_cases: - slot: 5 validator_indices: [14] results: - num_validators: 1000 + num_skipped_slots: 2 + states: + - slot: 63 + num_validators: 1003 + # slashed_validators: [11, 12, 13, 14, 42] + slashed_validators: [13, 42] # This line is incorrect, our implementation isn't processing attester_slashings. + exited_validators: [] + diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/config.rs b/beacon_node/beacon_chain/test_harness/src/manifest/config.rs index f960a0bb8..0e66a120b 100644 --- a/beacon_node/beacon_chain/test_harness/src/manifest/config.rs +++ b/beacon_node/beacon_chain/test_harness/src/manifest/config.rs @@ -7,6 +7,7 @@ pub type DepositTuple = (u64, Deposit, Keypair); pub type ProposerSlashingTuple = (u64, u64); pub type AttesterSlashingTuple = (u64, Vec); +#[derive(Debug)] pub struct Config { pub deposits_for_chain_start: usize, pub epoch_length: Option, diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs b/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs index 3149a7d16..d16912205 100644 --- a/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs +++ b/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs @@ -11,8 +11,10 @@ use yaml_rust::Yaml; mod config; mod results; +mod state_check; mod yaml_helpers; +#[derive(Debug)] pub struct Manifest { pub results: Results, pub config: Config, @@ -20,6 +22,7 @@ pub struct Manifest { pub struct ExecutionResult { pub chain: Vec, + pub spec: ChainSpec, } impl Manifest { @@ -54,7 +57,8 @@ impl Manifest { info!("Starting simulation across {} slots...", slots); - for slot_height in 0..slots { + // -1 slots because genesis counts as a slot. + for slot_height in 0..slots - 1 { // Feed deposits to the BeaconChain. if let Some(ref deposits) = self.config.deposits { for (slot, deposit, keypair) in deposits { @@ -113,51 +117,41 @@ impl Manifest { ExecutionResult { chain: harness.chain_dump().expect("Chain dump failed."), + spec: (*harness.spec).clone(), } } - pub fn assert_result_valid(&self, result: ExecutionResult) { + pub fn assert_result_valid(&self, execution_result: ExecutionResult) { info!("Verifying test results..."); + let spec = &execution_result.spec; - let skipped_slots = self - .config - .skip_slots - .clone() - .and_then(|slots| Some(slots.len())) - .unwrap_or_else(|| 0); - let expected_blocks = self.config.num_slots as usize + 1 - skipped_slots; - - assert_eq!(result.chain.len(), expected_blocks); - - info!( - "OK: Chain length is {} ({} skipped slots).", - result.chain.len(), - skipped_slots - ); - - if let Some(ref skip_slots) = self.config.skip_slots { - for checkpoint in &result.chain { - let block_slot = checkpoint.beacon_block.slot.as_u64(); - assert!( - !skip_slots.contains(&block_slot), - "Slot {} was not skipped.", - block_slot - ); - } - info!("OK: Skipped slots not present in chain."); - } - - if let Some(ref deposits) = self.config.deposits { - let latest_state = &result.chain.last().expect("Empty chain.").beacon_state; + if let Some(num_skipped_slots) = self.results.num_skipped_slots { assert_eq!( - latest_state.validator_registry.len(), - self.config.deposits_for_chain_start + deposits.len() + execution_result.chain.len(), + self.config.num_slots as usize - num_skipped_slots, + "actual skipped slots != expected." ); info!( - "OK: Validator registry has {} more validators.", - deposits.len() + "OK: Chain length is {} ({} skipped slots).", + execution_result.chain.len(), + num_skipped_slots ); } + + if let Some(ref state_checks) = self.results.state_checks { + for checkpoint in &execution_result.chain { + let state = &checkpoint.beacon_state; + + for state_check in state_checks { + let adjusted_state_slot = + state.slot - spec.genesis_epoch.start_slot(spec.epoch_length); + + if state_check.slot == adjusted_state_slot { + state_check.assert_valid(state, spec); + } + } + } + } } } diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/results.rs b/beacon_node/beacon_chain/test_harness/src/manifest/results.rs index 2d84e12dc..844695910 100644 --- a/beacon_node/beacon_chain/test_harness/src/manifest/results.rs +++ b/beacon_node/beacon_chain/test_harness/src/manifest/results.rs @@ -1,18 +1,28 @@ -use super::yaml_helpers::{as_usize, as_vec_u64}; +use super::state_check::StateCheck; +use super::yaml_helpers::as_usize; use yaml_rust::Yaml; +#[derive(Debug)] pub struct Results { - pub num_validators: Option, - pub slashed_validators: Option>, - pub exited_validators: Option>, + pub num_skipped_slots: Option, + pub state_checks: Option>, } impl Results { pub fn from_yaml(yaml: &Yaml) -> Self { Self { - num_validators: as_usize(&yaml, "num_validators"), - slashed_validators: as_vec_u64(&yaml, "slashed_validators"), - exited_validators: as_vec_u64(&yaml, "exited_validators"), + num_skipped_slots: as_usize(yaml, "num_skipped_slots"), + state_checks: parse_state_checks(yaml), } } } + +fn parse_state_checks(yaml: &Yaml) -> Option> { + let mut states = vec![]; + + for state_yaml in yaml["states"].as_vec()? { + states.push(StateCheck::from_yaml(state_yaml)); + } + + Some(states) +} diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/state_check.rs b/beacon_node/beacon_chain/test_harness/src/manifest/state_check.rs new file mode 100644 index 000000000..0415d4896 --- /dev/null +++ b/beacon_node/beacon_chain/test_harness/src/manifest/state_check.rs @@ -0,0 +1,84 @@ +use super::yaml_helpers::{as_u64, as_usize, as_vec_u64}; +use log::info; +use types::*; +use yaml_rust::Yaml; + +#[derive(Debug)] +pub struct StateCheck { + pub slot: Slot, + pub num_validators: Option, + pub slashed_validators: Option>, + pub exited_validators: Option>, +} + +impl StateCheck { + pub fn from_yaml(yaml: &Yaml) -> Self { + Self { + slot: Slot::from(as_u64(&yaml, "slot").expect("State must specify slot")), + num_validators: as_usize(&yaml, "num_validators"), + slashed_validators: as_vec_u64(&yaml, "slashed_validators"), + exited_validators: as_vec_u64(&yaml, "exited_validators"), + } + } + + pub fn assert_valid(&self, state: &BeaconState, spec: &ChainSpec) { + let state_epoch = state.slot.epoch(spec.epoch_length); + + info!("Running state check for slot height {}.", self.slot); + + assert_eq!( + self.slot, + state.slot - spec.genesis_epoch.start_slot(spec.epoch_length), + "State slot is invalid." + ); + + if let Some(num_validators) = self.num_validators { + assert_eq!( + state.validator_registry.len(), + num_validators, + "State validator count != expected." + ); + info!("OK: num_validators = {}.", num_validators); + } + + if let Some(ref slashed_validators) = self.slashed_validators { + let actually_slashed_validators: Vec = state + .validator_registry + .iter() + .enumerate() + .filter_map(|(i, validator)| { + if validator.is_penalized_at(state_epoch) { + Some(i as u64) + } else { + None + } + }) + .collect(); + assert_eq!( + actually_slashed_validators, *slashed_validators, + "Slashed validators != expected." + ); + info!("OK: slashed_validators = {:?}.", slashed_validators); + } + + if let Some(ref exited_validators) = self.exited_validators { + let actually_exited_validators: Vec = state + .validator_registry + .iter() + .enumerate() + .filter_map(|(i, validator)| { + if validator.is_exited_at(state_epoch) { + Some(i as u64) + } else { + None + } + }) + .collect(); + assert_eq!( + actually_exited_validators, *exited_validators, + "Exited validators != expected." + ); + info!("OK: exited_validators = {:?}.", exited_validators); + } + } +} diff --git a/eth2/types/src/validator.rs b/eth2/types/src/validator.rs index b832283a0..bc8d467ec 100644 --- a/eth2/types/src/validator.rs +++ b/eth2/types/src/validator.rs @@ -55,8 +55,16 @@ pub struct Validator { impl Validator { /// This predicate indicates if the validator represented by this record is considered "active" at `slot`. - pub fn is_active_at(&self, slot: Epoch) -> bool { - self.activation_epoch <= slot && slot < self.exit_epoch + pub fn is_active_at(&self, epoch: Epoch) -> bool { + self.activation_epoch <= epoch && epoch < self.exit_epoch + } + + pub fn is_exited_at(&self, epoch: Epoch) -> bool { + self.exit_epoch <= epoch + } + + pub fn is_penalized_at(&self, epoch: Epoch) -> bool { + self.penalized_epoch <= epoch } } From 9156aa220388bc6e734c41df805dc1d255827542 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 2 Mar 2019 20:20:06 +1100 Subject: [PATCH 072/132] Add info log when building test_harness chain dump It helps people know why they're waiting --- beacon_node/beacon_chain/test_harness/src/manifest/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs b/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs index d16912205..4b8385035 100644 --- a/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs +++ b/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs @@ -115,6 +115,8 @@ impl Manifest { info!("Test execution complete!"); + info!("Building chain dump for analysis..."); + ExecutionResult { chain: harness.chain_dump().expect("Chain dump failed."), spec: (*harness.spec).clone(), From 8a768819b0a832f323a6dc00c7e1f5149902aa14 Mon Sep 17 00:00:00 2001 From: mjkeating Date: Sat, 2 Mar 2019 09:59:01 -0800 Subject: [PATCH 073/132] brought algorithm in TreeHash macro to spec --- eth2/utils/ssz_derive/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eth2/utils/ssz_derive/src/lib.rs b/eth2/utils/ssz_derive/src/lib.rs index ac66526fe..f71bff709 100644 --- a/eth2/utils/ssz_derive/src/lib.rs +++ b/eth2/utils/ssz_derive/src/lib.rs @@ -147,12 +147,12 @@ pub fn ssz_tree_hash_derive(input: TokenStream) -> TokenStream { let output = quote! { impl ssz::TreeHash for #name { fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; + let mut list: Vec> = Vec::new(); #( - result.append(&mut self.#field_idents.hash_tree_root_internal()); + list.push(self.#field_idents.hash_tree_root_internal()); )* - ssz::hash(&result) + ssz::merkle_hash(&mut list) } } }; From 35ae1b6745bc2dda69c2d6787cdb09018905b71d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 3 Mar 2019 11:10:38 +1100 Subject: [PATCH 074/132] Add agg_pub to bls, add agg_sig.verify_multiple - Adds a new-type wrapper for `AggregatePublicKey`, just like all the other types. - Adds the `verify_multiple` method to the `AggregateSignature` newtype, as was introduced in a recent version of signature-schemes. --- eth2/utils/bls/src/aggregate_public_key.rs | 24 ++++++++++++++ eth2/utils/bls/src/aggregate_signature.rs | 37 ++++++++++++++++++++-- eth2/utils/bls/src/lib.rs | 4 +-- 3 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 eth2/utils/bls/src/aggregate_public_key.rs diff --git a/eth2/utils/bls/src/aggregate_public_key.rs b/eth2/utils/bls/src/aggregate_public_key.rs new file mode 100644 index 000000000..dcb08126c --- /dev/null +++ b/eth2/utils/bls/src/aggregate_public_key.rs @@ -0,0 +1,24 @@ +use super::PublicKey; +use bls_aggregates::AggregatePublicKey as RawAggregatePublicKey; + +/// A single BLS signature. +/// +/// This struct is a wrapper upon a base type and provides helper functions (e.g., SSZ +/// serialization). +#[derive(Debug, Clone)] +pub struct AggregatePublicKey(RawAggregatePublicKey); + +impl AggregatePublicKey { + pub fn new() -> Self { + AggregatePublicKey(RawAggregatePublicKey::new()) + } + + pub fn add(&mut self, public_key: &PublicKey) { + self.0.add(public_key.as_raw()) + } + + /// Returns the underlying signature. + pub fn as_raw(&self) -> &RawAggregatePublicKey { + &self.0 + } +} diff --git a/eth2/utils/bls/src/aggregate_signature.rs b/eth2/utils/bls/src/aggregate_signature.rs index 4ee79d0aa..2d8776353 100644 --- a/eth2/utils/bls/src/aggregate_signature.rs +++ b/eth2/utils/bls/src/aggregate_signature.rs @@ -1,5 +1,7 @@ use super::{AggregatePublicKey, Signature}; -use bls_aggregates::AggregateSignature as RawAggregateSignature; +use bls_aggregates::{ + AggregatePublicKey as RawAggregatePublicKey, AggregateSignature as RawAggregateSignature, +}; use serde::ser::{Serialize, Serializer}; use ssz::{ decode_ssz_list, hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash, @@ -33,7 +35,38 @@ impl AggregateSignature { domain: u64, aggregate_public_key: &AggregatePublicKey, ) -> bool { - self.0.verify(msg, domain, aggregate_public_key) + self.0.verify(msg, domain, aggregate_public_key.as_raw()) + } + + /// Verify this AggregateSignature against multiple AggregatePublickeys with multiple Messages. + /// + /// All PublicKeys related to a Message should be aggregated into one AggregatePublicKey. + /// Each AggregatePublicKey has a 1:1 ratio with a 32 byte Message. + pub fn verify_multiple( + &self, + messages: &[&[u8]], + domain: u64, + aggregate_public_keys: &[&AggregatePublicKey], + ) -> bool { + // TODO: the API for `RawAggregatePublicKey` shoudn't need to take an owned + // `AggregatePublicKey`. There is an issue to fix this, but in the meantime we need to + // clone. + // + // https://github.com/sigp/signature-schemes/issues/10 + let aggregate_public_keys: Vec = aggregate_public_keys + .iter() + .map(|pk| pk.as_raw()) + .cloned() + .collect(); + + // Messages are concatenated into one long message. + let mut msg: Vec = vec![]; + for message in messages { + msg.extend_from_slice(message); + } + + self.0 + .verify_multiple(&msg[..], domain, &aggregate_public_keys[..]) } } diff --git a/eth2/utils/bls/src/lib.rs b/eth2/utils/bls/src/lib.rs index 8f2e9fac0..865b8d82d 100644 --- a/eth2/utils/bls/src/lib.rs +++ b/eth2/utils/bls/src/lib.rs @@ -1,20 +1,20 @@ extern crate bls_aggregates; extern crate ssz; +mod aggregate_public_key; mod aggregate_signature; mod keypair; mod public_key; mod secret_key; mod signature; +pub use crate::aggregate_public_key::AggregatePublicKey; pub use crate::aggregate_signature::AggregateSignature; pub use crate::keypair::Keypair; pub use crate::public_key::PublicKey; pub use crate::secret_key::SecretKey; pub use crate::signature::Signature; -pub use self::bls_aggregates::AggregatePublicKey; - pub const BLS_AGG_SIG_BYTE_SIZE: usize = 96; use ssz::ssz_encode; From 3561d44cbe38b13b5faa485aa38c706df71c38cd Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 3 Mar 2019 11:12:18 +1100 Subject: [PATCH 075/132] Update per-block processing for new AggPub wrapper AggregatePublicKey newtype was introduced in previous commit --- eth2/state_processing/src/block_processable.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/eth2/state_processing/src/block_processable.rs b/eth2/state_processing/src/block_processable.rs index 4083d17d8..e434b0c83 100644 --- a/eth2/state_processing/src/block_processable.rs +++ b/eth2/state_processing/src/block_processable.rs @@ -383,11 +383,7 @@ fn validate_attestation_signature_optional( ); let mut group_public_key = AggregatePublicKey::new(); for participant in participants { - group_public_key.add( - state.validator_registry[participant as usize] - .pubkey - .as_raw(), - ) + group_public_key.add(&state.validator_registry[participant as usize].pubkey) } ensure!( attestation.verify_signature( From 59128f842af60fcc42f9d2a6449ec44e00409d5e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 3 Mar 2019 11:16:59 +1100 Subject: [PATCH 076/132] Add `verify_slashable_attestation` spec method As per v0.2.0 spec --- eth2/types/src/beacon_state.rs | 108 +++++++++++++++++++++++++++++++-- 1 file changed, 103 insertions(+), 5 deletions(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 6dfbf78ee..505d4d9de 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -1,10 +1,6 @@ use self::epoch_cache::EpochCache; use crate::test_utils::TestRandom; -use crate::{ - validator::StatusFlags, validator_registry::get_active_validator_indices, AttestationData, - Bitfield, ChainSpec, Crosslink, Deposit, DepositData, DepositInput, Epoch, Eth1Data, - Eth1DataVote, Fork, Hash256, PendingAttestation, PublicKey, Signature, Slot, Validator, -}; +use crate::{validator::StatusFlags, validator_registry::get_active_validator_indices, *}; use bls::verify_proof_of_possession; use honey_badger_split::SplitExt; use log::{debug, error, trace}; @@ -1145,6 +1141,108 @@ impl BeaconState { ) } + pub fn verify_bitfield(&self, bitfield: &Bitfield, committee_size: usize) -> bool { + if bitfield.num_bytes() != ((committee_size + 7) / 8) { + return false; + } + + for i in committee_size..(bitfield.num_bytes() * 8) { + match bitfield.get(i) { + Ok(bit) => { + if bit { + return false; + } + } + Err(_) => unreachable!(), + } + } + + true + } + + pub fn verify_slashable_attestation( + &self, + slashable_attestation: &SlashableAttestation, + spec: &ChainSpec, + ) -> bool { + if slashable_attestation.custody_bitfield.num_set_bits() > 0 { + return false; + } + + if slashable_attestation.validator_indices.is_empty() { + return false; + } + + for i in 0..(slashable_attestation.validator_indices.len() - 1) { + if slashable_attestation.validator_indices[i] + >= slashable_attestation.validator_indices[i + 1] + { + return false; + } + } + + if !self.verify_bitfield( + &slashable_attestation.custody_bitfield, + slashable_attestation.validator_indices.len(), + ) { + return false; + } + + if slashable_attestation.validator_indices.len() + > spec.max_indices_per_slashable_vote as usize + { + return false; + } + + let mut aggregate_pubs = vec![AggregatePublicKey::new(); 2]; + let mut message_exists = vec![false; 2]; + + for (i, v) in slashable_attestation.validator_indices.iter().enumerate() { + let custody_bit = match slashable_attestation.custody_bitfield.get(i) { + Ok(bit) => bit, + Err(_) => unreachable!(), + }; + + message_exists[custody_bit as usize] = true; + + match self.validator_registry.get(*v as usize) { + Some(validator) => { + aggregate_pubs[custody_bit as usize].add(&validator.pubkey); + } + None => return false, + }; + } + + let message_0 = AttestationDataAndCustodyBit { + data: slashable_attestation.data.clone(), + custody_bit: false, + } + .hash_tree_root(); + let message_1 = AttestationDataAndCustodyBit { + data: slashable_attestation.data.clone(), + custody_bit: true, + } + .hash_tree_root(); + + let mut messages = vec![]; + let mut keys = vec![]; + + if message_exists[0] { + messages.push(&message_0[..]); + keys.push(&aggregate_pubs[0]); + } + if message_exists[1] { + messages.push(&message_1[..]); + keys.push(&aggregate_pubs[1]); + } + + slashable_attestation.aggregate_signature.verify_multiple( + &messages[..], + spec.domain_attestation, + &keys[..], + ) + } + /// Return the block root at a recent `slot`. /// /// Spec v0.2.0 From 76a0ba2d6c452bc9472f5eb3cbb0063bf9e2f7cf Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 3 Mar 2019 11:18:12 +1100 Subject: [PATCH 077/132] Add attester slashing support to block processing At spec v0.2.0 --- .../state_processing/src/block_processable.rs | 18 +++++- .../verify_slashable_attestation.rs | 63 +++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 eth2/state_processing/src/block_processable/verify_slashable_attestation.rs diff --git a/eth2/state_processing/src/block_processable.rs b/eth2/state_processing/src/block_processable.rs index e434b0c83..32327aad3 100644 --- a/eth2/state_processing/src/block_processable.rs +++ b/eth2/state_processing/src/block_processable.rs @@ -1,3 +1,4 @@ +use self::verify_slashable_attestation::verify_slashable_attestation; use crate::SlotProcessingError; use hashing::hash; use int_to_bytes::int_to_bytes32; @@ -5,6 +6,8 @@ use log::{debug, trace}; use ssz::{ssz_encode, TreeHash}; use types::*; +mod verify_slashable_attestation; + const PHASE_0_CUSTODY_BIT: bool = false; #[derive(Debug, PartialEq)] @@ -23,7 +26,9 @@ pub enum Error { BadRandaoSignature, MaxProposerSlashingsExceeded, BadProposerSlashing, + MaxAttesterSlashingsExceed, MaxAttestationsExceeded, + BadAttesterSlashing, InvalidAttestation(AttestationValidationError), NoBlockRoot, MaxDepositsExceeded, @@ -82,7 +87,7 @@ impl BlockProcessable for BeaconState { } fn per_block_processing_signature_optional( - state: &mut BeaconState, + mut state: &mut BeaconState, block: &BeaconBlock, verify_block_signature: bool, spec: &ChainSpec, @@ -205,6 +210,17 @@ fn per_block_processing_signature_optional( state.penalize_validator(proposer_slashing.proposer_index as usize, spec)?; } + /* + * Attester slashings + */ + ensure!( + block.body.attester_slashings.len() as u64 <= spec.max_attester_slashings, + Error::MaxAttesterSlashingsExceed + ); + for attester_slashing in &block.body.attester_slashings { + verify_slashable_attestation(&mut state, &attester_slashing, spec)?; + } + /* * Attestations */ diff --git a/eth2/state_processing/src/block_processable/verify_slashable_attestation.rs b/eth2/state_processing/src/block_processable/verify_slashable_attestation.rs new file mode 100644 index 000000000..dddc2eb1f --- /dev/null +++ b/eth2/state_processing/src/block_processable/verify_slashable_attestation.rs @@ -0,0 +1,63 @@ +use super::Error; +use log::error; +use types::*; + +macro_rules! ensure { + ($condition: expr, $result: expr) => { + if !$condition { + return Err($result); + } + }; +} + +pub fn verify_slashable_attestation( + state: &mut BeaconState, + attester_slashing: &AttesterSlashing, + spec: &ChainSpec, +) -> Result<(), Error> { + let slashable_attestation_1 = &attester_slashing.slashable_attestation_1; + let slashable_attestation_2 = &attester_slashing.slashable_attestation_2; + + ensure!( + slashable_attestation_1.data != slashable_attestation_2.data, + Error::BadAttesterSlashing + ); + ensure!( + slashable_attestation_1.is_double_vote(slashable_attestation_2, spec) + | slashable_attestation_1.is_surround_vote(slashable_attestation_2, spec), + Error::BadAttesterSlashing + ); + error!("this a"); + ensure!( + state.verify_slashable_attestation(&slashable_attestation_1, spec), + Error::BadAttesterSlashing + ); + error!("this b"); + ensure!( + state.verify_slashable_attestation(&slashable_attestation_2, spec), + Error::BadAttesterSlashing + ); + error!("this c"); + + let mut slashable_indices = vec![]; + for i in &slashable_attestation_1.validator_indices { + let validator = state + .validator_registry + .get(*i as usize) + .ok_or_else(|| Error::BadAttesterSlashing)?; + + if slashable_attestation_1.validator_indices.contains(&i) + & !validator.is_penalized_at(state.current_epoch(spec)) + { + slashable_indices.push(i); + } + } + + ensure!(slashable_indices.len() >= 1, Error::BadAttesterSlashing); + + for i in slashable_indices { + state.penalize_validator(*i as usize, spec)?; + } + + Ok(()) +} From a8c3b5fdd8b46652931a681e0c224e271f12178e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 3 Mar 2019 11:19:27 +1100 Subject: [PATCH 078/132] Update test_harness yaml - Checks for attester slashing, now it is included in the chain. - Renames suite to be more specific, use normal Eth2.0 naming --- .../beacon_chain/test_harness/examples/chain.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/examples/chain.yaml b/beacon_node/beacon_chain/test_harness/examples/chain.yaml index f22d0874e..dd5aa7cc9 100644 --- a/beacon_node/beacon_chain/test_harness/examples/chain.yaml +++ b/beacon_node/beacon_chain/test_harness/examples/chain.yaml @@ -1,7 +1,7 @@ -title: Sample Ethereum Serenity State Transition Tests -summary: Testing full state transition block processing -test_suite: prysm -fork: sapphire +title: Validator Registry Tests +summary: Tests deposit and slashing effects on validator registry. +test_suite: validator_registry +fork: tchaikovsky version: 1.0 test_cases: - config: @@ -38,7 +38,6 @@ test_cases: states: - slot: 63 num_validators: 1003 - # slashed_validators: [11, 12, 13, 14, 42] - slashed_validators: [13, 42] # This line is incorrect, our implementation isn't processing attester_slashings. + slashed_validators: [11, 12, 13, 14, 42] exited_validators: [] From 87feeea1fd328c3e0bc7f72fae40c271a668728f Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 3 Mar 2019 11:31:08 +1100 Subject: [PATCH 079/132] Remove merkle_index from test_harness YAML IMO, this is an implementation detail that shouldn't be covered in these tests. --- .../beacon_chain/test_harness/examples/chain.yaml | 9 ++++----- .../beacon_chain/test_harness/src/manifest/config.rs | 11 ++++++++--- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/examples/chain.yaml b/beacon_node/beacon_chain/test_harness/examples/chain.yaml index dd5aa7cc9..b7fdda9bf 100644 --- a/beacon_node/beacon_chain/test_harness/examples/chain.yaml +++ b/beacon_node/beacon_chain/test_harness/examples/chain.yaml @@ -10,27 +10,26 @@ test_cases: num_slots: 64 skip_slots: [2, 3] deposits: + # At slot 1, create a new validator deposit of 32 ETH. - slot: 1 amount: 32 - merkle_index: 0 + # Trigger more deposits... - slot: 3 amount: 32 - merkle_index: 1 - slot: 5 amount: 32 - merkle_index: 2 proposer_slashings: # At slot 2, trigger a proposer slashing for validator #42. - slot: 2 validator_index: 42 - # At slot 8, trigger a proposer slashing for validator #13. + # Trigger another slashing... - slot: 8 validator_index: 13 attester_slashings: # At slot 2, trigger an attester slashing for validators #11 and #12. - slot: 2 validator_indices: [11, 12] - # At slot 5, trigger an attester slashing for validator #14. + # Trigger another slashing... - slot: 5 validator_indices: [14] results: diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/config.rs b/beacon_node/beacon_chain/test_harness/src/manifest/config.rs index 0e66a120b..be01d48d8 100644 --- a/beacon_node/beacon_chain/test_harness/src/manifest/config.rs +++ b/beacon_node/beacon_chain/test_harness/src/manifest/config.rs @@ -68,12 +68,17 @@ fn parse_deposits(yaml: &Yaml) -> Option> { let keypair = Keypair::random(); let proof_of_possession = create_proof_of_possession(&keypair); - let slot = as_u64(deposit, "slot").expect("Incomplete deposit"); + let slot = as_u64(deposit, "slot").expect("Incomplete deposit (slot)"); + let amount = + as_u64(deposit, "amount").expect("Incomplete deposit (amount)") * 1_000_000_000; + let deposit = Deposit { + // Note: `branch` and `index` will need to be updated once the spec defines their + // validity. branch: vec![], - index: as_u64(deposit, "merkle_index").unwrap(), + index: 0, deposit_data: DepositData { - amount: 32_000_000_000, + amount, timestamp: 1, deposit_input: DepositInput { pubkey: keypair.pk.clone(), From ede5685bc298313d564b38a24a5bbfcbe2a38339 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 3 Mar 2019 11:47:09 +1100 Subject: [PATCH 080/132] Fix warnings and clippy lints --- .../test_harness/src/beacon_chain_harness.rs | 11 ----------- .../src/validator_harness/local_signer.rs | 13 +------------ .../verify_slashable_attestation.rs | 6 +----- eth2/types/src/attester_slashing/builder.rs | 4 ++-- eth2/types/src/beacon_state.rs | 3 +-- eth2/types/src/proposer_slashing/builder.rs | 4 ++-- eth2/utils/bls/src/aggregate_public_key.rs | 2 +- 7 files changed, 8 insertions(+), 35 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index 9060c4ec1..d6608cd39 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -10,10 +10,7 @@ use fork_choice::BitwiseLMDGhost; use log::debug; use rayon::prelude::*; use slot_clock::TestingSlotClock; -use ssz::TreeHash; use std::collections::HashSet; -use std::fs::File; -use std::io::prelude::*; use std::iter::FromIterator; use std::sync::Arc; use types::*; @@ -291,12 +288,4 @@ impl BeaconChainHarness { pub fn chain_dump(&self) -> Result, BeaconChainError> { self.beacon_chain.chain_dump() } - - /// Write the output of `chain_dump` to a JSON file. - pub fn dump_to_file(&self, filename: String, chain_dump: &[CheckPoint]) { - let json = serde_json::to_string(chain_dump).unwrap(); - let mut file = File::create(filename).unwrap(); - file.write_all(json.as_bytes()) - .expect("Failed writing dump to file."); - } } diff --git a/beacon_node/beacon_chain/test_harness/src/validator_harness/local_signer.rs b/beacon_node/beacon_chain/test_harness/src/validator_harness/local_signer.rs index 3f249cb19..803af5045 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator_harness/local_signer.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator_harness/local_signer.rs @@ -1,27 +1,16 @@ use attester::Signer as AttesterSigner; use block_proposer::Signer as BlockProposerSigner; -use std::sync::RwLock; use types::{Keypair, Signature}; /// A test-only struct used to perform signing for a proposer or attester. pub struct LocalSigner { keypair: Keypair, - should_sign: RwLock, } impl LocalSigner { /// Produce a new TestSigner with signing enabled by default. pub fn new(keypair: Keypair) -> Self { - Self { - keypair, - should_sign: RwLock::new(true), - } - } - - /// If set to `false`, the service will refuse to sign all messages. Otherwise, all messages - /// will be signed. - pub fn enable_signing(&self, enabled: bool) { - *self.should_sign.write().unwrap() = enabled; + Self { keypair } } /// Sign some message. diff --git a/eth2/state_processing/src/block_processable/verify_slashable_attestation.rs b/eth2/state_processing/src/block_processable/verify_slashable_attestation.rs index dddc2eb1f..35ad67df0 100644 --- a/eth2/state_processing/src/block_processable/verify_slashable_attestation.rs +++ b/eth2/state_processing/src/block_processable/verify_slashable_attestation.rs @@ -1,5 +1,4 @@ use super::Error; -use log::error; use types::*; macro_rules! ensure { @@ -27,17 +26,14 @@ pub fn verify_slashable_attestation( | slashable_attestation_1.is_surround_vote(slashable_attestation_2, spec), Error::BadAttesterSlashing ); - error!("this a"); ensure!( state.verify_slashable_attestation(&slashable_attestation_1, spec), Error::BadAttesterSlashing ); - error!("this b"); ensure!( state.verify_slashable_attestation(&slashable_attestation_2, spec), Error::BadAttesterSlashing ); - error!("this c"); let mut slashable_indices = vec![]; for i in &slashable_attestation_1.validator_indices { @@ -53,7 +49,7 @@ pub fn verify_slashable_attestation( } } - ensure!(slashable_indices.len() >= 1, Error::BadAttesterSlashing); + ensure!(!slashable_indices.is_empty(), Error::BadAttesterSlashing); for i in slashable_indices { state.penalize_validator(*i as usize, spec)?; diff --git a/eth2/types/src/attester_slashing/builder.rs b/eth2/types/src/attester_slashing/builder.rs index d382cb64c..e53706192 100644 --- a/eth2/types/src/attester_slashing/builder.rs +++ b/eth2/types/src/attester_slashing/builder.rs @@ -16,8 +16,8 @@ impl AttesterSlashingBuilder { let shard = 0; let justified_epoch = Epoch::new(0); let epoch = Epoch::new(0); - let hash_1 = Hash256::from("1".as_bytes()); - let hash_2 = Hash256::from("2".as_bytes()); + let hash_1 = Hash256::from(&[1][..]); + let hash_2 = Hash256::from(&[2][..]); let mut slashable_attestation_1 = SlashableAttestation { validator_indices: validator_indices.to_vec(), diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 505d4d9de..932233445 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -419,8 +419,7 @@ impl BeaconState { committees_per_epoch ); - let active_validator_indices: Vec = - active_validator_indices.iter().cloned().collect(); + let active_validator_indices: Vec = active_validator_indices.to_vec(); let shuffled_active_validator_indices = shuffle_list( active_validator_indices, diff --git a/eth2/types/src/proposer_slashing/builder.rs b/eth2/types/src/proposer_slashing/builder.rs index a43a73ff0..363155a14 100644 --- a/eth2/types/src/proposer_slashing/builder.rs +++ b/eth2/types/src/proposer_slashing/builder.rs @@ -14,13 +14,13 @@ impl ProposerSlashingBuilder { let proposal_data_1 = ProposalSignedData { slot, shard, - block_root: Hash256::from("one".as_bytes()), + block_root: Hash256::from(&[1][..]), }; let proposal_data_2 = ProposalSignedData { slot, shard, - block_root: Hash256::from("two".as_bytes()), + block_root: Hash256::from(&[2][..]), }; let proposal_signature_1 = { diff --git a/eth2/utils/bls/src/aggregate_public_key.rs b/eth2/utils/bls/src/aggregate_public_key.rs index dcb08126c..2174a43cb 100644 --- a/eth2/utils/bls/src/aggregate_public_key.rs +++ b/eth2/utils/bls/src/aggregate_public_key.rs @@ -5,7 +5,7 @@ use bls_aggregates::AggregatePublicKey as RawAggregatePublicKey; /// /// This struct is a wrapper upon a base type and provides helper functions (e.g., SSZ /// serialization). -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct AggregatePublicKey(RawAggregatePublicKey); impl AggregatePublicKey { From f5e4fe29d7e54fb6ad66c79f2581f947af70db56 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 3 Mar 2019 11:54:51 +1100 Subject: [PATCH 081/132] Add comments to new `BeaconChain` methods - Adds comments - Also drops a message from `warn` down to `debug`. It was giving warnings even on an Ok result. --- beacon_node/beacon_chain/src/beacon_chain.rs | 22 ++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 625a89197..e6fd2a134 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -5,7 +5,7 @@ use db::{ ClientDB, DBError, }; use fork_choice::{ForkChoice, ForkChoiceError}; -use log::{debug, trace, warn}; +use log::{debug, trace}; use parking_lot::{RwLock, RwLockReadGuard}; use slot_clock::SlotClock; use ssz::ssz_encode; @@ -368,16 +368,22 @@ where Ok(aggregation_outcome) } + /// Accept some deposit and queue it for inclusion in an appropriate block. pub fn receive_deposit_for_inclusion(&self, deposit: Deposit) { // TODO: deposits are not check for validity; check them. self.deposits_for_inclusion.write().push(deposit); } + /// Return a vec of deposits suitable for inclusion in some block. pub fn get_deposits_for_block(&self) -> Vec { // TODO: deposits are indiscriminately included; check them for validity. self.deposits_for_inclusion.read().clone() } + /// Takes a list of `Deposits` that were included in recent blocks and removes them from the + /// inclusion queue. + /// + /// This ensures that `Deposits` are not included twice in successive blocks. pub fn set_deposits_as_included(&self, included_deposits: &[Deposit]) { // TODO: method does not take forks into account; consider this. let mut indices_to_delete = vec![]; @@ -396,6 +402,7 @@ where } } + /// Accept some proposer slashing and queue it for inclusion in an appropriate block. pub fn receive_proposer_slashing_for_inclusion(&self, proposer_slashing: ProposerSlashing) { // TODO: proposer_slashings are not check for validity; check them. self.proposer_slashings_for_inclusion @@ -403,11 +410,16 @@ where .push(proposer_slashing); } + /// Return a vec of proposer slashings suitable for inclusion in some block. pub fn get_proposer_slashings_for_block(&self) -> Vec { // TODO: proposer_slashings are indiscriminately included; check them for validity. self.proposer_slashings_for_inclusion.read().clone() } + /// Takes a list of `ProposerSlashings` that were included in recent blocks and removes them + /// from the inclusion queue. + /// + /// This ensures that `ProposerSlashings` are not included twice in successive blocks. pub fn set_proposer_slashings_as_included( &self, included_proposer_slashings: &[ProposerSlashing], @@ -434,6 +446,7 @@ where } } + /// Accept some attester slashing and queue it for inclusion in an appropriate block. pub fn receive_attester_slashing_for_inclusion(&self, attester_slashing: AttesterSlashing) { // TODO: attester_slashings are not check for validity; check them. self.attester_slashings_for_inclusion @@ -441,11 +454,16 @@ where .push(attester_slashing); } + /// Return a vec of attester slashings suitable for inclusion in some block. pub fn get_attester_slashings_for_block(&self) -> Vec { // TODO: attester_slashings are indiscriminately included; check them for validity. self.attester_slashings_for_inclusion.read().clone() } + /// Takes a list of `AttesterSlashings` that were included in recent blocks and removes them + /// from the inclusion queue. + /// + /// This ensures that `AttesterSlashings` are not included twice in successive blocks. pub fn set_attester_slashings_as_included( &self, included_attester_slashings: &[AttesterSlashing], @@ -668,7 +686,7 @@ where let result = state.per_block_processing_without_verifying_block_signature(&block, &self.spec); - warn!( + debug!( "BeaconNode::produce_block: state processing result: {:?}", result ); From 1703508385801427e86f7731a177b8a69dd18e87 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 3 Mar 2019 12:02:58 +1100 Subject: [PATCH 082/132] Add comments to new `BeaconChainHarness` methods. --- .../test_harness/src/beacon_chain_harness.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index d6608cd39..2f375f7fa 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -240,6 +240,11 @@ impl BeaconChainHarness { debug!("Free attestations processed."); } + /// Signs a message using some validators secret key with the `Fork` info from the latest state + /// of the `BeaconChain`. + /// + /// Useful for producing slashable messages and other objects that `BeaconChainHarness` does + /// not produce naturally. pub fn validator_sign( &self, validator_index: usize, @@ -259,6 +264,11 @@ impl BeaconChainHarness { Some(Signature::new(message, domain, &validator.keypair.sk)) } + /// Submit a deposit to the `BeaconChain` and, if given a keypair, create a new + /// `ValidatorHarness` instance for this validator. + /// + /// If a new `ValidatorHarness` was created, the validator should become fully operational as + /// if the validator were created during `BeaconChainHarness` instantiation. pub fn add_deposit(&mut self, deposit: Deposit, keypair: Option) { self.beacon_chain.receive_deposit_for_inclusion(deposit); @@ -270,16 +280,19 @@ impl BeaconChainHarness { } } + /// Submit a proposer slashing to the `BeaconChain` for inclusion in some block. pub fn add_proposer_slashing(&mut self, proposer_slashing: ProposerSlashing) { self.beacon_chain .receive_proposer_slashing_for_inclusion(proposer_slashing); } + /// Submit an attester slashing to the `BeaconChain` for inclusion in some block. pub fn add_attester_slashing(&mut self, attester_slashing: AttesterSlashing) { self.beacon_chain .receive_attester_slashing_for_inclusion(attester_slashing); } + /// Executes the fork choice rule on the `BeaconChain`, selecting a new canonical head. pub fn run_fork_choice(&mut self) { self.beacon_chain.fork_choice().unwrap() } From 60cfdf6e554ec59555518495f74d297eb221aef9 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Sun, 3 Mar 2019 13:35:15 +1100 Subject: [PATCH 083/132] Convert bitwise ghost to use u64 block heights. --- eth2/fork_choice/src/bitwise_lmd_ghost.rs | 18 +++++++++--------- eth2/types/src/slot_height.rs | 1 - 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/eth2/fork_choice/src/bitwise_lmd_ghost.rs b/eth2/fork_choice/src/bitwise_lmd_ghost.rs index 58cd2b5a3..60aa38fe7 100644 --- a/eth2/fork_choice/src/bitwise_lmd_ghost.rs +++ b/eth2/fork_choice/src/bitwise_lmd_ghost.rs @@ -24,22 +24,22 @@ use types::{ /// Compute the base-2 logarithm of an integer, floored (rounded down) #[inline] -fn log2_int(x: u32) -> u32 { +fn log2_int(x: u64) -> u32 { if x == 0 { return 0; } - 31 - x.leading_zeros() + 63 - x.leading_zeros() } -fn power_of_2_below(x: u32) -> u32 { - 2u32.pow(log2_int(x)) +fn power_of_2_below(x: u64) -> u64 { + 2u64.pow(log2_int(x)) } /// Stores the necessary data structures to run the optimised bitwise lmd ghost algorithm. pub struct BitwiseLMDGhost { /// A cache of known ancestors at given heights for a specific block. //TODO: Consider FnvHashMap - cache: HashMap, Hash256>, + cache: HashMap, Hash256>, /// Log lookup table for blocks to their ancestors. //TODO: Verify we only want/need a size 16 log lookup ancestors: Vec>, @@ -141,7 +141,7 @@ where } } // check if the result is stored in our cache - let cache_key = CacheKey::new(&block_hash, target_height.as_u32()); + let cache_key = CacheKey::new(&block_hash, target_height.as_u64()); if let Some(ancestor) = self.cache.get(&cache_key) { return Some(*ancestor); } @@ -149,7 +149,7 @@ where // not in the cache recursively search for ancestors using a log-lookup if let Some(ancestor) = { let ancestor_lookup = self.ancestors - [log2_int((block_height - target_height - 1u64).as_u32()) as usize] + [log2_int((block_height - target_height - 1u64).as_u64()) as usize] .get(&block_hash) //TODO: Panic if we can't lookup and fork choice fails .expect("All blocks should be added to the ancestor log lookup table"); @@ -374,7 +374,7 @@ impl ForkChoice for BitwiseLMDGhost { // logarithmic lookup blocks to see if there are obvious winners, if so, // progress to the next iteration. let mut step = - power_of_2_below(self.max_known_height.saturating_sub(block_height).as_u32()) / 2; + power_of_2_below(self.max_known_height.saturating_sub(block_height).as_u64()) / 2; while step > 0 { trace!("Current Step: {}", step); if let Some(clear_winner) = self.get_clear_winner( @@ -474,7 +474,7 @@ mod tests { #[test] pub fn test_power_of_2_below_large() { - let pow: u32 = 1 << 24; + let pow: u64 = 1 << 24; for x in (pow - 20)..(pow + 20) { assert!(power_of_2_below(x) <= x, "{}", x); } diff --git a/eth2/types/src/slot_height.rs b/eth2/types/src/slot_height.rs index afa0ff775..f9370f485 100644 --- a/eth2/types/src/slot_height.rs +++ b/eth2/types/src/slot_height.rs @@ -13,7 +13,6 @@ use std::ops::{Add, AddAssign, Div, DivAssign, Mul, MulAssign, Rem, Sub, SubAssi pub struct SlotHeight(u64); impl_common!(SlotHeight); -impl_into_u32!(SlotHeight); // SlotHeight can be converted to u32 impl SlotHeight { pub fn new(slot: u64) -> SlotHeight { From 7b729349435bd104a1c4d31310621199b3fa11c2 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 3 Mar 2019 15:07:54 +1100 Subject: [PATCH 084/132] Add comments to test_harness::Manifest --- .../beacon_chain/test_harness/src/bin.rs | 12 ++++++ .../beacon_chain/test_harness/src/lib.rs | 28 +++++++++++++ .../test_harness/src/manifest/config.rs | 14 +++++++ .../test_harness/src/manifest/mod.rs | 40 +++++++++++++++++-- .../test_harness/src/manifest/results.rs | 6 +++ .../test_harness/src/manifest/state_check.rs | 14 +++++++ 6 files changed, 111 insertions(+), 3 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/src/bin.rs b/beacon_node/beacon_chain/test_harness/src/bin.rs index b6a0529c7..5411efc4a 100644 --- a/beacon_node/beacon_chain/test_harness/src/bin.rs +++ b/beacon_node/beacon_chain/test_harness/src/bin.rs @@ -37,6 +37,18 @@ fn main() { }; for doc in &docs { + // For each `test_cases` YAML in the document, build a `Manifest`, execute it and + // assert that the execution result matches the manifest description. + // + // In effect, for each `test_case` a new `BeaconChainHarness` is created from genesis + // and a new `BeaconChain` is built as per the manifest. + // + // After the `BeaconChain` has been built out as per the manifest, a dump of all blocks + // and states in the chain is obtained and checked against the `results` specified in + // the `test_case`. + // + // If any of the expectations in the results are not met, the process + // panics with a message. for test_case in doc["test_cases"].as_vec().unwrap() { let manifest = Manifest::from_yaml(test_case); manifest.assert_result_valid(manifest.execute()) diff --git a/beacon_node/beacon_chain/test_harness/src/lib.rs b/beacon_node/beacon_chain/test_harness/src/lib.rs index b04fc6996..a7ada2433 100644 --- a/beacon_node/beacon_chain/test_harness/src/lib.rs +++ b/beacon_node/beacon_chain/test_harness/src/lib.rs @@ -1,4 +1,32 @@ +//! Provides a testing environment for the `BeaconChain`, `Attester` and `BlockProposer` objects. +//! +//! This environment bypasses networking client runtimes and connects the `Attester` and `Proposer` +//! directly to the `BeaconChain` via an `Arc`. +//! +//! The `BeaconChainHarness` contains a single `BeaconChain` instance and many `ValidatorHarness` +//! instances. All of the `ValidatorHarness` instances work to advance the `BeaconChain` by +//! producing blocks and attestations. +//! +//! Example: +//! ``` +//! use test_harness::BeaconChainHarness; +//! use types::ChainSpec; +//! +//! let validator_count = 8; +//! let spec = ChainSpec::few_validators(); +//! +//! let mut harness = BeaconChainHarness::new(spec, validator_count); +//! +//! harness.advance_chain_with_block(); +//! +//! let chain = harness.chain_dump().unwrap(); +//! +//! // One block should have been built on top of the genesis block. +//! assert_eq!(chain.len(), 2); +//! ``` + mod beacon_chain_harness; +pub mod manifest; mod validator_harness; pub use self::beacon_chain_harness::BeaconChainHarness; diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/config.rs b/beacon_node/beacon_chain/test_harness/src/manifest/config.rs index be01d48d8..8c88ee5d1 100644 --- a/beacon_node/beacon_chain/test_harness/src/manifest/config.rs +++ b/beacon_node/beacon_chain/test_harness/src/manifest/config.rs @@ -7,18 +7,29 @@ pub type DepositTuple = (u64, Deposit, Keypair); pub type ProposerSlashingTuple = (u64, u64); pub type AttesterSlashingTuple = (u64, Vec); +/// Defines the execution of a `BeaconStateHarness` across a series of slots. #[derive(Debug)] pub struct Config { + /// Initial validators. pub deposits_for_chain_start: usize, + /// Number of slots in an epoch. pub epoch_length: Option, + /// Number of slots to build before ending execution. pub num_slots: u64, + /// Number of slots that should be skipped due to inactive validator. pub skip_slots: Option>, + /// Deposits to be included during execution. pub deposits: Option>, + /// Proposer slashings to be included during execution. pub proposer_slashings: Option>, + /// Attester slashings to be including during execution. pub attester_slashings: Option>, } impl Config { + /// Load from a YAML document. + /// + /// Expects to receive the `config` section of the document. pub fn from_yaml(yaml: &Yaml) -> Self { Self { deposits_for_chain_start: as_usize(&yaml, "deposits_for_chain_start") @@ -33,6 +44,7 @@ impl Config { } } +/// Parse the `attester_slashings` section of the YAML document. fn parse_attester_slashings(yaml: &Yaml) -> Option> { let mut slashings = vec![]; @@ -47,6 +59,7 @@ fn parse_attester_slashings(yaml: &Yaml) -> Option> { Some(slashings) } +/// Parse the `proposer_slashings` section of the YAML document. fn parse_proposer_slashings(yaml: &Yaml) -> Option> { let mut slashings = vec![]; @@ -61,6 +74,7 @@ fn parse_proposer_slashings(yaml: &Yaml) -> Option> { Some(slashings) } +/// Parse the `deposits` section of the YAML document. fn parse_deposits(yaml: &Yaml) -> Option> { let mut deposits = vec![]; diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs b/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs index 4b8385035..7b9c9cb02 100644 --- a/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs +++ b/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs @@ -1,5 +1,6 @@ -use self::config::Config; -use self::results::Results; +//! Defines execution and testing specs for a `BeaconChainHarness` instance. Supports loading from +//! a YAML file. + use crate::beacon_chain_harness::BeaconChainHarness; use beacon_chain::CheckPoint; use log::{info, warn}; @@ -14,18 +15,36 @@ mod results; mod state_check; mod yaml_helpers; +pub use config::Config; +pub use results::Results; +pub use state_check::StateCheck; + +/// Defines the execution and testing of a `BeaconChainHarness` instantiation. +/// +/// Typical workflow is: +/// +/// 1. Instantiate the `Manifest` from YAML: `let manifest = Manifest::from_yaml(&my_yaml);` +/// 2. Execute the manifest: `let result = manifest.execute();` +/// 3. Test the results against the manifest: `manifest.assert_result_valid(result);` #[derive(Debug)] pub struct Manifest { - pub results: Results, + /// Defines the execution. pub config: Config, + /// Defines tests to run against the execution result. + pub results: Results, } +/// The result of executing a `Manifest`. +/// pub struct ExecutionResult { + /// The canonical beacon chain generated from the execution. pub chain: Vec, + /// The spec used for execution. pub spec: ChainSpec, } impl Manifest { + /// Load the manifest from a YAML document. pub fn from_yaml(test_case: &Yaml) -> Self { Self { results: Results::from_yaml(&test_case["results"]), @@ -33,6 +52,9 @@ impl Manifest { } } + /// Return a `ChainSpec::foundation()`. + /// + /// If specified in `config`, returns it with a modified `epoch_length`. fn spec(&self) -> ChainSpec { let mut spec = ChainSpec::foundation(); @@ -43,6 +65,7 @@ impl Manifest { spec } + /// Executes the manifest, returning an `ExecutionResult`. pub fn execute(&self) -> ExecutionResult { let spec = self.spec(); let validator_count = self.config.deposits_for_chain_start; @@ -123,6 +146,11 @@ impl Manifest { } } + /// Checks that the `ExecutionResult` is consistent with the specifications in `self.results`. + /// + /// # Panics + /// + /// Panics with a message if any result does not match exepectations. pub fn assert_result_valid(&self, execution_result: ExecutionResult) { info!("Verifying test results..."); let spec = &execution_result.spec; @@ -157,6 +185,9 @@ impl Manifest { } } +/// Builds an `AttesterSlashing` for some `validator_indices`. +/// +/// Signs the message using a `BeaconChainHarness`. fn build_double_vote_attester_slashing( harness: &BeaconChainHarness, validator_indices: &[u64], @@ -170,6 +201,9 @@ fn build_double_vote_attester_slashing( AttesterSlashingBuilder::double_vote(validator_indices, signer, &harness.spec) } +/// Builds an `ProposerSlashing` for some `validator_index`. +/// +/// Signs the message using a `BeaconChainHarness`. fn build_proposer_slashing(harness: &BeaconChainHarness, validator_index: u64) -> ProposerSlashing { let signer = |validator_index: u64, message: &[u8], epoch: Epoch, domain: u64| { harness diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/results.rs b/beacon_node/beacon_chain/test_harness/src/manifest/results.rs index 844695910..d16c91874 100644 --- a/beacon_node/beacon_chain/test_harness/src/manifest/results.rs +++ b/beacon_node/beacon_chain/test_harness/src/manifest/results.rs @@ -2,6 +2,8 @@ use super::state_check::StateCheck; use super::yaml_helpers::as_usize; use yaml_rust::Yaml; +/// A series of tests to be carried out upon an `ExecutionResult`, returned from executing a +/// `Manifest`. #[derive(Debug)] pub struct Results { pub num_skipped_slots: Option, @@ -9,6 +11,9 @@ pub struct Results { } impl Results { + /// Load from a YAML document. + /// + /// Expects the `results` section of the YAML document. pub fn from_yaml(yaml: &Yaml) -> Self { Self { num_skipped_slots: as_usize(yaml, "num_skipped_slots"), @@ -17,6 +22,7 @@ impl Results { } } +/// Parse the `state_checks` section of the YAML document. fn parse_state_checks(yaml: &Yaml) -> Option> { let mut states = vec![]; diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/state_check.rs b/beacon_node/beacon_chain/test_harness/src/manifest/state_check.rs index 0415d4896..a729811d5 100644 --- a/beacon_node/beacon_chain/test_harness/src/manifest/state_check.rs +++ b/beacon_node/beacon_chain/test_harness/src/manifest/state_check.rs @@ -3,15 +3,24 @@ use log::info; use types::*; use yaml_rust::Yaml; +/// Tests to be conducted upon a `BeaconState` object generated during the execution of a +/// `Manifest`. #[derive(Debug)] pub struct StateCheck { + /// Checked against `beacon_state.slot`. pub slot: Slot, + /// Checked against `beacon_state.validator_registry.len()`. pub num_validators: Option, + /// A list of validator indices which have been penalized. Must be in ascending order. pub slashed_validators: Option>, + /// A list of validator indices which have been exited. Must be in ascending order. pub exited_validators: Option>, } impl StateCheck { + /// Load from a YAML document. + /// + /// Expects the `state_check` section of the YAML document. pub fn from_yaml(yaml: &Yaml) -> Self { Self { slot: Slot::from(as_u64(&yaml, "slot").expect("State must specify slot")), @@ -21,6 +30,11 @@ impl StateCheck { } } + /// Performs all checks against a `BeaconState` + /// + /// # Panics + /// + /// Panics with an error message if any test fails. pub fn assert_valid(&self, state: &BeaconState, spec: &ChainSpec) { let state_epoch = state.slot.epoch(spec.epoch_length); From 48fc7091096eede7bccfb41a542d0aa290b73512 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 3 Mar 2019 15:08:13 +1100 Subject: [PATCH 085/132] Fix failing test --- beacon_node/beacon_chain/test_harness/tests/chain.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/tests/chain.rs b/beacon_node/beacon_chain/test_harness/tests/chain.rs index 1b29a412f..238e567ad 100644 --- a/beacon_node/beacon_chain/test_harness/tests/chain.rs +++ b/beacon_node/beacon_chain/test_harness/tests/chain.rs @@ -41,6 +41,4 @@ fn it_can_produce_past_first_epoch_boundary() { let dump = harness.chain_dump().expect("Chain dump failed."); assert_eq!(dump.len() as u64, blocks + 1); // + 1 for genesis block. - - harness.dump_to_file("/tmp/chaindump.json".to_string(), &dump); } From a29eca57a1c047828ce38026a8b65326396129b3 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 3 Mar 2019 15:12:19 +1100 Subject: [PATCH 086/132] Rename test_harness::manifest to test_case I thing `TestCase` is better than manifest -- a manifest is more of a list of items than a series of steps and checks. Plus it conflicts with a Cargo manifest. --- .../beacon_chain/test_harness/src/bin.rs | 20 +++++++++---------- .../beacon_chain/test_harness/src/lib.rs | 2 +- .../src/{manifest => test_case}/config.rs | 0 .../src/{manifest => test_case}/mod.rs | 16 +++++++-------- .../src/{manifest => test_case}/results.rs | 2 +- .../{manifest => test_case}/state_check.rs | 2 +- .../{manifest => test_case}/yaml_helpers.rs | 0 7 files changed, 21 insertions(+), 21 deletions(-) rename beacon_node/beacon_chain/test_harness/src/{manifest => test_case}/config.rs (100%) rename beacon_node/beacon_chain/test_harness/src/{manifest => test_case}/mod.rs (94%) rename beacon_node/beacon_chain/test_harness/src/{manifest => test_case}/results.rs (98%) rename beacon_node/beacon_chain/test_harness/src/{manifest => test_case}/state_check.rs (99%) rename beacon_node/beacon_chain/test_harness/src/{manifest => test_case}/yaml_helpers.rs (100%) diff --git a/beacon_node/beacon_chain/test_harness/src/bin.rs b/beacon_node/beacon_chain/test_harness/src/bin.rs index 5411efc4a..b4a2cf05e 100644 --- a/beacon_node/beacon_chain/test_harness/src/bin.rs +++ b/beacon_node/beacon_chain/test_harness/src/bin.rs @@ -1,11 +1,11 @@ use clap::{App, Arg}; use env_logger::{Builder, Env}; -use manifest::Manifest; use std::{fs::File, io::prelude::*}; +use test_case::TestCase; use yaml_rust::YamlLoader; mod beacon_chain_harness; -mod manifest; +mod test_case; mod validator_harness; use validator_harness::ValidatorHarness; @@ -14,12 +14,12 @@ fn main() { let matches = App::new("Lighthouse Test Harness Runner") .version("0.0.1") .author("Sigma Prime ") - .about("Runs `test_harness` using a YAML manifest.") + .about("Runs `test_harness` using a YAML test_case.") .arg( Arg::with_name("yaml") .long("yaml") .value_name("FILE") - .help("YAML file manifest.") + .help("YAML file test_case.") .required(true), ) .get_matches(); @@ -37,21 +37,21 @@ fn main() { }; for doc in &docs { - // For each `test_cases` YAML in the document, build a `Manifest`, execute it and - // assert that the execution result matches the manifest description. + // For each `test_cases` YAML in the document, build a `TestCase`, execute it and + // assert that the execution result matches the test_case description. // // In effect, for each `test_case` a new `BeaconChainHarness` is created from genesis - // and a new `BeaconChain` is built as per the manifest. + // and a new `BeaconChain` is built as per the test_case. // - // After the `BeaconChain` has been built out as per the manifest, a dump of all blocks + // After the `BeaconChain` has been built out as per the test_case, a dump of all blocks // and states in the chain is obtained and checked against the `results` specified in // the `test_case`. // // If any of the expectations in the results are not met, the process // panics with a message. for test_case in doc["test_cases"].as_vec().unwrap() { - let manifest = Manifest::from_yaml(test_case); - manifest.assert_result_valid(manifest.execute()) + let test_case = TestCase::from_yaml(test_case); + test_case.assert_result_valid(test_case.execute()) } } } diff --git a/beacon_node/beacon_chain/test_harness/src/lib.rs b/beacon_node/beacon_chain/test_harness/src/lib.rs index a7ada2433..2303dc064 100644 --- a/beacon_node/beacon_chain/test_harness/src/lib.rs +++ b/beacon_node/beacon_chain/test_harness/src/lib.rs @@ -26,7 +26,7 @@ //! ``` mod beacon_chain_harness; -pub mod manifest; +pub mod test_case; mod validator_harness; pub use self::beacon_chain_harness::BeaconChainHarness; diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/config.rs b/beacon_node/beacon_chain/test_harness/src/test_case/config.rs similarity index 100% rename from beacon_node/beacon_chain/test_harness/src/manifest/config.rs rename to beacon_node/beacon_chain/test_harness/src/test_case/config.rs diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs b/beacon_node/beacon_chain/test_harness/src/test_case/mod.rs similarity index 94% rename from beacon_node/beacon_chain/test_harness/src/manifest/mod.rs rename to beacon_node/beacon_chain/test_harness/src/test_case/mod.rs index 7b9c9cb02..f6d8d42e8 100644 --- a/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs +++ b/beacon_node/beacon_chain/test_harness/src/test_case/mod.rs @@ -23,18 +23,18 @@ pub use state_check::StateCheck; /// /// Typical workflow is: /// -/// 1. Instantiate the `Manifest` from YAML: `let manifest = Manifest::from_yaml(&my_yaml);` -/// 2. Execute the manifest: `let result = manifest.execute();` -/// 3. Test the results against the manifest: `manifest.assert_result_valid(result);` +/// 1. Instantiate the `TestCase` from YAML: `let test_case = TestCase::from_yaml(&my_yaml);` +/// 2. Execute the test_case: `let result = test_case.execute();` +/// 3. Test the results against the test_case: `test_case.assert_result_valid(result);` #[derive(Debug)] -pub struct Manifest { +pub struct TestCase { /// Defines the execution. pub config: Config, /// Defines tests to run against the execution result. pub results: Results, } -/// The result of executing a `Manifest`. +/// The result of executing a `TestCase`. /// pub struct ExecutionResult { /// The canonical beacon chain generated from the execution. @@ -43,8 +43,8 @@ pub struct ExecutionResult { pub spec: ChainSpec, } -impl Manifest { - /// Load the manifest from a YAML document. +impl TestCase { + /// Load the test case from a YAML document. pub fn from_yaml(test_case: &Yaml) -> Self { Self { results: Results::from_yaml(&test_case["results"]), @@ -65,7 +65,7 @@ impl Manifest { spec } - /// Executes the manifest, returning an `ExecutionResult`. + /// Executes the test case, returning an `ExecutionResult`. pub fn execute(&self) -> ExecutionResult { let spec = self.spec(); let validator_count = self.config.deposits_for_chain_start; diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/results.rs b/beacon_node/beacon_chain/test_harness/src/test_case/results.rs similarity index 98% rename from beacon_node/beacon_chain/test_harness/src/manifest/results.rs rename to beacon_node/beacon_chain/test_harness/src/test_case/results.rs index d16c91874..596418c0f 100644 --- a/beacon_node/beacon_chain/test_harness/src/manifest/results.rs +++ b/beacon_node/beacon_chain/test_harness/src/test_case/results.rs @@ -3,7 +3,7 @@ use super::yaml_helpers::as_usize; use yaml_rust::Yaml; /// A series of tests to be carried out upon an `ExecutionResult`, returned from executing a -/// `Manifest`. +/// `TestCase`. #[derive(Debug)] pub struct Results { pub num_skipped_slots: Option, diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/state_check.rs b/beacon_node/beacon_chain/test_harness/src/test_case/state_check.rs similarity index 99% rename from beacon_node/beacon_chain/test_harness/src/manifest/state_check.rs rename to beacon_node/beacon_chain/test_harness/src/test_case/state_check.rs index a729811d5..90c622894 100644 --- a/beacon_node/beacon_chain/test_harness/src/manifest/state_check.rs +++ b/beacon_node/beacon_chain/test_harness/src/test_case/state_check.rs @@ -4,7 +4,7 @@ use types::*; use yaml_rust::Yaml; /// Tests to be conducted upon a `BeaconState` object generated during the execution of a -/// `Manifest`. +/// `TestCase`. #[derive(Debug)] pub struct StateCheck { /// Checked against `beacon_state.slot`. diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/yaml_helpers.rs b/beacon_node/beacon_chain/test_harness/src/test_case/yaml_helpers.rs similarity index 100% rename from beacon_node/beacon_chain/test_harness/src/manifest/yaml_helpers.rs rename to beacon_node/beacon_chain/test_harness/src/test_case/yaml_helpers.rs From ec0e13b764b12fe10390c4b84be7f8f31fede8d0 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 3 Mar 2019 15:32:44 +1100 Subject: [PATCH 087/132] Add comments to new functions --- .../block_processable/verify_slashable_attestation.rs | 2 ++ eth2/types/src/attester_slashing/builder.rs | 11 +++++++++++ eth2/types/src/beacon_state.rs | 6 ++++++ eth2/types/src/proposer_slashing/builder.rs | 11 +++++++++++ eth2/types/src/validator.rs | 4 +++- 5 files changed, 33 insertions(+), 1 deletion(-) diff --git a/eth2/state_processing/src/block_processable/verify_slashable_attestation.rs b/eth2/state_processing/src/block_processable/verify_slashable_attestation.rs index 35ad67df0..a406af24e 100644 --- a/eth2/state_processing/src/block_processable/verify_slashable_attestation.rs +++ b/eth2/state_processing/src/block_processable/verify_slashable_attestation.rs @@ -9,6 +9,8 @@ macro_rules! ensure { }; } +/// Returns `Ok(())` if some `AttesterSlashing` is valid to be included in some `BeaconState`, +/// otherwise returns an `Err`. pub fn verify_slashable_attestation( state: &mut BeaconState, attester_slashing: &AttesterSlashing, diff --git a/eth2/types/src/attester_slashing/builder.rs b/eth2/types/src/attester_slashing/builder.rs index e53706192..ed203d6e1 100644 --- a/eth2/types/src/attester_slashing/builder.rs +++ b/eth2/types/src/attester_slashing/builder.rs @@ -1,9 +1,20 @@ use crate::*; use ssz::TreeHash; +/// Builds an `AttesterSlashing`. pub struct AttesterSlashingBuilder(); impl AttesterSlashingBuilder { + /// Builds an `AttesterSlashing` that is a double vote. + /// + /// The `signer` function is used to sign the double-vote and accepts: + /// + /// - `validator_index: u64` + /// - `message: &[u8]` + /// - `epoch: Epoch` + /// - `domain: u64` + /// + /// Where domain is a domain "constant" (e.g., `spec.domain_attestation`). pub fn double_vote( validator_indices: &[u64], signer: F, diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 932233445..a9e2f2673 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -1140,6 +1140,9 @@ impl BeaconState { ) } + /// Verify ``bitfield`` against the ``committee_size``. + /// + /// Spec v0.2.0 pub fn verify_bitfield(&self, bitfield: &Bitfield, committee_size: usize) -> bool { if bitfield.num_bytes() != ((committee_size + 7) / 8) { return false; @@ -1159,6 +1162,9 @@ impl BeaconState { true } + /// Verify validity of ``slashable_attestation`` fields. + /// + /// Spec v0.2.0 pub fn verify_slashable_attestation( &self, slashable_attestation: &SlashableAttestation, diff --git a/eth2/types/src/proposer_slashing/builder.rs b/eth2/types/src/proposer_slashing/builder.rs index 363155a14..7923ff74d 100644 --- a/eth2/types/src/proposer_slashing/builder.rs +++ b/eth2/types/src/proposer_slashing/builder.rs @@ -1,9 +1,20 @@ use crate::*; use ssz::TreeHash; +/// Builds a `ProposerSlashing`. pub struct ProposerSlashingBuilder(); impl ProposerSlashingBuilder { + /// Builds a `ProposerSlashing` that is a double vote. + /// + /// The `signer` function is used to sign the double-vote and accepts: + /// + /// - `validator_index: u64` + /// - `message: &[u8]` + /// - `epoch: Epoch` + /// - `domain: u64` + /// + /// Where domain is a domain "constant" (e.g., `spec.domain_attestation`). pub fn double_vote(proposer_index: u64, signer: F, spec: &ChainSpec) -> ProposerSlashing where F: Fn(u64, &[u8], Epoch, u64) -> Signature, diff --git a/eth2/types/src/validator.rs b/eth2/types/src/validator.rs index bc8d467ec..587a48a1f 100644 --- a/eth2/types/src/validator.rs +++ b/eth2/types/src/validator.rs @@ -54,15 +54,17 @@ pub struct Validator { } impl Validator { - /// This predicate indicates if the validator represented by this record is considered "active" at `slot`. + /// Returns `true` if the validator is considered active at some epoch. pub fn is_active_at(&self, epoch: Epoch) -> bool { self.activation_epoch <= epoch && epoch < self.exit_epoch } + /// Returns `true` if the validator is considered exited at some epoch. pub fn is_exited_at(&self, epoch: Epoch) -> bool { self.exit_epoch <= epoch } + /// Returns `true` if the validator is considered penalized at some epoch. pub fn is_penalized_at(&self, epoch: Epoch) -> bool { self.penalized_epoch <= epoch } From 58002f68e1952dc83dbe4a5a7e4605e13c72138d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 3 Mar 2019 16:14:40 +1100 Subject: [PATCH 088/132] Move test_harness yaml file --- .../{examples/chain.yaml => specs/validator_registry.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename beacon_node/beacon_chain/test_harness/{examples/chain.yaml => specs/validator_registry.yaml} (100%) diff --git a/beacon_node/beacon_chain/test_harness/examples/chain.yaml b/beacon_node/beacon_chain/test_harness/specs/validator_registry.yaml similarity index 100% rename from beacon_node/beacon_chain/test_harness/examples/chain.yaml rename to beacon_node/beacon_chain/test_harness/specs/validator_registry.yaml From 697d1ef62657f0825246b7ef3e404f5efa4550c5 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 3 Mar 2019 16:15:00 +1100 Subject: [PATCH 089/132] Add CLI option for log-level to test_harness --- beacon_node/beacon_chain/test_harness/src/bin.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/test_harness/src/bin.rs b/beacon_node/beacon_chain/test_harness/src/bin.rs index b4a2cf05e..283cb0dfa 100644 --- a/beacon_node/beacon_chain/test_harness/src/bin.rs +++ b/beacon_node/beacon_chain/test_harness/src/bin.rs @@ -22,9 +22,20 @@ fn main() { .help("YAML file test_case.") .required(true), ) + .arg( + Arg::with_name("log") + .long("log-level") + .value_name("LOG_LEVEL") + .help("Logging level.") + .possible_values(&["error", "warn", "info", "debug", "trace"]) + .default_value("debug") + .required(true), + ) .get_matches(); - Builder::from_env(Env::default().default_filter_or("debug")).init(); + if let Some(log_level) = matches.value_of("log") { + Builder::from_env(Env::default().default_filter_or(log_level)).init(); + } if let Some(yaml_file) = matches.value_of("yaml") { let docs = { From 9d77f2b1a8b7971e3d5ab3928ae7a85ccbd45ff6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 3 Mar 2019 16:38:25 +1100 Subject: [PATCH 090/132] Add README to test_harness --- .../beacon_chain/test_harness/README.md | 150 ++++++++++++++++++ .../beacon_chain/test_harness/src/lib.rs | 2 +- 2 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 beacon_node/beacon_chain/test_harness/README.md diff --git a/beacon_node/beacon_chain/test_harness/README.md b/beacon_node/beacon_chain/test_harness/README.md new file mode 100644 index 000000000..12cbbe062 --- /dev/null +++ b/beacon_node/beacon_chain/test_harness/README.md @@ -0,0 +1,150 @@ +# Test Harness + +Provides a testing environment for the `BeaconChain`, `Attester` and `BlockProposer` objects. + +This environment bypasses networking and client run-times and connects the `Attester` and `Proposer` +directly to the `BeaconChain` via an `Arc`. + +The `BeaconChainHarness` contains a single `BeaconChain` instance and many `ValidatorHarness` +instances. All of the `ValidatorHarness` instances work to advance the `BeaconChain` by +producing blocks and attestations. + +The crate consists of a library and binary, examples for using both are +described below. + +## YAML + +Both the library and the binary are capable of parsing tests from a YAML file, +in fact this is the sole purpose of the binary. + +You can find YAML test cases [here](specs/). An example is included below: + +```yaml +title: Validator Registry Tests +summary: Tests deposit and slashing effects on validator registry. +test_suite: validator_registry +fork: tchaikovsky +version: 1.0 +test_cases: + - config: + epoch_length: 64 + deposits_for_chain_start: 1000 + num_slots: 64 + skip_slots: [2, 3] + deposits: + # At slot 1, create a new validator deposit of 32 ETH. + - slot: 1 + amount: 32 + # Trigger more deposits... + - slot: 3 + amount: 32 + - slot: 5 + amount: 32 + proposer_slashings: + # At slot 2, trigger a proposer slashing for validator #42. + - slot: 2 + validator_index: 42 + # Trigger another slashing... + - slot: 8 + validator_index: 13 + attester_slashings: + # At slot 2, trigger an attester slashing for validators #11 and #12. + - slot: 2 + validator_indices: [11, 12] + # Trigger another slashing... + - slot: 5 + validator_indices: [14] + results: + num_skipped_slots: 2 + states: + - slot: 63 + num_validators: 1003 + slashed_validators: [11, 12, 13, 14, 42] + exited_validators: [] + +``` + +Thanks to [prsym](http://github.com/prysmaticlabs/prysm) for coming up with the +base YAML format. + +### Notes + +Wherever `slot` is used, it is actually the "slot height", or slots since +genesis. This allows the tests to disregard the `GENESIS_EPOCH`. + +### Differences from Prysmatic's format + +1. The detail for `deposits`, `proposer_slashings` and `attester_slashings` is + ommitted from the test specification. It assumed they should be valid + objects. +2. There is a `states` list in `results` that runs checks against any state + specified by a `slot` number. This is in contrast to the variables in + `results` that assume the last (highest) state should be inspected. + +#### Reasoning + +Respective reasonings for above changes: + +1. This removes the concerns of the actual object structure from the tests. + This allows for more variation in the deposits/slashings objects without + needing to update the tests. Also, it makes it makes it easier to create + tests. +2. This gives more fine-grained control over the tests. It allows for checking + that certain events happened at certain times whilst making the tests only + slightly more verbose. + +_Notes: it may be useful to add an extra field to each slashing type to +indicate if it should be valid or not. It also may be useful to add an option +for double-vote/surround-vote attester slashings. The `amount` field was left +on `deposits` as it changes the behaviour of state significantly._ + +## Binary Usage Example + +Follow these steps to run as a binary: + +1. Navigate to the root of this crate (where this readme is located) +2. Run `$ cargo run --release -- --yaml examples/validator_registry.yaml` + +_Note: the `--release` flag builds the binary without all the debugging +instrumentation. The test is much faster built using `--release`. As is +customary in cargo, the flags before `--` are passed to cargo and the flags +after are passed to the binary._ + +### CLI Options + +``` +Lighthouse Test Harness Runner 0.0.1 +Sigma Prime +Runs `test_harness` using a YAML test_case. + +USAGE: + test_harness --log-level --yaml + +FLAGS: + -h, --help Prints help information + -V, --version Prints version information + +OPTIONS: + --log-level Logging level. [default: debug] [possible values: error, warn, info, debug, trace] + --yaml YAML file test_case. +``` + + +## Library Usage Example + +```rust +use test_harness::BeaconChainHarness; +use types::ChainSpec; + +let validator_count = 8; +let spec = ChainSpec::few_validators(); + +let mut harness = BeaconChainHarness::new(spec, validator_count); + +harness.advance_chain_with_block(); + +let chain = harness.chain_dump().unwrap(); + +// One block should have been built on top of the genesis block. +assert_eq!(chain.len(), 2); +``` diff --git a/beacon_node/beacon_chain/test_harness/src/lib.rs b/beacon_node/beacon_chain/test_harness/src/lib.rs index 2303dc064..0703fd4a5 100644 --- a/beacon_node/beacon_chain/test_harness/src/lib.rs +++ b/beacon_node/beacon_chain/test_harness/src/lib.rs @@ -1,6 +1,6 @@ //! Provides a testing environment for the `BeaconChain`, `Attester` and `BlockProposer` objects. //! -//! This environment bypasses networking client runtimes and connects the `Attester` and `Proposer` +//! This environment bypasses networking and client run-times and connects the `Attester` and `Proposer` //! directly to the `BeaconChain` via an `Arc`. //! //! The `BeaconChainHarness` contains a single `BeaconChain` instance and many `ValidatorHarness` From c28c07c17d40f7c5f8a647ee40f3bf14e535a075 Mon Sep 17 00:00:00 2001 From: thojest Date: Sun, 3 Mar 2019 14:15:00 +0100 Subject: [PATCH 091/132] validator_client: added default_value for spec; used unreachable macro for custom spec (lighthouse-252) --- validator_client/src/main.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index f81c3e40c..4344bc16e 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -50,7 +50,8 @@ fn main() { .short("s") .help("Configuration of Beacon Chain") .takes_value(true) - .possible_values(&["foundation", "few_validators"]), + .possible_values(&["foundation", "few_validators"]) + .default_value("foundation"), ) .get_matches(); @@ -77,11 +78,8 @@ fn main() { match spec_str { "foundation" => config.spec = ChainSpec::foundation(), "few_validators" => config.spec = ChainSpec::few_validators(), - // Should be impossible - _ => { - error!(log, "Invalid ChainSpec defined"; "spec" => spec_str); - return; - } + // Should be impossible due to clap's `possible_values(..)` function. + _ => unreachable!(), }; } From 3aaa3ea0243bb355fc9c2ead3cb6a8e939c4eec1 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 4 Mar 2019 09:30:09 +1100 Subject: [PATCH 092/132] Use clearer types in test_harness::Config --- .../test_harness/src/test_case/config.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/src/test_case/config.rs b/beacon_node/beacon_chain/test_harness/src/test_case/config.rs index 8c88ee5d1..d08e7fe40 100644 --- a/beacon_node/beacon_chain/test_harness/src/test_case/config.rs +++ b/beacon_node/beacon_chain/test_harness/src/test_case/config.rs @@ -3,9 +3,12 @@ use bls::create_proof_of_possession; use types::*; use yaml_rust::Yaml; -pub type DepositTuple = (u64, Deposit, Keypair); -pub type ProposerSlashingTuple = (u64, u64); -pub type AttesterSlashingTuple = (u64, Vec); +pub type ValidatorIndex = u64; +pub type ValidatorIndices = Vec; + +pub type DepositTuple = (SlotHeight, Deposit, Keypair); +pub type ProposerSlashingTuple = (SlotHeight, ValidatorIndex); +pub type AttesterSlashingTuple = (SlotHeight, ValidatorIndices); /// Defines the execution of a `BeaconStateHarness` across a series of slots. #[derive(Debug)] @@ -53,7 +56,7 @@ fn parse_attester_slashings(yaml: &Yaml) -> Option> { let validator_indices = as_vec_u64(slashing, "validator_indices") .expect("Incomplete attester_slashing (validator_indices)"); - slashings.push((slot, validator_indices)); + slashings.push((SlotHeight::from(slot), validator_indices)); } Some(slashings) @@ -68,7 +71,7 @@ fn parse_proposer_slashings(yaml: &Yaml) -> Option> { let validator_index = as_u64(slashing, "validator_index") .expect("Incomplete proposer slashing (validator_index)"); - slashings.push((slot, validator_index)); + slashings.push((SlotHeight::from(slot), validator_index)); } Some(slashings) @@ -102,7 +105,7 @@ fn parse_deposits(yaml: &Yaml) -> Option> { }, }; - deposits.push((slot, deposit, keypair)); + deposits.push((SlotHeight::from(slot), deposit, keypair)); } Some(deposits) From bc4acd9a5cb1fc31b1a480ca701721de0b3f8daf Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 4 Mar 2019 09:43:19 +1100 Subject: [PATCH 093/132] Re-work module structure in test_harness Obeys the standard where structs live in files with the same name --- .../test_harness/src/{test_case/mod.rs => test_case.rs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename beacon_node/beacon_chain/test_harness/src/{test_case/mod.rs => test_case.rs} (100%) diff --git a/beacon_node/beacon_chain/test_harness/src/test_case/mod.rs b/beacon_node/beacon_chain/test_harness/src/test_case.rs similarity index 100% rename from beacon_node/beacon_chain/test_harness/src/test_case/mod.rs rename to beacon_node/beacon_chain/test_harness/src/test_case.rs From f4d8b41a0919e6c68c0b0b13fcc4a340c62269fe Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 4 Mar 2019 12:20:59 +1100 Subject: [PATCH 094/132] Add Exit inclusion queue to BeaconChain --- beacon_node/beacon_chain/src/beacon_chain.rs | 66 ++++++++++++++++++-- 1 file changed, 62 insertions(+), 4 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index e6fd2a134..81d17f024 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -65,6 +65,7 @@ pub struct BeaconChain { pub slot_clock: U, pub attestation_aggregator: RwLock, pub deposits_for_inclusion: RwLock>, + pub exits_for_inclusion: RwLock>, pub proposer_slashings_for_inclusion: RwLock>, pub attester_slashings_for_inclusion: RwLock>, canonical_head: RwLock, @@ -134,6 +135,7 @@ where slot_clock, attestation_aggregator, deposits_for_inclusion: RwLock::new(vec![]), + exits_for_inclusion: RwLock::new(vec![]), proposer_slashings_for_inclusion: RwLock::new(vec![]), attester_slashings_for_inclusion: RwLock::new(vec![]), state: RwLock::new(genesis_state), @@ -370,13 +372,17 @@ where /// Accept some deposit and queue it for inclusion in an appropriate block. pub fn receive_deposit_for_inclusion(&self, deposit: Deposit) { - // TODO: deposits are not check for validity; check them. + // TODO: deposits are not checked for validity; check them. + // + // https://github.com/sigp/lighthouse/issues/276 self.deposits_for_inclusion.write().push(deposit); } /// Return a vec of deposits suitable for inclusion in some block. pub fn get_deposits_for_block(&self) -> Vec { // TODO: deposits are indiscriminately included; check them for validity. + // + // https://github.com/sigp/lighthouse/issues/275 self.deposits_for_inclusion.read().clone() } @@ -386,6 +392,8 @@ where /// This ensures that `Deposits` are not included twice in successive blocks. pub fn set_deposits_as_included(&self, included_deposits: &[Deposit]) { // TODO: method does not take forks into account; consider this. + // + // https://github.com/sigp/lighthouse/issues/275 let mut indices_to_delete = vec![]; for included in included_deposits { @@ -402,9 +410,49 @@ where } } + /// Accept some exit and queue it for inclusion in an appropriate block. + pub fn receive_exit_for_inclusion(&self, exit: Exit) { + // TODO: exits are not checked for validity; check them. + // + // https://github.com/sigp/lighthouse/issues/276 + self.exits_for_inclusion.write().push(exit); + } + + /// Return a vec of exits suitable for inclusion in some block. + pub fn get_exits_for_block(&self) -> Vec { + // TODO: exits are indiscriminately included; check them for validity. + // + // https://github.com/sigp/lighthouse/issues/275 + self.exits_for_inclusion.read().clone() + } + + /// Takes a list of `Deposits` that were included in recent blocks and removes them from the + /// inclusion queue. + /// + /// This ensures that `Deposits` are not included twice in successive blocks. + pub fn set_exits_as_included(&self, included_exits: &[Exit]) { + // TODO: method does not take forks into account; consider this. + let mut indices_to_delete = vec![]; + + for included in included_exits { + for (i, for_inclusion) in self.exits_for_inclusion.read().iter().enumerate() { + if included == for_inclusion { + indices_to_delete.push(i); + } + } + } + + let exits_for_inclusion = &mut self.exits_for_inclusion.write(); + for i in indices_to_delete { + exits_for_inclusion.remove(i); + } + } + /// Accept some proposer slashing and queue it for inclusion in an appropriate block. pub fn receive_proposer_slashing_for_inclusion(&self, proposer_slashing: ProposerSlashing) { - // TODO: proposer_slashings are not check for validity; check them. + // TODO: proposer_slashings are not checked for validity; check them. + // + // https://github.com/sigp/lighthouse/issues/276 self.proposer_slashings_for_inclusion .write() .push(proposer_slashing); @@ -413,6 +461,8 @@ where /// Return a vec of proposer slashings suitable for inclusion in some block. pub fn get_proposer_slashings_for_block(&self) -> Vec { // TODO: proposer_slashings are indiscriminately included; check them for validity. + // + // https://github.com/sigp/lighthouse/issues/275 self.proposer_slashings_for_inclusion.read().clone() } @@ -425,6 +475,8 @@ where included_proposer_slashings: &[ProposerSlashing], ) { // TODO: method does not take forks into account; consider this. + // + // https://github.com/sigp/lighthouse/issues/275 let mut indices_to_delete = vec![]; for included in included_proposer_slashings { @@ -448,7 +500,9 @@ where /// Accept some attester slashing and queue it for inclusion in an appropriate block. pub fn receive_attester_slashing_for_inclusion(&self, attester_slashing: AttesterSlashing) { - // TODO: attester_slashings are not check for validity; check them. + // TODO: attester_slashings are not checked for validity; check them. + // + // https://github.com/sigp/lighthouse/issues/276 self.attester_slashings_for_inclusion .write() .push(attester_slashing); @@ -457,6 +511,8 @@ where /// Return a vec of attester slashings suitable for inclusion in some block. pub fn get_attester_slashings_for_block(&self) -> Vec { // TODO: attester_slashings are indiscriminately included; check them for validity. + // + // https://github.com/sigp/lighthouse/issues/275 self.attester_slashings_for_inclusion.read().clone() } @@ -469,6 +525,8 @@ where included_attester_slashings: &[AttesterSlashing], ) { // TODO: method does not take forks into account; consider this. + // + // https://github.com/sigp/lighthouse/issues/275 let mut indices_to_delete = vec![]; for included in included_attester_slashings { @@ -678,7 +736,7 @@ where attester_slashings: self.get_attester_slashings_for_block(), attestations, deposits: self.get_deposits_for_block(), - exits: vec![], + exits: self.get_exits_for_block(), }, }; From ef006bfb2cc30bc5906f4e1bb7210847dca42e2c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 4 Mar 2019 12:21:24 +1100 Subject: [PATCH 095/132] Add Exit support to test_harness --- .../specs/validator_registry.yaml | 5 +++ .../test_harness/src/beacon_chain_harness.rs | 9 ++++ .../test_harness/src/test_case.rs | 42 +++++++++++++++++++ .../test_harness/src/test_case/config.rs | 19 +++++++++ .../test_harness/src/test_case/state_check.rs | 32 +++++++++++++- eth2/types/src/validator.rs | 5 +++ 6 files changed, 111 insertions(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/test_harness/specs/validator_registry.yaml b/beacon_node/beacon_chain/test_harness/specs/validator_registry.yaml index b7fdda9bf..5851d6d12 100644 --- a/beacon_node/beacon_chain/test_harness/specs/validator_registry.yaml +++ b/beacon_node/beacon_chain/test_harness/specs/validator_registry.yaml @@ -18,6 +18,10 @@ test_cases: amount: 32 - slot: 5 amount: 32 + exits: + # At slot 10, submit an exit for validator #50. + - slot: 10 + validator_index: 50 proposer_slashings: # At slot 2, trigger a proposer slashing for validator #42. - slot: 2 @@ -39,4 +43,5 @@ test_cases: num_validators: 1003 slashed_validators: [11, 12, 13, 14, 42] exited_validators: [] + exit_initiated_validators: [50] diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index 2f375f7fa..40672c11a 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -280,6 +280,15 @@ impl BeaconChainHarness { } } + /// Submit an exit to the `BeaconChain` for inclusion in some block. + /// + /// Note: the `ValidatorHarness` for this validator continues to exist. Once it is exited it + /// will stop receiving duties from the beacon chain and just do nothing when prompted to + /// produce/attest. + pub fn add_exit(&mut self, exit: Exit) { + self.beacon_chain.receive_exit_for_inclusion(exit); + } + /// Submit a proposer slashing to the `BeaconChain` for inclusion in some block. pub fn add_proposer_slashing(&mut self, proposer_slashing: ProposerSlashing) { self.beacon_chain diff --git a/beacon_node/beacon_chain/test_harness/src/test_case.rs b/beacon_node/beacon_chain/test_harness/src/test_case.rs index f6d8d42e8..cdac7d3cc 100644 --- a/beacon_node/beacon_chain/test_harness/src/test_case.rs +++ b/beacon_node/beacon_chain/test_harness/src/test_case.rs @@ -4,6 +4,7 @@ use crate::beacon_chain_harness::BeaconChainHarness; use beacon_chain::CheckPoint; use log::{info, warn}; +use ssz::TreeHash; use types::*; use types::{ attester_slashing::AttesterSlashingBuilder, proposer_slashing::ProposerSlashingBuilder, @@ -121,6 +122,20 @@ impl TestCase { } } + // Feed exits to the BeaconChain. + if let Some(ref exits) = self.config.exits { + for (slot, validator_index) in exits { + if *slot == slot_height { + info!( + "Including exit at slot height {} for validator {}.", + slot_height, validator_index + ); + let exit = build_exit(&harness, *validator_index); + harness.add_exit(exit); + } + } + } + // Build a block or skip a slot. match self.config.skip_slots { Some(ref skip_slots) if skip_slots.contains(&slot_height) => { @@ -185,6 +200,33 @@ impl TestCase { } } +fn build_exit(harness: &BeaconChainHarness, validator_index: u64) -> Exit { + let epoch = harness + .beacon_chain + .state + .read() + .current_epoch(&harness.spec); + + let mut exit = Exit { + epoch, + validator_index, + signature: Signature::empty_signature(), + }; + + let message = exit.hash_tree_root(); + + exit.signature = harness + .validator_sign( + validator_index as usize, + &message[..], + epoch, + harness.spec.domain_exit, + ) + .expect("Unable to sign Exit"); + + exit +} + /// Builds an `AttesterSlashing` for some `validator_indices`. /// /// Signs the message using a `BeaconChainHarness`. diff --git a/beacon_node/beacon_chain/test_harness/src/test_case/config.rs b/beacon_node/beacon_chain/test_harness/src/test_case/config.rs index d08e7fe40..19bce8e76 100644 --- a/beacon_node/beacon_chain/test_harness/src/test_case/config.rs +++ b/beacon_node/beacon_chain/test_harness/src/test_case/config.rs @@ -7,6 +7,7 @@ pub type ValidatorIndex = u64; pub type ValidatorIndices = Vec; pub type DepositTuple = (SlotHeight, Deposit, Keypair); +pub type ExitTuple = (SlotHeight, ValidatorIndex); pub type ProposerSlashingTuple = (SlotHeight, ValidatorIndex); pub type AttesterSlashingTuple = (SlotHeight, ValidatorIndices); @@ -27,6 +28,8 @@ pub struct Config { pub proposer_slashings: Option>, /// Attester slashings to be including during execution. pub attester_slashings: Option>, + /// Exits to be including during execution. + pub exits: Option>, } impl Config { @@ -43,10 +46,26 @@ impl Config { deposits: parse_deposits(&yaml), proposer_slashings: parse_proposer_slashings(&yaml), attester_slashings: parse_attester_slashings(&yaml), + exits: parse_exits(&yaml), } } } +/// Parse the `attester_slashings` section of the YAML document. +fn parse_exits(yaml: &Yaml) -> Option> { + let mut tuples = vec![]; + + for exit in yaml["exits"].as_vec()? { + let slot = as_u64(exit, "slot").expect("Incomplete exit (slot)"); + let validator_index = + as_u64(exit, "validator_index").expect("Incomplete exit (validator_index)"); + + tuples.push((SlotHeight::from(slot), validator_index)); + } + + Some(tuples) +} + /// Parse the `attester_slashings` section of the YAML document. fn parse_attester_slashings(yaml: &Yaml) -> Option> { let mut slashings = vec![]; diff --git a/beacon_node/beacon_chain/test_harness/src/test_case/state_check.rs b/beacon_node/beacon_chain/test_harness/src/test_case/state_check.rs index 90c622894..c44992a97 100644 --- a/beacon_node/beacon_chain/test_harness/src/test_case/state_check.rs +++ b/beacon_node/beacon_chain/test_harness/src/test_case/state_check.rs @@ -13,8 +13,10 @@ pub struct StateCheck { pub num_validators: Option, /// A list of validator indices which have been penalized. Must be in ascending order. pub slashed_validators: Option>, - /// A list of validator indices which have been exited. Must be in ascending order. + /// A list of validator indices which have been fully exited. Must be in ascending order. pub exited_validators: Option>, + /// A list of validator indices which have had an exit initiated. Must be in ascending order. + pub exit_initiated_validators: Option>, } impl StateCheck { @@ -27,6 +29,7 @@ impl StateCheck { num_validators: as_usize(&yaml, "num_validators"), slashed_validators: as_vec_u64(&yaml, "slashed_validators"), exited_validators: as_vec_u64(&yaml, "exited_validators"), + exit_initiated_validators: as_vec_u64(&yaml, "exit_initiated_validators"), } } @@ -40,6 +43,7 @@ impl StateCheck { info!("Running state check for slot height {}.", self.slot); + // Check the state slot. assert_eq!( self.slot, state.slot - spec.genesis_epoch.start_slot(spec.epoch_length), @@ -55,6 +59,7 @@ impl StateCheck { info!("OK: num_validators = {}.", num_validators); } + // Check for slashed validators. if let Some(ref slashed_validators) = self.slashed_validators { let actually_slashed_validators: Vec = state .validator_registry @@ -75,6 +80,7 @@ impl StateCheck { info!("OK: slashed_validators = {:?}.", slashed_validators); } + // Check for exited validators. if let Some(ref exited_validators) = self.exited_validators { let actually_exited_validators: Vec = state .validator_registry @@ -94,5 +100,29 @@ impl StateCheck { ); info!("OK: exited_validators = {:?}.", exited_validators); } + + // Check for validators that have initiated exit. + if let Some(ref exit_initiated_validators) = self.exit_initiated_validators { + let actual: Vec = state + .validator_registry + .iter() + .enumerate() + .filter_map(|(i, validator)| { + if validator.has_initiated_exit() { + Some(i as u64) + } else { + None + } + }) + .collect(); + assert_eq!( + actual, *exit_initiated_validators, + "Exit initiated validators != expected." + ); + info!( + "OK: exit_initiated_validators = {:?}.", + exit_initiated_validators + ); + } } } diff --git a/eth2/types/src/validator.rs b/eth2/types/src/validator.rs index 587a48a1f..42a2b31f2 100644 --- a/eth2/types/src/validator.rs +++ b/eth2/types/src/validator.rs @@ -68,6 +68,11 @@ impl Validator { pub fn is_penalized_at(&self, epoch: Epoch) -> bool { self.penalized_epoch <= epoch } + + /// Returns `true` if the validator is considered penalized at some epoch. + pub fn has_initiated_exit(&self) -> bool { + self.status_flags == Some(StatusFlags::InitiatedExit) + } } impl Default for Validator { From 4a57aec4723f7ddf9a8f8afad6fcaf70b4c31318 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 4 Mar 2019 14:24:29 +1100 Subject: [PATCH 096/132] Unfinished progress --- eth2/types/src/lib.rs | 2 ++ eth2/types/src/proposal.rs | 46 +++++++++++++++++++++++++++++ eth2/types/src/proposer_slashing.rs | 12 ++++---- 3 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 eth2/types/src/proposal.rs diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index 4f196b9e9..ff73a6ea6 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -19,6 +19,7 @@ pub mod exit; pub mod fork; pub mod free_attestation; pub mod pending_attestation; +pub mod proposal; pub mod proposal_signed_data; pub mod proposer_slashing; pub mod readers; @@ -57,6 +58,7 @@ pub use crate::exit::Exit; pub use crate::fork::Fork; pub use crate::free_attestation::FreeAttestation; pub use crate::pending_attestation::PendingAttestation; +pub use crate::proposal::Proposal; pub use crate::proposal_signed_data::ProposalSignedData; pub use crate::proposer_slashing::ProposerSlashing; pub use crate::slashable_attestation::SlashableAttestation; diff --git a/eth2/types/src/proposal.rs b/eth2/types/src/proposal.rs new file mode 100644 index 000000000..ac92dbce8 --- /dev/null +++ b/eth2/types/src/proposal.rs @@ -0,0 +1,46 @@ +use crate::test_utils::TestRandom; +use crate::{Hash256, Slot}; +use bls::Signature; +use rand::RngCore; +use serde_derive::Serialize; +use ssz_derive::{Decode, Encode, TreeHash}; +use test_random_derive::TestRandom; + +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] +pub struct Proposal { + pub slot: Slot, + /// Shard number (spec.beacon_chain_shard_number for beacon chain) + pub shard: u64, + pub block_root: Hash256, + pub signature: Signature, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::{ssz_encode, Decodable, TreeHash}; + + #[test] + pub fn test_ssz_round_trip() { + let mut rng = XorShiftRng::from_seed([42; 16]); + let original = Proposal::random_for_test(&mut rng); + + let bytes = ssz_encode(&original); + let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); + + assert_eq!(original, decoded); + } + + #[test] + pub fn test_hash_tree_root_internal() { + let mut rng = XorShiftRng::from_seed([42; 16]); + let original = Proposal::random_for_test(&mut rng); + + let result = original.hash_tree_root_internal(); + + assert_eq!(result.len(), 32); + // TODO: Add further tests + // https://github.com/sigp/lighthouse/issues/170 + } +} diff --git a/eth2/types/src/proposer_slashing.rs b/eth2/types/src/proposer_slashing.rs index ea30d46ec..3b6f57c40 100644 --- a/eth2/types/src/proposer_slashing.rs +++ b/eth2/types/src/proposer_slashing.rs @@ -1,6 +1,5 @@ -use super::ProposalSignedData; +use super::Proposal; use crate::test_utils::TestRandom; -use bls::Signature; use rand::RngCore; use serde_derive::Serialize; use ssz_derive::{Decode, Encode, TreeHash}; @@ -10,13 +9,14 @@ mod builder; pub use builder::ProposerSlashingBuilder; +/// Two conflicting proposals from the same proposer (validator). +/// +/// Spec v0.4.0 #[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] pub struct ProposerSlashing { pub proposer_index: u64, - pub proposal_data_1: ProposalSignedData, - pub proposal_signature_1: Signature, - pub proposal_data_2: ProposalSignedData, - pub proposal_signature_2: Signature, + pub proposer_1: Proposal, + pub proposer_2: Proposal, } #[cfg(test)] From 94122a73345fe7f5bca23fdec981910ef72c7415 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 4 Mar 2019 15:47:48 +1100 Subject: [PATCH 097/132] Add SignedRoot methods --- eth2/types/src/proposal.rs | 35 +++++++++++++-- eth2/utils/ssz/src/lib.rs | 2 + eth2/utils/ssz/src/signed_root.rs | 5 +++ eth2/utils/ssz_derive/src/lib.rs | 75 +++++++++++++++++++++++++++++++ 4 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 eth2/utils/ssz/src/signed_root.rs diff --git a/eth2/types/src/proposal.rs b/eth2/types/src/proposal.rs index ac92dbce8..af69731cb 100644 --- a/eth2/types/src/proposal.rs +++ b/eth2/types/src/proposal.rs @@ -3,10 +3,11 @@ use crate::{Hash256, Slot}; use bls::Signature; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz::TreeHash; +use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; use test_random_derive::TestRandom; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)] pub struct Proposal { pub slot: Slot, /// Shard number (spec.beacon_chain_shard_number for beacon chain) @@ -19,7 +20,7 @@ pub struct Proposal { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; + use ssz::{ssz_encode, Decodable, SignedRoot, TreeHash}; #[test] pub fn test_ssz_round_trip() { @@ -43,4 +44,32 @@ mod tests { // TODO: Add further tests // https://github.com/sigp/lighthouse/issues/170 } + + #[derive(TreeHash)] + struct SignedProposal { + pub slot: Slot, + pub shard: u64, + pub block_root: Hash256, + } + + impl Into for Proposal { + fn into(self) -> SignedProposal { + SignedProposal { + slot: self.slot, + shard: self.shard, + block_root: self.block_root, + } + } + } + + #[test] + pub fn test_signed_root() { + let mut rng = XorShiftRng::from_seed([42; 16]); + let original = Proposal::random_for_test(&mut rng); + + let other: SignedProposal = original.clone().into(); + + assert_eq!(original.signed_root(), other.hash_tree_root()); + } + } diff --git a/eth2/utils/ssz/src/lib.rs b/eth2/utils/ssz/src/lib.rs index a6baa35a7..7c29667af 100644 --- a/eth2/utils/ssz/src/lib.rs +++ b/eth2/utils/ssz/src/lib.rs @@ -12,6 +12,7 @@ extern crate ethereum_types; pub mod decode; pub mod encode; +mod signed_root; pub mod tree_hash; mod impl_decode; @@ -20,6 +21,7 @@ mod impl_tree_hash; pub use crate::decode::{decode_ssz, decode_ssz_list, Decodable, DecodeError}; pub use crate::encode::{Encodable, SszStream}; +pub use crate::signed_root::SignedRoot; pub use crate::tree_hash::{merkle_hash, TreeHash}; pub use hashing::hash; diff --git a/eth2/utils/ssz/src/signed_root.rs b/eth2/utils/ssz/src/signed_root.rs new file mode 100644 index 000000000..f7aeca4af --- /dev/null +++ b/eth2/utils/ssz/src/signed_root.rs @@ -0,0 +1,5 @@ +use crate::TreeHash; + +pub trait SignedRoot: TreeHash { + fn signed_root(&self) -> Vec; +} diff --git a/eth2/utils/ssz_derive/src/lib.rs b/eth2/utils/ssz_derive/src/lib.rs index f71bff709..4c6b9dc11 100644 --- a/eth2/utils/ssz_derive/src/lib.rs +++ b/eth2/utils/ssz_derive/src/lib.rs @@ -158,3 +158,78 @@ pub fn ssz_tree_hash_derive(input: TokenStream) -> TokenStream { }; output.into() } + +/// Returns `true` if some `Ident` should be considered to be a signature type. +fn type_ident_is_signature(ident: &syn::Ident) -> bool { + match ident.to_string().as_ref() { + "Signature" => true, + "AggregateSignature" => true, + _ => false, + } +} + +/// Takes a `Field` where the type (`ty`) portion is a path (e.g., `types::Signature`) and returns +/// the final `Ident` in that path. +/// +/// E.g., for `types::Signature` returns `Signature`. +fn final_type_ident<'a>(field: &'a syn::Field) -> &'a syn::Ident { + match &field.ty { + syn::Type::Path(path) => &path.path.segments.last().unwrap().value().ident, + _ => panic!("ssz_derive only supports Path types."), + } +} + +/// Implements `ssz::TreeHash` for some `struct`, whilst excluding any fields following and +/// including a field that is of type "Signature" or "AggregateSignature". +/// +/// See: +/// https://github.com/ethereum/eth2.0-specs/blob/master/specs/simple-serialize.md#signed-roots +/// +/// This is a rather horrendous macro, it will read the type of the object as a string and decide +/// if it's a signature by matching that string against "Signature" or "AggregateSignature". So, +/// it's important that you use those exact words as your type -- don't alias it to something else. +/// +/// If you can think of a better way to do this, please make an issue! +/// +/// Fields are processed in the order they are defined. +#[proc_macro_derive(SignedRoot)] +pub fn ssz_signed_root_derive(input: TokenStream) -> TokenStream { + let item = parse_macro_input!(input as DeriveInput); + + let name = &item.ident; + + let struct_data = match &item.data { + syn::Data::Struct(s) => s, + _ => panic!("ssz_derive only supports structs."), + }; + + let mut field_idents: Vec<&syn::Ident> = vec![]; + + for field in struct_data.fields.iter() { + let final_type_ident = final_type_ident(&field); + + if type_ident_is_signature(final_type_ident) { + break; + } else { + let ident = field + .ident + .as_ref() + .expect("ssz_derive only supports named_struct fields."); + field_idents.push(ident); + } + } + + let output = quote! { + impl ssz::SignedRoot for #name { + fn signed_root(&self) -> Vec { + let mut list: Vec> = Vec::new(); + #( + list.push(self.#field_idents.hash_tree_root_internal()); + )* + + ssz::merkle_hash(&mut list) + } + } + }; + output.into() +} From 9769ca46656d9b8d6c8c2b625a4b4a22b59129ac Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 4 Mar 2019 17:13:50 +1100 Subject: [PATCH 098/132] Update all struct definitions Does not compile --- eth2/types/src/attestation.rs | 32 +--- eth2/types/src/attestation_data.rs | 46 ++--- .../src/attestation_data_and_custody_bit.rs | 3 + eth2/types/src/attester_slashing.rs | 3 + eth2/types/src/beacon_block.rs | 28 +-- eth2/types/src/beacon_block_body.rs | 8 +- eth2/types/src/beacon_state.rs | 169 ++++++++++-------- eth2/types/src/crosslink.rs | 13 +- eth2/types/src/deposit.rs | 3 + eth2/types/src/deposit_data.rs | 3 + eth2/types/src/deposit_input.rs | 3 + eth2/types/src/eth1_data.rs | 4 +- eth2/types/src/eth1_data_vote.rs | 4 +- eth2/types/src/fork.rs | 3 + eth2/types/src/lib.rs | 8 +- eth2/types/src/pending_attestation.rs | 3 + eth2/types/src/proposal.rs | 3 + eth2/types/src/proposer_slashing.rs | 4 +- eth2/types/src/proposer_slashing/builder.rs | 22 +-- eth2/types/src/slashable_attestation.rs | 17 +- eth2/types/src/transfer.rs | 51 ++++++ eth2/types/src/validator.rs | 132 ++------------ eth2/types/src/{exit.rs => voluntary_exit.rs} | 14 +- 23 files changed, 261 insertions(+), 315 deletions(-) create mode 100644 eth2/types/src/transfer.rs rename eth2/types/src/{exit.rs => voluntary_exit.rs} (74%) diff --git a/eth2/types/src/attestation.rs b/eth2/types/src/attestation.rs index a0c8505b8..03ef8ce48 100644 --- a/eth2/types/src/attestation.rs +++ b/eth2/types/src/attestation.rs @@ -1,12 +1,15 @@ -use super::{AggregatePublicKey, AggregateSignature, AttestationData, Bitfield, Hash256}; +use super::{AggregateSignature, AttestationData, Bitfield}; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; use ssz::TreeHash; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; use test_random_derive::TestRandom; -#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, TreeHash, TestRandom)] +/// Details an attestation that can be slashable. +/// +/// Spec v0.4.0 +#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)] pub struct Attestation { pub aggregation_bitfield: Bitfield, pub data: AttestationData, @@ -14,29 +17,6 @@ pub struct Attestation { pub aggregate_signature: AggregateSignature, } -impl Attestation { - pub fn canonical_root(&self) -> Hash256 { - Hash256::from(&self.hash_tree_root()[..]) - } - - pub fn signable_message(&self, custody_bit: bool) -> Vec { - self.data.signable_message(custody_bit) - } - - pub fn verify_signature( - &self, - group_public_key: &AggregatePublicKey, - custody_bit: bool, - domain: u64, - ) -> bool { - self.aggregate_signature.verify( - &self.signable_message(custody_bit), - domain, - group_public_key, - ) - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data.rs index e23cdab46..1dfadfb1d 100644 --- a/eth2/types/src/attestation_data.rs +++ b/eth2/types/src/attestation_data.rs @@ -1,31 +1,33 @@ use crate::test_utils::TestRandom; -use crate::{AttestationDataAndCustodyBit, Crosslink, Epoch, Hash256, Slot}; +use crate::{Crosslink, Epoch, Hash256, Slot}; use rand::RngCore; use serde_derive::Serialize; use ssz::TreeHash; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; use test_random_derive::TestRandom; -pub const SSZ_ATTESTION_DATA_LENGTH: usize = { - 8 + // slot - 8 + // shard - 32 + // beacon_block_hash - 32 + // epoch_boundary_root - 32 + // shard_block_hash - 32 + // latest_crosslink_hash - 8 + // justified_epoch - 32 // justified_block_root -}; - +/// The data upon which an attestation is based. +/// +/// Spec v0.4.0 #[derive( - Debug, Clone, PartialEq, Default, Serialize, Hash, Encode, Decode, TreeHash, TestRandom, + Debug, + Clone, + PartialEq, + Default, + Serialize, + Hash, + Encode, + Decode, + TreeHash, + TestRandom, + SignedRoot, )] pub struct AttestationData { pub slot: Slot, pub shard: u64, pub beacon_block_root: Hash256, pub epoch_boundary_root: Hash256, - pub shard_block_root: Hash256, + pub crosslink_data_root: Hash256, pub latest_crosslink: Crosslink, pub justified_epoch: Epoch, pub justified_block_root: Hash256, @@ -33,20 +35,6 @@ pub struct AttestationData { impl Eq for AttestationData {} -impl AttestationData { - pub fn canonical_root(&self) -> Hash256 { - Hash256::from(&self.hash_tree_root()[..]) - } - - pub fn signable_message(&self, custody_bit: bool) -> Vec { - let attestation_data_and_custody_bit = AttestationDataAndCustodyBit { - data: self.clone(), - custody_bit, - }; - attestation_data_and_custody_bit.hash_tree_root() - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/attestation_data_and_custody_bit.rs b/eth2/types/src/attestation_data_and_custody_bit.rs index 9175863ae..2f652609e 100644 --- a/eth2/types/src/attestation_data_and_custody_bit.rs +++ b/eth2/types/src/attestation_data_and_custody_bit.rs @@ -4,6 +4,9 @@ use rand::RngCore; use serde_derive::Serialize; use ssz_derive::{Decode, Encode, TreeHash}; +/// Used for pairing an attestation with a proof-of-custody. +/// +/// Spec v0.4.0 #[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode, TreeHash)] pub struct AttestationDataAndCustodyBit { pub data: AttestationData, diff --git a/eth2/types/src/attester_slashing.rs b/eth2/types/src/attester_slashing.rs index ac75a2562..1cb671960 100644 --- a/eth2/types/src/attester_slashing.rs +++ b/eth2/types/src/attester_slashing.rs @@ -8,6 +8,9 @@ mod builder; pub use builder::AttesterSlashingBuilder; +/// Two conflicting attestations. +/// +/// Spec v0.4.0 #[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] pub struct AttesterSlashing { pub slashable_attestation_1: SlashableAttestation, diff --git a/eth2/types/src/beacon_block.rs b/eth2/types/src/beacon_block.rs index cb4e6668b..53b0dac80 100644 --- a/eth2/types/src/beacon_block.rs +++ b/eth2/types/src/beacon_block.rs @@ -4,18 +4,21 @@ use bls::Signature; use rand::RngCore; use serde_derive::Serialize; use ssz::TreeHash; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; use test_random_derive::TestRandom; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] +/// A block of the `BeaconChain`. +/// +/// Spec v0.4.0 +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)] pub struct BeaconBlock { pub slot: Slot, pub parent_root: Hash256, pub state_root: Hash256, pub randao_reveal: Signature, pub eth1_data: Eth1Data, - pub signature: Signature, pub body: BeaconBlockBody, + pub signature: Signature, } impl BeaconBlock { @@ -40,25 +43,6 @@ impl BeaconBlock { }, } } - - pub fn canonical_root(&self) -> Hash256 { - Hash256::from(&self.hash_tree_root()[..]) - } - - pub fn proposal_root(&self, spec: &ChainSpec) -> Hash256 { - let block_without_signature_root = { - let mut block_without_signature = self.clone(); - block_without_signature.signature = spec.empty_signature.clone(); - block_without_signature.canonical_root() - }; - - let proposal = ProposalSignedData { - slot: self.slot, - shard: spec.beacon_chain_shard_number, - block_root: block_without_signature_root, - }; - Hash256::from(&proposal.hash_tree_root()[..]) - } } #[cfg(test)] diff --git a/eth2/types/src/beacon_block_body.rs b/eth2/types/src/beacon_block_body.rs index 2b343b970..13c82e42f 100644 --- a/eth2/types/src/beacon_block_body.rs +++ b/eth2/types/src/beacon_block_body.rs @@ -1,17 +1,21 @@ -use super::{Attestation, AttesterSlashing, Deposit, Exit, ProposerSlashing}; +use super::{Attestation, AttesterSlashing, Deposit, ProposerSlashing, Transfer, VolutaryExit}; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; +/// The body of a `BeaconChain` block, containing operations. +/// +/// Spec v0.4.0 #[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, TreeHash, TestRandom)] pub struct BeaconBlockBody { pub proposer_slashings: Vec, pub attester_slashings: Vec, pub attestations: Vec, pub deposits: Vec, - pub exits: Vec, + pub voluntary_exits: Vec, + pub transfers: Vec, } #[cfg(test)] diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index a9e2f2673..c96beff52 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -1,6 +1,6 @@ use self::epoch_cache::EpochCache; use crate::test_utils::TestRandom; -use crate::{validator::StatusFlags, validator_registry::get_active_validator_indices, *}; +use crate::{validator_registry::get_active_validator_indices, *}; use bls::verify_proof_of_possession; use honey_badger_split::SplitExt; use log::{debug, error, trace}; @@ -82,12 +82,12 @@ pub struct BeaconState { // Randomness and committees pub latest_randao_mixes: Vec, - pub previous_epoch_start_shard: u64, - pub current_epoch_start_shard: u64, - pub previous_calculation_epoch: Epoch, - pub current_calculation_epoch: Epoch, - pub previous_epoch_seed: Hash256, - pub current_epoch_seed: Hash256, + pub previous_shuffling_start_shard: u64, + pub current_shuffling_start_shard: u64, + pub previous_shuffling_epoch: Epoch, + pub current_shuffling_epoch: Epoch, + pub previous_shuffling_seed: Hash256, + pub current_shuffling_seed: Hash256, // Finality pub previous_justified_epoch: Epoch, @@ -98,8 +98,8 @@ pub struct BeaconState { // Recent state pub latest_crosslinks: Vec, pub latest_block_roots: Vec, - pub latest_index_roots: Vec, - pub latest_penalized_balances: Vec, + pub latest_active_index_roots: Vec, + pub latest_slashed_balances: Vec, pub latest_attestations: Vec, pub batched_block_roots: Vec, @@ -107,7 +107,7 @@ pub struct BeaconState { pub latest_eth1_data: Eth1Data, pub eth1_data_votes: Vec, - // Caching + // Caching (not in the spec) pub cache_index_offset: usize, pub caches: Vec, } @@ -148,12 +148,12 @@ impl BeaconState { * Randomness and committees */ latest_randao_mixes: vec![spec.zero_hash; spec.latest_randao_mixes_length as usize], - previous_epoch_start_shard: spec.genesis_start_shard, - current_epoch_start_shard: spec.genesis_start_shard, - previous_calculation_epoch: spec.genesis_epoch, - current_calculation_epoch: spec.genesis_epoch, - previous_epoch_seed: spec.zero_hash, - current_epoch_seed: spec.zero_hash, + previous_shuffling_start_shard: spec.genesis_start_shard, + current_shuffling_start_shard: spec.genesis_start_shard, + previous_shuffling_epoch: spec.genesis_epoch, + current_shuffling_epoch: spec.genesis_epoch, + previous_shuffling_seed: spec.zero_hash, + current_shuffling_seed: spec.zero_hash, /* * Finality @@ -168,8 +168,11 @@ impl BeaconState { */ latest_crosslinks: vec![initial_crosslink; spec.shard_count as usize], latest_block_roots: vec![spec.zero_hash; spec.latest_block_roots_length as usize], - latest_index_roots: vec![spec.zero_hash; spec.latest_index_roots_length as usize], - latest_penalized_balances: vec![0; spec.latest_penalized_exit_length as usize], + latest_active_index_roots: vec![ + spec.zero_hash; + spec.latest_active_index_roots_length as usize + ], + latest_slashed_balances: vec![0; spec.latest_penalized_exit_length as usize], latest_attestations: vec![], batched_block_roots: vec![], @@ -218,9 +221,10 @@ impl BeaconState { &genesis_state.validator_registry, spec.genesis_epoch, )); - genesis_state.latest_index_roots = - vec![genesis_active_index_root; spec.latest_index_roots_length]; - genesis_state.current_epoch_seed = genesis_state.generate_seed(spec.genesis_epoch, spec)?; + genesis_state.latest_active_index_roots = + vec![genesis_active_index_root; spec.latest_active_index_roots_length]; + genesis_state.current_shuffling_seed = + genesis_state.generate_seed(spec.genesis_epoch, spec)?; Ok(genesis_state) } @@ -440,7 +444,7 @@ impl BeaconState { /// Spec v0.2.0 fn get_previous_epoch_committee_count(&self, spec: &ChainSpec) -> u64 { let previous_active_validators = - get_active_validator_indices(&self.validator_registry, self.previous_calculation_epoch); + get_active_validator_indices(&self.validator_registry, self.previous_shuffling_epoch); self.get_epoch_committee_count(previous_active_validators.len(), spec) } @@ -449,7 +453,7 @@ impl BeaconState { /// Spec v0.2.0 pub fn get_current_epoch_committee_count(&self, spec: &ChainSpec) -> u64 { let current_active_validators = - get_active_validator_indices(&self.validator_registry, self.current_calculation_epoch); + get_active_validator_indices(&self.validator_registry, self.current_shuffling_epoch); self.get_epoch_committee_count(current_active_validators.len(), spec) } @@ -468,13 +472,17 @@ impl BeaconState { pub fn get_active_index_root(&self, epoch: Epoch, spec: &ChainSpec) -> Option { let current_epoch = self.current_epoch(spec); - let earliest_index_root = current_epoch - Epoch::from(spec.latest_index_roots_length) + let earliest_index_root = current_epoch + - Epoch::from(spec.latest_active_index_roots_length) + Epoch::from(spec.entry_exit_delay) + 1; let latest_index_root = current_epoch + spec.entry_exit_delay; if (epoch >= earliest_index_root) & (epoch <= latest_index_root) { - Some(self.latest_index_roots[epoch.as_usize() % spec.latest_index_roots_length]) + Some( + self.latest_active_index_roots + [epoch.as_usize() % spec.latest_active_index_roots_length], + ) } else { None } @@ -566,17 +574,17 @@ impl BeaconState { trace!("get_committee_params_at_slot: current_epoch"); Ok(( self.get_current_epoch_committee_count(spec), - self.current_epoch_seed, - self.current_calculation_epoch, - self.current_epoch_start_shard, + self.current_shuffling_seed, + self.current_shuffling_epoch, + self.current_shuffling_start_shard, )) } else if epoch == previous_epoch { trace!("get_committee_params_at_slot: previous_epoch"); Ok(( self.get_previous_epoch_committee_count(spec), - self.previous_epoch_seed, - self.previous_calculation_epoch, - self.previous_epoch_start_shard, + self.previous_shuffling_seed, + self.previous_shuffling_epoch, + self.previous_shuffling_start_shard, )) } else if epoch == next_epoch { trace!("get_committee_params_at_slot: next_epoch"); @@ -587,16 +595,19 @@ impl BeaconState { let next_seed = self.generate_seed(next_epoch, spec)?; ( next_seed, - (self.current_epoch_start_shard + current_committees_per_epoch) + (self.current_shuffling_start_shard + current_committees_per_epoch) % spec.shard_count, ) } else if (epochs_since_last_registry_update > 1) & epochs_since_last_registry_update.is_power_of_two() { let next_seed = self.generate_seed(next_epoch, spec)?; - (next_seed, self.current_epoch_start_shard) + (next_seed, self.current_shuffling_start_shard) } else { - (self.current_epoch_seed, self.current_epoch_start_shard) + ( + self.current_shuffling_seed, + self.current_shuffling_start_shard, + ) }; Ok(( self.get_next_epoch_committee_count(spec), @@ -715,9 +726,9 @@ impl BeaconState { let epoch_index: usize = current_epoch.as_usize() % spec.latest_penalized_exit_length; - let total_at_start = self.latest_penalized_balances + let total_at_start = self.latest_slashed_balances [(epoch_index + 1) % spec.latest_penalized_exit_length]; - let total_at_end = self.latest_penalized_balances[epoch_index]; + let total_at_end = self.latest_slashed_balances[epoch_index]; let total_penalities = total_at_end.saturating_sub(total_at_start); let penalty = self.get_effective_balance(index, spec) * std::cmp::min(total_penalities * 3, total_balance) @@ -983,7 +994,7 @@ impl BeaconState { self.exit_validator(validator_index, spec); let current_epoch = self.current_epoch(spec); - self.latest_penalized_balances + self.latest_slashed_balances [current_epoch.as_usize() % spec.latest_penalized_exit_length] += self.get_effective_balance(validator_index, spec); @@ -1329,20 +1340,20 @@ impl Encodable for BeaconState { s.append(&self.validator_balances); s.append(&self.validator_registry_update_epoch); s.append(&self.latest_randao_mixes); - s.append(&self.previous_epoch_start_shard); - s.append(&self.current_epoch_start_shard); - s.append(&self.previous_calculation_epoch); - s.append(&self.current_calculation_epoch); - s.append(&self.previous_epoch_seed); - s.append(&self.current_epoch_seed); + s.append(&self.previous_shuffling_start_shard); + s.append(&self.current_shuffling_start_shard); + s.append(&self.previous_shuffling_epoch); + s.append(&self.current_shuffling_epoch); + s.append(&self.previous_shuffling_seed); + s.append(&self.current_shuffling_seed); s.append(&self.previous_justified_epoch); s.append(&self.justified_epoch); s.append(&self.justification_bitfield); s.append(&self.finalized_epoch); s.append(&self.latest_crosslinks); s.append(&self.latest_block_roots); - s.append(&self.latest_index_roots); - s.append(&self.latest_penalized_balances); + s.append(&self.latest_active_index_roots); + s.append(&self.latest_slashed_balances); s.append(&self.latest_attestations); s.append(&self.batched_block_roots); s.append(&self.latest_eth1_data); @@ -1359,20 +1370,20 @@ impl Decodable for BeaconState { let (validator_balances, i) = <_>::ssz_decode(bytes, i)?; let (validator_registry_update_epoch, i) = <_>::ssz_decode(bytes, i)?; let (latest_randao_mixes, i) = <_>::ssz_decode(bytes, i)?; - let (previous_epoch_start_shard, i) = <_>::ssz_decode(bytes, i)?; - let (current_epoch_start_shard, i) = <_>::ssz_decode(bytes, i)?; - let (previous_calculation_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (current_calculation_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (previous_epoch_seed, i) = <_>::ssz_decode(bytes, i)?; - let (current_epoch_seed, i) = <_>::ssz_decode(bytes, i)?; + let (previous_shuffling_start_shard, i) = <_>::ssz_decode(bytes, i)?; + let (current_shuffling_start_shard, i) = <_>::ssz_decode(bytes, i)?; + let (previous_shuffling_epoch, i) = <_>::ssz_decode(bytes, i)?; + let (current_shuffling_epoch, i) = <_>::ssz_decode(bytes, i)?; + let (previous_shuffling_seed, i) = <_>::ssz_decode(bytes, i)?; + let (current_shuffling_seed, i) = <_>::ssz_decode(bytes, i)?; let (previous_justified_epoch, i) = <_>::ssz_decode(bytes, i)?; let (justified_epoch, i) = <_>::ssz_decode(bytes, i)?; let (justification_bitfield, i) = <_>::ssz_decode(bytes, i)?; let (finalized_epoch, i) = <_>::ssz_decode(bytes, i)?; let (latest_crosslinks, i) = <_>::ssz_decode(bytes, i)?; let (latest_block_roots, i) = <_>::ssz_decode(bytes, i)?; - let (latest_index_roots, i) = <_>::ssz_decode(bytes, i)?; - let (latest_penalized_balances, i) = <_>::ssz_decode(bytes, i)?; + let (latest_active_index_roots, i) = <_>::ssz_decode(bytes, i)?; + let (latest_slashed_balances, i) = <_>::ssz_decode(bytes, i)?; let (latest_attestations, i) = <_>::ssz_decode(bytes, i)?; let (batched_block_roots, i) = <_>::ssz_decode(bytes, i)?; let (latest_eth1_data, i) = <_>::ssz_decode(bytes, i)?; @@ -1387,20 +1398,20 @@ impl Decodable for BeaconState { validator_balances, validator_registry_update_epoch, latest_randao_mixes, - previous_epoch_start_shard, - current_epoch_start_shard, - previous_calculation_epoch, - current_calculation_epoch, - previous_epoch_seed, - current_epoch_seed, + previous_shuffling_start_shard, + current_shuffling_start_shard, + previous_shuffling_epoch, + current_shuffling_epoch, + previous_shuffling_seed, + current_shuffling_seed, previous_justified_epoch, justified_epoch, justification_bitfield, finalized_epoch, latest_crosslinks, latest_block_roots, - latest_index_roots, - latest_penalized_balances, + latest_active_index_roots, + latest_slashed_balances, latest_attestations, batched_block_roots, latest_eth1_data, @@ -1427,20 +1438,24 @@ impl TreeHash for BeaconState { .hash_tree_root_internal(), ); result.append(&mut self.latest_randao_mixes.hash_tree_root_internal()); - result.append(&mut self.previous_epoch_start_shard.hash_tree_root_internal()); - result.append(&mut self.current_epoch_start_shard.hash_tree_root_internal()); - result.append(&mut self.previous_calculation_epoch.hash_tree_root_internal()); - result.append(&mut self.current_calculation_epoch.hash_tree_root_internal()); - result.append(&mut self.previous_epoch_seed.hash_tree_root_internal()); - result.append(&mut self.current_epoch_seed.hash_tree_root_internal()); + result.append( + &mut self + .previous_shuffling_start_shard + .hash_tree_root_internal(), + ); + result.append(&mut self.current_shuffling_start_shard.hash_tree_root_internal()); + result.append(&mut self.previous_shuffling_epoch.hash_tree_root_internal()); + result.append(&mut self.current_shuffling_epoch.hash_tree_root_internal()); + result.append(&mut self.previous_shuffling_seed.hash_tree_root_internal()); + result.append(&mut self.current_shuffling_seed.hash_tree_root_internal()); result.append(&mut self.previous_justified_epoch.hash_tree_root_internal()); result.append(&mut self.justified_epoch.hash_tree_root_internal()); result.append(&mut self.justification_bitfield.hash_tree_root_internal()); result.append(&mut self.finalized_epoch.hash_tree_root_internal()); result.append(&mut self.latest_crosslinks.hash_tree_root_internal()); result.append(&mut self.latest_block_roots.hash_tree_root_internal()); - result.append(&mut self.latest_index_roots.hash_tree_root_internal()); - result.append(&mut self.latest_penalized_balances.hash_tree_root_internal()); + result.append(&mut self.latest_active_index_roots.hash_tree_root_internal()); + result.append(&mut self.latest_slashed_balances.hash_tree_root_internal()); result.append(&mut self.latest_attestations.hash_tree_root_internal()); result.append(&mut self.batched_block_roots.hash_tree_root_internal()); result.append(&mut self.latest_eth1_data.hash_tree_root_internal()); @@ -1459,20 +1474,20 @@ impl TestRandom for BeaconState { validator_balances: <_>::random_for_test(rng), validator_registry_update_epoch: <_>::random_for_test(rng), latest_randao_mixes: <_>::random_for_test(rng), - previous_epoch_start_shard: <_>::random_for_test(rng), - current_epoch_start_shard: <_>::random_for_test(rng), - previous_calculation_epoch: <_>::random_for_test(rng), - current_calculation_epoch: <_>::random_for_test(rng), - previous_epoch_seed: <_>::random_for_test(rng), - current_epoch_seed: <_>::random_for_test(rng), + previous_shuffling_start_shard: <_>::random_for_test(rng), + current_shuffling_start_shard: <_>::random_for_test(rng), + previous_shuffling_epoch: <_>::random_for_test(rng), + current_shuffling_epoch: <_>::random_for_test(rng), + previous_shuffling_seed: <_>::random_for_test(rng), + current_shuffling_seed: <_>::random_for_test(rng), previous_justified_epoch: <_>::random_for_test(rng), justified_epoch: <_>::random_for_test(rng), justification_bitfield: <_>::random_for_test(rng), finalized_epoch: <_>::random_for_test(rng), latest_crosslinks: <_>::random_for_test(rng), latest_block_roots: <_>::random_for_test(rng), - latest_index_roots: <_>::random_for_test(rng), - latest_penalized_balances: <_>::random_for_test(rng), + latest_active_index_roots: <_>::random_for_test(rng), + latest_slashed_balances: <_>::random_for_test(rng), latest_attestations: <_>::random_for_test(rng), batched_block_roots: <_>::random_for_test(rng), latest_eth1_data: <_>::random_for_test(rng), diff --git a/eth2/types/src/crosslink.rs b/eth2/types/src/crosslink.rs index 11fb3386d..142d0f894 100644 --- a/eth2/types/src/crosslink.rs +++ b/eth2/types/src/crosslink.rs @@ -5,6 +5,9 @@ use serde_derive::Serialize; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; +/// Specifies the block hash for a shard at an epoch. +/// +/// Spec v0.4.0 #[derive( Debug, Clone, PartialEq, Default, Serialize, Hash, Encode, Decode, TreeHash, TestRandom, )] @@ -13,16 +16,6 @@ pub struct Crosslink { pub shard_block_root: Hash256, } -impl Crosslink { - /// Generates a new instance where `dynasty` and `hash` are both zero. - pub fn zero() -> Self { - Self { - epoch: Epoch::new(0), - shard_block_root: Hash256::zero(), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/deposit.rs b/eth2/types/src/deposit.rs index 02da32cfe..2e69ea599 100644 --- a/eth2/types/src/deposit.rs +++ b/eth2/types/src/deposit.rs @@ -5,6 +5,9 @@ use serde_derive::Serialize; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; +/// A deposit to potentially become a beacon chain validator. +/// +/// Spec v0.4.0 #[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] pub struct Deposit { pub branch: Vec, diff --git a/eth2/types/src/deposit_data.rs b/eth2/types/src/deposit_data.rs index 349207791..1eb2722a9 100644 --- a/eth2/types/src/deposit_data.rs +++ b/eth2/types/src/deposit_data.rs @@ -5,6 +5,9 @@ use serde_derive::Serialize; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; +/// Data generated by the deposit contract. +/// +/// Spec v0.4.0 #[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] pub struct DepositData { pub amount: u64, diff --git a/eth2/types/src/deposit_input.rs b/eth2/types/src/deposit_input.rs index 1f3b22779..c4c79c3d1 100644 --- a/eth2/types/src/deposit_input.rs +++ b/eth2/types/src/deposit_input.rs @@ -6,6 +6,9 @@ use serde_derive::Serialize; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; +/// The data supplied by the user to the deposit contract. +/// +/// Spec v0.4.0 #[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] pub struct DepositInput { pub pubkey: PublicKey, diff --git a/eth2/types/src/eth1_data.rs b/eth2/types/src/eth1_data.rs index 8eabbabc7..2c817ca38 100644 --- a/eth2/types/src/eth1_data.rs +++ b/eth2/types/src/eth1_data.rs @@ -5,7 +5,9 @@ use serde_derive::Serialize; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; -// Note: this is refer to as DepositRootVote in specs +/// Contains data obtained from the Eth1 chain. +/// +/// Spec v0.4.0 #[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, TreeHash, TestRandom)] pub struct Eth1Data { pub deposit_root: Hash256, diff --git a/eth2/types/src/eth1_data_vote.rs b/eth2/types/src/eth1_data_vote.rs index fa30b9052..898145575 100644 --- a/eth2/types/src/eth1_data_vote.rs +++ b/eth2/types/src/eth1_data_vote.rs @@ -5,7 +5,9 @@ use serde_derive::Serialize; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; -// Note: this is refer to as DepositRootVote in specs +/// A summation of votes for some `Eth1Data`. +/// +/// Spec v0.4.0 #[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, TreeHash, TestRandom)] pub struct Eth1DataVote { pub eth1_data: Eth1Data, diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs index 5b13a2388..783a65302 100644 --- a/eth2/types/src/fork.rs +++ b/eth2/types/src/fork.rs @@ -4,6 +4,9 @@ use serde_derive::Serialize; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; +/// Specifies a fork of the `BeaconChain`, to prevent replay attacks. +/// +/// Spec v0.4.0 #[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode, TreeHash, TestRandom)] pub struct Fork { pub previous_version: u64, diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index ff73a6ea6..b4b124294 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -15,7 +15,6 @@ pub mod deposit_data; pub mod deposit_input; pub mod eth1_data; pub mod eth1_data_vote; -pub mod exit; pub mod fork; pub mod free_attestation; pub mod pending_attestation; @@ -26,6 +25,8 @@ pub mod readers; pub mod shard_reassignment_record; pub mod slashable_attestation; pub mod slashable_vote_data; +pub mod transfer; +pub mod voluntary_exit; #[macro_use] pub mod slot_epoch_macros; pub mod slot_epoch; @@ -54,7 +55,6 @@ pub use crate::deposit_data::DepositData; pub use crate::deposit_input::DepositInput; pub use crate::eth1_data::Eth1Data; pub use crate::eth1_data_vote::Eth1DataVote; -pub use crate::exit::Exit; pub use crate::fork::Fork; pub use crate::free_attestation::FreeAttestation; pub use crate::pending_attestation::PendingAttestation; @@ -65,8 +65,10 @@ pub use crate::slashable_attestation::SlashableAttestation; pub use crate::slashable_vote_data::SlashableVoteData; pub use crate::slot_epoch::{Epoch, Slot}; pub use crate::slot_height::SlotHeight; -pub use crate::validator::{StatusFlags as ValidatorStatusFlags, Validator}; +pub use crate::transfer::Transfer; +pub use crate::validator::Validator; pub use crate::validator_registry_delta_block::ValidatorRegistryDeltaBlock; +pub use crate::voluntary_exit::VolutaryExit; pub type Hash256 = H256; pub type Address = H160; diff --git a/eth2/types/src/pending_attestation.rs b/eth2/types/src/pending_attestation.rs index 84eb59207..0430d18ba 100644 --- a/eth2/types/src/pending_attestation.rs +++ b/eth2/types/src/pending_attestation.rs @@ -5,6 +5,9 @@ use serde_derive::Serialize; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; +/// An attestation that has been included in the state but not yet fully processed. +/// +/// Spec v0.4.0 #[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, TreeHash, TestRandom)] pub struct PendingAttestation { pub aggregation_bitfield: Bitfield, diff --git a/eth2/types/src/proposal.rs b/eth2/types/src/proposal.rs index af69731cb..b1fd737a0 100644 --- a/eth2/types/src/proposal.rs +++ b/eth2/types/src/proposal.rs @@ -7,6 +7,9 @@ use ssz::TreeHash; use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; use test_random_derive::TestRandom; +/// A proposal for some shard or beacon block. +/// +/// Spec v0.4.0 #[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)] pub struct Proposal { pub slot: Slot, diff --git a/eth2/types/src/proposer_slashing.rs b/eth2/types/src/proposer_slashing.rs index 3b6f57c40..f86e7f3a8 100644 --- a/eth2/types/src/proposer_slashing.rs +++ b/eth2/types/src/proposer_slashing.rs @@ -15,8 +15,8 @@ pub use builder::ProposerSlashingBuilder; #[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] pub struct ProposerSlashing { pub proposer_index: u64, - pub proposer_1: Proposal, - pub proposer_2: Proposal, + pub proposal_1: Proposal, + pub proposal_2: Proposal, } #[cfg(test)] diff --git a/eth2/types/src/proposer_slashing/builder.rs b/eth2/types/src/proposer_slashing/builder.rs index 7923ff74d..66f74a2e2 100644 --- a/eth2/types/src/proposer_slashing/builder.rs +++ b/eth2/types/src/proposer_slashing/builder.rs @@ -1,5 +1,5 @@ use crate::*; -use ssz::TreeHash; +use ssz::SignedRoot; /// Builds a `ProposerSlashing`. pub struct ProposerSlashingBuilder(); @@ -22,27 +22,29 @@ impl ProposerSlashingBuilder { let slot = Slot::new(0); let shard = 0; - let proposal_data_1 = ProposalSignedData { + let mut proposal_1 = Proposal { slot, shard, block_root: Hash256::from(&[1][..]), + signature: Signature::empty_signature(), }; - let proposal_data_2 = ProposalSignedData { + let mut proposal_2 = Proposal { slot, shard, block_root: Hash256::from(&[2][..]), + signature: Signature::empty_signature(), }; - let proposal_signature_1 = { - let message = proposal_data_1.hash_tree_root(); + proposal_1.signature = { + let message = proposal_1.signed_root(); let epoch = slot.epoch(spec.epoch_length); let domain = spec.domain_proposal; signer(proposer_index, &message[..], epoch, domain) }; - let proposal_signature_2 = { - let message = proposal_data_2.hash_tree_root(); + proposal_2.signature = { + let message = proposal_2.signed_root(); let epoch = slot.epoch(spec.epoch_length); let domain = spec.domain_proposal; signer(proposer_index, &message[..], epoch, domain) @@ -50,10 +52,8 @@ impl ProposerSlashingBuilder { ProposerSlashing { proposer_index, - proposal_data_1, - proposal_signature_1, - proposal_data_2, - proposal_signature_2, + proposal_1, + proposal_2, } } } diff --git a/eth2/types/src/slashable_attestation.rs b/eth2/types/src/slashable_attestation.rs index 8ad582ce6..9f2ccab60 100644 --- a/eth2/types/src/slashable_attestation.rs +++ b/eth2/types/src/slashable_attestation.rs @@ -1,11 +1,18 @@ use crate::{test_utils::TestRandom, AggregateSignature, AttestationData, Bitfield, ChainSpec}; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz::TreeHash; +use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; use test_random_derive::TestRandom; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] +/// Details an attestation that can be slashable. +/// +/// To be included in an `AttesterSlashing`. +/// +/// Spec v0.4.0 +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)] pub struct SlashableAttestation { + /// Lists validator registry indices, not committee indices. pub validator_indices: Vec, pub data: AttestationData, pub custody_bitfield: Bitfield, @@ -15,21 +22,21 @@ pub struct SlashableAttestation { impl SlashableAttestation { /// Check if ``attestation_data_1`` and ``attestation_data_2`` have the same target. /// - /// Spec v0.3.0 + /// Spec v0.4.0 pub fn is_double_vote(&self, other: &SlashableAttestation, spec: &ChainSpec) -> bool { self.data.slot.epoch(spec.epoch_length) == other.data.slot.epoch(spec.epoch_length) } /// Check if ``attestation_data_1`` surrounds ``attestation_data_2``. /// - /// Spec v0.3.0 + /// Spec v0.4.0 pub fn is_surround_vote(&self, other: &SlashableAttestation, spec: &ChainSpec) -> bool { let source_epoch_1 = self.data.justified_epoch; let source_epoch_2 = other.data.justified_epoch; let target_epoch_1 = self.data.slot.epoch(spec.epoch_length); let target_epoch_2 = other.data.slot.epoch(spec.epoch_length); - (source_epoch_1 < source_epoch_2) && (target_epoch_2 < target_epoch_1) + (source_epoch_1 < source_epoch_2) & (target_epoch_2 < target_epoch_1) } } diff --git a/eth2/types/src/transfer.rs b/eth2/types/src/transfer.rs new file mode 100644 index 000000000..f23d3845e --- /dev/null +++ b/eth2/types/src/transfer.rs @@ -0,0 +1,51 @@ +use super::Slot; +use crate::test_utils::TestRandom; +use bls::{PublicKey, Signature}; +use rand::RngCore; +use serde_derive::Serialize; +use ssz_derive::{Decode, Encode, TreeHash}; +use test_random_derive::TestRandom; + +/// The data submitted to the deposit contract. +/// +/// Spec v0.4.0 +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] +pub struct Transfer { + pub from: u64, + pub to: u64, + pub amount: u64, + pub fee: u64, + pub slot: Slot, + pub pubkey: PublicKey, + pub signature: Signature, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::{ssz_encode, Decodable, TreeHash}; + + #[test] + pub fn test_ssz_round_trip() { + let mut rng = XorShiftRng::from_seed([42; 16]); + let original = Transfer::random_for_test(&mut rng); + + let bytes = ssz_encode(&original); + let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); + + assert_eq!(original, decoded); + } + + #[test] + pub fn test_hash_tree_root_internal() { + let mut rng = XorShiftRng::from_seed([42; 16]); + let original = Transfer::random_for_test(&mut rng); + + let result = original.hash_tree_root_internal(); + + assert_eq!(result.len(), 32); + // TODO: Add further tests + // https://github.com/sigp/lighthouse/issues/170 + } +} diff --git a/eth2/types/src/validator.rs b/eth2/types/src/validator.rs index 587a48a1f..5b5dab6bf 100644 --- a/eth2/types/src/validator.rs +++ b/eth2/types/src/validator.rs @@ -2,55 +2,21 @@ use crate::{test_utils::TestRandom, Epoch, Hash256, PublicKey}; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz_derive::{Decode, Encode, TreeHash}; +use test_random_derive::TestRandom; -const STATUS_FLAG_INITIATED_EXIT: u8 = 1; -const STATUS_FLAG_WITHDRAWABLE: u8 = 2; - -#[derive(Debug, PartialEq, Clone, Copy, Serialize)] -pub enum StatusFlags { - InitiatedExit, - Withdrawable, -} - -struct StatusFlagsDecodeError; - -impl From for DecodeError { - fn from(_: StatusFlagsDecodeError) -> DecodeError { - DecodeError::Invalid - } -} - -/// Handles the serialization logic for the `status_flags` field of the `Validator`. -fn status_flag_to_byte(flag: Option) -> u8 { - if let Some(flag) = flag { - match flag { - StatusFlags::InitiatedExit => STATUS_FLAG_INITIATED_EXIT, - StatusFlags::Withdrawable => STATUS_FLAG_WITHDRAWABLE, - } - } else { - 0 - } -} - -/// Handles the deserialization logic for the `status_flags` field of the `Validator`. -fn status_flag_from_byte(flag: u8) -> Result, StatusFlagsDecodeError> { - match flag { - 0 => Ok(None), - 1 => Ok(Some(StatusFlags::InitiatedExit)), - 2 => Ok(Some(StatusFlags::Withdrawable)), - _ => Err(StatusFlagsDecodeError), - } -} - -#[derive(Debug, Clone, PartialEq, Serialize)] +/// Information about a `BeaconChain` validator. +/// +/// Spec v0.4.0 +#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, TestRandom)] pub struct Validator { pub pubkey: PublicKey, pub withdrawal_credentials: Hash256, pub activation_epoch: Epoch, pub exit_epoch: Epoch, pub withdrawal_epoch: Epoch, - pub penalized_epoch: Epoch, - pub status_flags: Option, + pub initiated_exit: bool, + pub slashed: bool, } impl Validator { @@ -64,9 +30,9 @@ impl Validator { self.exit_epoch <= epoch } - /// Returns `true` if the validator is considered penalized at some epoch. - pub fn is_penalized_at(&self, epoch: Epoch) -> bool { - self.penalized_epoch <= epoch + /// Returns `true` if the validator is able to withdraw at some epoch. + pub fn is_withdrawable_at(&self, epoch: Epoch) -> bool { + self.withdrawal_epoch <= epoch } } @@ -85,82 +51,6 @@ impl Default for Validator { } } -impl TestRandom for StatusFlags { - fn random_for_test(rng: &mut T) -> Self { - let options = vec![StatusFlags::InitiatedExit, StatusFlags::Withdrawable]; - options[(rng.next_u32() as usize) % options.len()] - } -} - -impl Encodable for Validator { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.pubkey); - s.append(&self.withdrawal_credentials); - s.append(&self.activation_epoch); - s.append(&self.exit_epoch); - s.append(&self.withdrawal_epoch); - s.append(&self.penalized_epoch); - s.append(&status_flag_to_byte(self.status_flags)); - } -} - -impl Decodable for Validator { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (pubkey, i) = <_>::ssz_decode(bytes, i)?; - let (withdrawal_credentials, i) = <_>::ssz_decode(bytes, i)?; - let (activation_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (exit_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (withdrawal_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (penalized_epoch, i) = <_>::ssz_decode(bytes, i)?; - let (status_flags_byte, i): (u8, usize) = <_>::ssz_decode(bytes, i)?; - - let status_flags = status_flag_from_byte(status_flags_byte)?; - - Ok(( - Self { - pubkey, - withdrawal_credentials, - activation_epoch, - exit_epoch, - withdrawal_epoch, - penalized_epoch, - status_flags, - }, - i, - )) - } -} - -impl TreeHash for Validator { - fn hash_tree_root_internal(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.pubkey.hash_tree_root_internal()); - result.append(&mut self.withdrawal_credentials.hash_tree_root_internal()); - result.append(&mut self.activation_epoch.hash_tree_root_internal()); - result.append(&mut self.exit_epoch.hash_tree_root_internal()); - result.append(&mut self.withdrawal_epoch.hash_tree_root_internal()); - result.append(&mut self.penalized_epoch.hash_tree_root_internal()); - result.append( - &mut u64::from(status_flag_to_byte(self.status_flags)).hash_tree_root_internal(), - ); - hash(&result) - } -} - -impl TestRandom for Validator { - fn random_for_test(rng: &mut T) -> Self { - Self { - pubkey: <_>::random_for_test(rng), - withdrawal_credentials: <_>::random_for_test(rng), - activation_epoch: <_>::random_for_test(rng), - exit_epoch: <_>::random_for_test(rng), - withdrawal_epoch: <_>::random_for_test(rng), - penalized_epoch: <_>::random_for_test(rng), - status_flags: Some(<_>::random_for_test(rng)), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/types/src/exit.rs b/eth2/types/src/voluntary_exit.rs similarity index 74% rename from eth2/types/src/exit.rs rename to eth2/types/src/voluntary_exit.rs index 5b41fcc7a..3cb0a85b7 100644 --- a/eth2/types/src/exit.rs +++ b/eth2/types/src/voluntary_exit.rs @@ -2,11 +2,15 @@ use crate::{test_utils::TestRandom, Epoch}; use bls::Signature; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz::TreeHash; +use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; use test_random_derive::TestRandom; -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] -pub struct Exit { +/// An exit voluntarily submitted a validator who wishes to withdraw. +/// +/// Spec v0.4.0 +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)] +pub struct VolutaryExit { pub epoch: Epoch, pub validator_index: u64, pub signature: Signature, @@ -21,7 +25,7 @@ mod tests { #[test] pub fn test_ssz_round_trip() { let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Exit::random_for_test(&mut rng); + let original = VolutaryExit::random_for_test(&mut rng); let bytes = ssz_encode(&original); let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); @@ -32,7 +36,7 @@ mod tests { #[test] pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Exit::random_for_test(&mut rng); + let original = VolutaryExit::random_for_test(&mut rng); let result = original.hash_tree_root_internal(); From 7f10d41121f5daf38cd1be16a4fe1746365bc00c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 4 Mar 2019 17:17:07 +1100 Subject: [PATCH 099/132] Remove old spec types They have been made obsolete --- eth2/types/src/casper_slashing.rs | 42 ----- eth2/types/src/lib.rs | 8 - eth2/types/src/proposal_signed_data.rs | 43 ----- eth2/types/src/slashable_vote_data.rs | 154 ------------------ .../src/validator_registry_delta_block.rs | 59 ------- 5 files changed, 306 deletions(-) delete mode 100644 eth2/types/src/casper_slashing.rs delete mode 100644 eth2/types/src/proposal_signed_data.rs delete mode 100644 eth2/types/src/slashable_vote_data.rs delete mode 100644 eth2/types/src/validator_registry_delta_block.rs diff --git a/eth2/types/src/casper_slashing.rs b/eth2/types/src/casper_slashing.rs deleted file mode 100644 index cb1e46ee5..000000000 --- a/eth2/types/src/casper_slashing.rs +++ /dev/null @@ -1,42 +0,0 @@ -use super::SlashableVoteData; -use crate::test_utils::TestRandom; -use rand::RngCore; -use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, TreeHash}; -use test_random_derive::TestRandom; - -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] -pub struct CasperSlashing { - pub slashable_vote_data_1: SlashableVoteData, - pub slashable_vote_data_2: SlashableVoteData, -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = CasperSlashing::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = CasperSlashing::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } -} diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index b4b124294..f54a24b01 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -7,7 +7,6 @@ pub mod attester_slashing; pub mod beacon_block; pub mod beacon_block_body; pub mod beacon_state; -pub mod casper_slashing; pub mod chain_spec; pub mod crosslink; pub mod deposit; @@ -19,12 +18,10 @@ pub mod fork; pub mod free_attestation; pub mod pending_attestation; pub mod proposal; -pub mod proposal_signed_data; pub mod proposer_slashing; pub mod readers; pub mod shard_reassignment_record; pub mod slashable_attestation; -pub mod slashable_vote_data; pub mod transfer; pub mod voluntary_exit; #[macro_use] @@ -33,7 +30,6 @@ pub mod slot_epoch; pub mod slot_height; pub mod validator; pub mod validator_registry; -pub mod validator_registry_delta_block; use ethereum_types::{H160, H256, U256}; use std::collections::HashMap; @@ -47,7 +43,6 @@ pub use crate::beacon_block_body::BeaconBlockBody; pub use crate::beacon_state::{ BeaconState, Error as BeaconStateError, InclusionError, RelativeEpoch, }; -pub use crate::casper_slashing::CasperSlashing; pub use crate::chain_spec::ChainSpec; pub use crate::crosslink::Crosslink; pub use crate::deposit::Deposit; @@ -59,15 +54,12 @@ pub use crate::fork::Fork; pub use crate::free_attestation::FreeAttestation; pub use crate::pending_attestation::PendingAttestation; pub use crate::proposal::Proposal; -pub use crate::proposal_signed_data::ProposalSignedData; pub use crate::proposer_slashing::ProposerSlashing; pub use crate::slashable_attestation::SlashableAttestation; -pub use crate::slashable_vote_data::SlashableVoteData; pub use crate::slot_epoch::{Epoch, Slot}; pub use crate::slot_height::SlotHeight; pub use crate::transfer::Transfer; pub use crate::validator::Validator; -pub use crate::validator_registry_delta_block::ValidatorRegistryDeltaBlock; pub use crate::voluntary_exit::VolutaryExit; pub type Hash256 = H256; diff --git a/eth2/types/src/proposal_signed_data.rs b/eth2/types/src/proposal_signed_data.rs deleted file mode 100644 index 6f6048ffc..000000000 --- a/eth2/types/src/proposal_signed_data.rs +++ /dev/null @@ -1,43 +0,0 @@ -use crate::test_utils::TestRandom; -use crate::{Hash256, Slot}; -use rand::RngCore; -use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, TreeHash}; -use test_random_derive::TestRandom; - -#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, TreeHash, TestRandom)] -pub struct ProposalSignedData { - pub slot: Slot, - pub shard: u64, - pub block_root: Hash256, -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = ProposalSignedData::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = ProposalSignedData::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } -} diff --git a/eth2/types/src/slashable_vote_data.rs b/eth2/types/src/slashable_vote_data.rs deleted file mode 100644 index 31dd9e0a8..000000000 --- a/eth2/types/src/slashable_vote_data.rs +++ /dev/null @@ -1,154 +0,0 @@ -use super::AttestationData; -use crate::chain_spec::ChainSpec; -use crate::test_utils::TestRandom; -use bls::AggregateSignature; -use rand::RngCore; -use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, TreeHash}; -use test_random_derive::TestRandom; - -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] -pub struct SlashableVoteData { - pub custody_bit_0_indices: Vec, - pub custody_bit_1_indices: Vec, - pub data: AttestationData, - pub aggregate_signature: AggregateSignature, -} - -impl SlashableVoteData { - /// Check if ``attestation_data_1`` and ``attestation_data_2`` have the same target. - /// - /// Spec v0.3.0 - pub fn is_double_vote(&self, other: &SlashableVoteData, spec: &ChainSpec) -> bool { - self.data.slot.epoch(spec.epoch_length) == other.data.slot.epoch(spec.epoch_length) - } - - /// Check if ``attestation_data_1`` surrounds ``attestation_data_2``. - /// - /// Spec v0.3.0 - pub fn is_surround_vote(&self, other: &SlashableVoteData, spec: &ChainSpec) -> bool { - let source_epoch_1 = self.data.justified_epoch; - let source_epoch_2 = other.data.justified_epoch; - let target_epoch_1 = self.data.slot.epoch(spec.epoch_length); - let target_epoch_2 = other.data.slot.epoch(spec.epoch_length); - - (source_epoch_1 < source_epoch_2) && (target_epoch_2 < target_epoch_1) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::chain_spec::ChainSpec; - use crate::slot_epoch::{Epoch, Slot}; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - - #[test] - pub fn test_is_double_vote_true() { - let spec = ChainSpec::foundation(); - let slashable_vote_first = create_slashable_vote_data(1, 1, &spec); - let slashable_vote_second = create_slashable_vote_data(1, 1, &spec); - - assert_eq!( - slashable_vote_first.is_double_vote(&slashable_vote_second, &spec), - true - ) - } - - #[test] - pub fn test_is_double_vote_false() { - let spec = ChainSpec::foundation(); - let slashable_vote_first = create_slashable_vote_data(1, 1, &spec); - let slashable_vote_second = create_slashable_vote_data(2, 1, &spec); - - assert_eq!( - slashable_vote_first.is_double_vote(&slashable_vote_second, &spec), - false - ); - } - - #[test] - pub fn test_is_surround_vote_true() { - let spec = ChainSpec::foundation(); - let slashable_vote_first = create_slashable_vote_data(2, 1, &spec); - let slashable_vote_second = create_slashable_vote_data(1, 2, &spec); - - assert_eq!( - slashable_vote_first.is_surround_vote(&slashable_vote_second, &spec), - true - ); - } - - #[test] - pub fn test_is_surround_vote_true_realistic() { - let spec = ChainSpec::foundation(); - let slashable_vote_first = create_slashable_vote_data(4, 1, &spec); - let slashable_vote_second = create_slashable_vote_data(3, 2, &spec); - - assert_eq!( - slashable_vote_first.is_surround_vote(&slashable_vote_second, &spec), - true - ); - } - - #[test] - pub fn test_is_surround_vote_false_source_epoch_fails() { - let spec = ChainSpec::foundation(); - let slashable_vote_first = create_slashable_vote_data(2, 2, &spec); - let slashable_vote_second = create_slashable_vote_data(1, 1, &spec); - - assert_eq!( - slashable_vote_first.is_surround_vote(&slashable_vote_second, &spec), - false - ); - } - - #[test] - pub fn test_is_surround_vote_false_target_epoch_fails() { - let spec = ChainSpec::foundation(); - let slashable_vote_first = create_slashable_vote_data(1, 1, &spec); - let slashable_vote_second = create_slashable_vote_data(2, 2, &spec); - - assert_eq!( - slashable_vote_first.is_surround_vote(&slashable_vote_second, &spec), - false - ); - } - - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = SlashableVoteData::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = SlashableVoteData::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } - - fn create_slashable_vote_data( - slot_factor: u64, - justified_epoch: u64, - spec: &ChainSpec, - ) -> SlashableVoteData { - let mut rng = XorShiftRng::from_seed([42; 16]); - let mut slashable_vote = SlashableVoteData::random_for_test(&mut rng); - - slashable_vote.data.slot = Slot::new(slot_factor * spec.epoch_length); - slashable_vote.data.justified_epoch = Epoch::new(justified_epoch); - slashable_vote - } -} diff --git a/eth2/types/src/validator_registry_delta_block.rs b/eth2/types/src/validator_registry_delta_block.rs deleted file mode 100644 index 0746875f0..000000000 --- a/eth2/types/src/validator_registry_delta_block.rs +++ /dev/null @@ -1,59 +0,0 @@ -use crate::{test_utils::TestRandom, Hash256, Slot}; -use bls::PublicKey; -use rand::RngCore; -use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, TreeHash}; -use test_random_derive::TestRandom; - -// The information gathered from the PoW chain validator registration function. -#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, TreeHash, TestRandom)] -pub struct ValidatorRegistryDeltaBlock { - pub latest_registry_delta_root: Hash256, - pub validator_index: u32, - pub pubkey: PublicKey, - pub slot: Slot, - pub flag: u64, -} - -impl Default for ValidatorRegistryDeltaBlock { - /// Yields a "default" `Validator`. Primarily used for testing. - fn default() -> Self { - Self { - latest_registry_delta_root: Hash256::zero(), - validator_index: std::u32::MAX, - pubkey: PublicKey::default(), - slot: Slot::from(std::u64::MAX), - flag: std::u64::MAX, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = ValidatorRegistryDeltaBlock::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = ValidatorRegistryDeltaBlock::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } -} From a1af65ce1a75c03c6c9fead20be88bd729cb74e1 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 4 Mar 2019 17:48:19 +1100 Subject: [PATCH 100/132] Update ChainSpec to v0.4.0 --- .../src/attestation_data_and_custody_bit.rs | 3 +- eth2/types/src/beacon_block.rs | 2 +- eth2/types/src/beacon_state.rs | 20 +++-- eth2/types/src/chain_spec.rs | 73 ++++++++++--------- eth2/types/src/test_utils/mod.rs | 6 ++ eth2/types/src/validator.rs | 15 ++-- 6 files changed, 62 insertions(+), 57 deletions(-) diff --git a/eth2/types/src/attestation_data_and_custody_bit.rs b/eth2/types/src/attestation_data_and_custody_bit.rs index 2f652609e..83018c194 100644 --- a/eth2/types/src/attestation_data_and_custody_bit.rs +++ b/eth2/types/src/attestation_data_and_custody_bit.rs @@ -17,8 +17,7 @@ impl TestRandom for AttestationDataAndCustodyBit { fn random_for_test(rng: &mut T) -> Self { Self { data: <_>::random_for_test(rng), - // TODO: deal with bools - custody_bit: false, + custody_bit: <_>::random_for_test(rng), } } } diff --git a/eth2/types/src/beacon_block.rs b/eth2/types/src/beacon_block.rs index 53b0dac80..d8e50d9ca 100644 --- a/eth2/types/src/beacon_block.rs +++ b/eth2/types/src/beacon_block.rs @@ -1,5 +1,5 @@ use crate::test_utils::TestRandom; -use crate::{BeaconBlockBody, ChainSpec, Eth1Data, Hash256, ProposalSignedData, Slot}; +use crate::{BeaconBlockBody, ChainSpec, Eth1Data, Hash256, Slot}; use bls::Signature; use rand::RngCore; use serde_derive::Serialize; diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index c96beff52..b420f51d5 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -772,7 +772,7 @@ impl BeaconState { /// Update validator registry, activating/exiting validators if possible. /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn update_validator_registry(&mut self, spec: &ChainSpec) { let current_epoch = self.current_epoch(spec); let active_validator_indices = @@ -788,8 +788,8 @@ impl BeaconState { for index in 0..self.validator_registry.len() { let validator = &self.validator_registry[index]; - if (validator.activation_epoch > self.get_entry_exit_effect_epoch(current_epoch, spec)) - && self.validator_balances[index] >= spec.max_deposit_amount + if (validator.activation_epoch == spec.far_future_epoch) + & (self.validator_balances[index] == spec.max_deposit_amount) { balance_churn += self.get_effective_balance(index, spec); if balance_churn > max_balance_churn { @@ -803,9 +803,7 @@ impl BeaconState { for index in 0..self.validator_registry.len() { let validator = &self.validator_registry[index]; - if (validator.exit_epoch > self.get_entry_exit_effect_epoch(current_epoch, spec)) - && validator.status_flags == Some(StatusFlags::InitiatedExit) - { + if (validator.exit_epoch == spec.far_future_epoch) & (validator.initiated_exit) { balance_churn += self.get_effective_balance(index, spec); if balance_churn > max_balance_churn { break; @@ -958,10 +956,9 @@ impl BeaconState { /// Initiate an exit for the validator of the given `index`. /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn initiate_validator_exit(&mut self, validator_index: usize) { - // TODO: the spec does an `|=` here, ensure this isn't buggy. - self.validator_registry[validator_index].status_flags = Some(StatusFlags::InitiatedExit); + self.validator_registry[validator_index].initiated_exit = true; } /// Exit the validator of the given `index`. @@ -1019,9 +1016,10 @@ impl BeaconState { /// Initiate an exit for the validator of the given `index`. /// /// Spec v0.2.0 - pub fn prepare_validator_for_withdrawal(&mut self, validator_index: usize) { + pub fn prepare_validator_for_withdrawal(&mut self, validator_index: usize, spec: &ChainSpec) { //TODO: we're not ANDing here, we're setting. Potentially wrong. - self.validator_registry[validator_index].status_flags = Some(StatusFlags::Withdrawable); + self.validator_registry[validator_index].withdrawable_epoch = + self.current_epoch(spec) + spec.min_validator_withdrawability_delay; } /// Iterate through the validator registry and eject active validators with balance below diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs index 706ad417a..ac5d0a815 100644 --- a/eth2/types/src/chain_spec.rs +++ b/eth2/types/src/chain_spec.rs @@ -5,7 +5,7 @@ const GWEI: u64 = 1_000_000_000; /// Holds all the "constants" for a BeaconChain. /// -/// Spec v0.2.0 +/// Spec v0.4.0 #[derive(PartialEq, Debug, Clone)] pub struct ChainSpec { /* @@ -16,7 +16,7 @@ pub struct ChainSpec { pub max_balance_churn_quotient: u64, pub beacon_chain_shard_number: u64, pub max_indices_per_slashable_vote: u64, - pub max_withdrawals_per_epoch: u64, + pub max_exit_dequeues_per_epoch: u64, pub shuffle_round_count: u8, /* @@ -48,13 +48,13 @@ pub struct ChainSpec { /* * Time parameters */ - pub slot_duration: u64, + pub seconds_per_slot: u64, pub min_attestation_inclusion_delay: u64, - pub epoch_length: u64, - pub seed_lookahead: Epoch, - pub entry_exit_delay: u64, - pub eth1_data_voting_period: u64, - pub min_validator_withdrawal_epochs: Epoch, + pub slots_per_epoch: u64, + pub min_seed_lookahead: Epoch, + pub activation_exit_delay: u64, + pub epochs_per_eth1_voting_period: u64, + pub min_validator_withdrawability_delay: Epoch, /* * State list lengths @@ -62,15 +62,16 @@ pub struct ChainSpec { pub latest_block_roots_length: usize, pub latest_randao_mixes_length: usize, pub latest_index_roots_length: usize, - pub latest_penalized_exit_length: usize, + pub latest_slashed_exit_length: usize, /* * Reward and penalty quotients */ pub base_reward_quotient: u64, pub whistleblower_reward_quotient: u64, - pub includer_reward_quotient: u64, + pub attestation_inclusion_reward_quotient: u64, pub inactivity_penalty_quotient: u64, + pub min_penalty_quotient: u64, /* * Max operations per block @@ -79,7 +80,8 @@ pub struct ChainSpec { pub max_attester_slashings: u64, pub max_attestations: u64, pub max_deposits: u64, - pub max_exits: u64, + pub max_voluntary_exits: u64, + pub max_transfers: u64, /* * Signature domains @@ -89,19 +91,17 @@ pub struct ChainSpec { pub domain_proposal: u64, pub domain_exit: u64, pub domain_randao: u64, + pub domain_transfer: u64, } impl ChainSpec { - /// Returns a `ChainSpec` compatible with the specification from Ethereum Foundation. + /// Returns a `ChainSpec` compatible with the Ethereum Foundation specification. /// - /// Of course, the actual foundation specs are unknown at this point so these are just a rough - /// estimate. - /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn foundation() -> Self { - let genesis_slot = Slot::new(2_u64.pow(19)); - let epoch_length = 64; - let genesis_epoch = genesis_slot.epoch(epoch_length); + let genesis_slot = Slot::new(2_u64.pow(32)); + let slots_per_epoch = 64; + let genesis_epoch = genesis_slot.epoch(slots_per_epoch); Self { /* @@ -112,7 +112,7 @@ impl ChainSpec { max_balance_churn_quotient: 32, beacon_chain_shard_number: u64::max_value(), max_indices_per_slashable_vote: 4_096, - max_withdrawals_per_epoch: 4, + max_exit_dequeues_per_epoch: 4, shuffle_round_count: 90, /* @@ -133,7 +133,7 @@ impl ChainSpec { * Initial Values */ genesis_fork_version: 0, - genesis_slot: Slot::new(2_u64.pow(19)), + genesis_slot, genesis_epoch, genesis_start_shard: 0, far_future_epoch: Epoch::new(u64::max_value()), @@ -144,13 +144,13 @@ impl ChainSpec { /* * Time parameters */ - slot_duration: 6, + seconds_per_slot: 6, min_attestation_inclusion_delay: 4, - epoch_length, - seed_lookahead: Epoch::new(1), - entry_exit_delay: 4, - eth1_data_voting_period: 16, - min_validator_withdrawal_epochs: Epoch::new(256), + slots_per_epoch, + min_seed_lookahead: Epoch::new(1), + activation_exit_delay: 4, + epochs_per_eth1_voting_period: 16, + min_validator_withdrawability_delay: Epoch::new(256), /* * State list lengths @@ -158,15 +158,16 @@ impl ChainSpec { latest_block_roots_length: 8_192, latest_randao_mixes_length: 8_192, latest_index_roots_length: 8_192, - latest_penalized_exit_length: 8_192, + latest_slashed_exit_length: 8_192, /* * Reward and penalty quotients */ base_reward_quotient: 32, whistleblower_reward_quotient: 512, - includer_reward_quotient: 8, + attestation_inclusion_reward_quotient: 8, inactivity_penalty_quotient: 16_777_216, + min_penalty_quotient: 32, /* * Max operations per block @@ -175,7 +176,8 @@ impl ChainSpec { max_attester_slashings: 1, max_attestations: 128, max_deposits: 16, - max_exits: 16, + max_voluntary_exits: 16, + max_transfers: 16, /* * Signature domains @@ -185,6 +187,7 @@ impl ChainSpec { domain_proposal: 2, domain_exit: 3, domain_randao: 4, + domain_transfer: 5, } } } @@ -192,18 +195,18 @@ impl ChainSpec { impl ChainSpec { /// Returns a `ChainSpec` compatible with the specification suitable for 8 validators. /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn few_validators() -> Self { - let genesis_slot = Slot::new(2_u64.pow(19)); - let epoch_length = 8; - let genesis_epoch = genesis_slot.epoch(epoch_length); + let genesis_slot = Slot::new(2_u64.pow(32)); + let slots_per_epoch = 8; + let genesis_epoch = genesis_slot.epoch(slots_per_epoch); Self { shard_count: 8, target_committee_size: 1, genesis_slot, genesis_epoch, - epoch_length, + slots_per_epoch, ..ChainSpec::foundation() } } diff --git a/eth2/types/src/test_utils/mod.rs b/eth2/types/src/test_utils/mod.rs index eb54f2a53..82e060fca 100644 --- a/eth2/types/src/test_utils/mod.rs +++ b/eth2/types/src/test_utils/mod.rs @@ -17,6 +17,12 @@ where fn random_for_test(rng: &mut T) -> Self; } +impl TestRandom for bool { + fn random_for_test(rng: &mut T) -> Self { + (rng.next_u32() % 2) == 1 + } +} + impl TestRandom for u64 { fn random_for_test(rng: &mut T) -> Self { rng.next_u64() diff --git a/eth2/types/src/validator.rs b/eth2/types/src/validator.rs index 5b5dab6bf..43701ca05 100644 --- a/eth2/types/src/validator.rs +++ b/eth2/types/src/validator.rs @@ -1,20 +1,19 @@ use crate::{test_utils::TestRandom, Epoch, Hash256, PublicKey}; use rand::RngCore; use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; use ssz_derive::{Decode, Encode, TreeHash}; use test_random_derive::TestRandom; /// Information about a `BeaconChain` validator. /// /// Spec v0.4.0 -#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, TestRandom)] +#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, TestRandom, TreeHash)] pub struct Validator { pub pubkey: PublicKey, pub withdrawal_credentials: Hash256, pub activation_epoch: Epoch, pub exit_epoch: Epoch, - pub withdrawal_epoch: Epoch, + pub withdrawable_epoch: Epoch, pub initiated_exit: bool, pub slashed: bool, } @@ -32,7 +31,7 @@ impl Validator { /// Returns `true` if the validator is able to withdraw at some epoch. pub fn is_withdrawable_at(&self, epoch: Epoch) -> bool { - self.withdrawal_epoch <= epoch + self.withdrawable_epoch <= epoch } } @@ -44,9 +43,9 @@ impl Default for Validator { withdrawal_credentials: Hash256::default(), activation_epoch: Epoch::from(std::u64::MAX), exit_epoch: Epoch::from(std::u64::MAX), - withdrawal_epoch: Epoch::from(std::u64::MAX), - penalized_epoch: Epoch::from(std::u64::MAX), - status_flags: None, + withdrawable_epoch: Epoch::from(std::u64::MAX), + initiated_exit: false, + slashed: false, } } } @@ -55,7 +54,7 @@ impl Default for Validator { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; + use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_ssz_round_trip() { From 663d39739f735a03a497832d4ea890f153e47416 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 4 Mar 2019 17:51:54 +1100 Subject: [PATCH 101/132] Do project-wide s/epoch_length/slots_per_epoch/g --- beacon_node/beacon_chain/src/beacon_chain.rs | 4 +-- .../beacon_chain/test_harness/README.md | 2 +- .../test_harness/benches/state_transition.rs | 8 ++--- .../specs/validator_registry.yaml | 2 +- .../test_harness/src/beacon_chain_harness.rs | 8 ++--- .../test_harness/src/test_case/config.rs | 4 +-- .../test_harness/src/test_case/mod.rs | 8 ++--- .../test_harness/src/test_case/state_check.rs | 4 +-- .../beacon_chain/test_harness/tests/chain.rs | 2 +- eth2/attester/src/lib.rs | 6 ++-- eth2/attester/src/test_utils/epoch_map.rs | 10 +++---- eth2/block_proposer/src/lib.rs | 8 ++--- .../src/test_utils/epoch_map.rs | 8 ++--- eth2/fork_choice/src/bitwise_lmd_ghost.rs | 2 +- eth2/fork_choice/src/slow_lmd_ghost.rs | 2 +- .../state_processing/src/block_processable.rs | 14 ++++----- .../state_processing/src/epoch_processable.rs | 16 +++++----- eth2/state_processing/src/slot_processable.rs | 2 +- eth2/types/src/beacon_state.rs | 26 ++++++++-------- eth2/types/src/beacon_state/builder.rs | 19 ++++++------ eth2/types/src/beacon_state/epoch_cache.rs | 6 ++-- eth2/types/src/beacon_state/tests.rs | 4 +-- eth2/types/src/proposer_slashing/builder.rs | 4 +-- eth2/types/src/slashable_attestation.rs | 8 ++--- eth2/types/src/slot_epoch.rs | 30 +++++++++---------- eth2/types/src/slot_height.rs | 4 +-- validator_client/src/duties/epoch_duties.rs | 8 ++--- validator_client/src/duties/mod.rs | 4 +-- validator_client/src/main.rs | 2 +- 29 files changed, 113 insertions(+), 112 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index e6fd2a134..e33912f7a 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -312,7 +312,7 @@ where .state .read() .get_block_root( - justified_epoch.start_slot(self.spec.epoch_length), + justified_epoch.start_slot(self.spec.slots_per_epoch), &self.spec, ) .ok_or_else(|| Error::BadRecentBlockRoots)?; @@ -333,7 +333,7 @@ where epoch_boundary_root, shard_block_root: Hash256::zero(), latest_crosslink: Crosslink { - epoch: self.state.read().slot.epoch(self.spec.epoch_length), + epoch: self.state.read().slot.epoch(self.spec.slots_per_epoch), shard_block_root: Hash256::zero(), }, justified_epoch, diff --git a/beacon_node/beacon_chain/test_harness/README.md b/beacon_node/beacon_chain/test_harness/README.md index 12cbbe062..9dfd90d60 100644 --- a/beacon_node/beacon_chain/test_harness/README.md +++ b/beacon_node/beacon_chain/test_harness/README.md @@ -27,7 +27,7 @@ fork: tchaikovsky version: 1.0 test_cases: - config: - epoch_length: 64 + slots_per_epoch: 64 deposits_for_chain_start: 1000 num_slots: 64 skip_slots: [2, 3] diff --git a/beacon_node/beacon_chain/test_harness/benches/state_transition.rs b/beacon_node/beacon_chain/test_harness/benches/state_transition.rs index aa2a858fa..7d1c44653 100644 --- a/beacon_node/beacon_chain/test_harness/benches/state_transition.rs +++ b/beacon_node/beacon_chain/test_harness/benches/state_transition.rs @@ -11,7 +11,7 @@ fn mid_epoch_state_transition(c: &mut Criterion) { let validator_count = 1000; let mut rig = BeaconChainHarness::new(ChainSpec::foundation(), validator_count); - let epoch_depth = (rig.spec.epoch_length * 2) + (rig.spec.epoch_length / 2); + let epoch_depth = (rig.spec.slots_per_epoch * 2) + (rig.spec.slots_per_epoch / 2); for _ in 0..epoch_depth { rig.advance_chain_with_block(); @@ -19,7 +19,7 @@ fn mid_epoch_state_transition(c: &mut Criterion) { let state = rig.beacon_chain.state.read().clone(); - assert!((state.slot + 1) % rig.spec.epoch_length != 0); + assert!((state.slot + 1) % rig.spec.slots_per_epoch != 0); c.bench_function("mid-epoch state transition 10k validators", move |b| { let state = state.clone(); @@ -36,7 +36,7 @@ fn epoch_boundary_state_transition(c: &mut Criterion) { let validator_count = 10000; let mut rig = BeaconChainHarness::new(ChainSpec::foundation(), validator_count); - let epoch_depth = rig.spec.epoch_length * 2; + let epoch_depth = rig.spec.slots_per_epoch * 2; for _ in 0..(epoch_depth - 1) { rig.advance_chain_with_block(); @@ -44,7 +44,7 @@ fn epoch_boundary_state_transition(c: &mut Criterion) { let state = rig.beacon_chain.state.read().clone(); - assert_eq!((state.slot + 1) % rig.spec.epoch_length, 0); + assert_eq!((state.slot + 1) % rig.spec.slots_per_epoch, 0); c.bench( "routines", diff --git a/beacon_node/beacon_chain/test_harness/specs/validator_registry.yaml b/beacon_node/beacon_chain/test_harness/specs/validator_registry.yaml index b7fdda9bf..56dd66558 100644 --- a/beacon_node/beacon_chain/test_harness/specs/validator_registry.yaml +++ b/beacon_node/beacon_chain/test_harness/specs/validator_registry.yaml @@ -5,7 +5,7 @@ fork: tchaikovsky version: 1.0 test_cases: - config: - epoch_length: 64 + slots_per_epoch: 64 deposits_for_chain_start: 1000 num_slots: 64 skip_slots: [2, 3] diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index 2f375f7fa..ed5fddabf 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -125,13 +125,13 @@ impl BeaconChainHarness { let nth_slot = slot - slot - .epoch(self.spec.epoch_length) - .start_slot(self.spec.epoch_length); - let nth_epoch = slot.epoch(self.spec.epoch_length) - self.spec.genesis_epoch; + .epoch(self.spec.slots_per_epoch) + .start_slot(self.spec.slots_per_epoch); + let nth_epoch = slot.epoch(self.spec.slots_per_epoch) - self.spec.genesis_epoch; debug!( "Advancing BeaconChain to slot {}, epoch {} (epoch height: {}, slot {} in epoch.).", slot, - slot.epoch(self.spec.epoch_length), + slot.epoch(self.spec.slots_per_epoch), nth_epoch, nth_slot ); diff --git a/beacon_node/beacon_chain/test_harness/src/test_case/config.rs b/beacon_node/beacon_chain/test_harness/src/test_case/config.rs index 8c88ee5d1..440c76cd2 100644 --- a/beacon_node/beacon_chain/test_harness/src/test_case/config.rs +++ b/beacon_node/beacon_chain/test_harness/src/test_case/config.rs @@ -13,7 +13,7 @@ pub struct Config { /// Initial validators. pub deposits_for_chain_start: usize, /// Number of slots in an epoch. - pub epoch_length: Option, + pub slots_per_epoch: Option, /// Number of slots to build before ending execution. pub num_slots: u64, /// Number of slots that should be skipped due to inactive validator. @@ -34,7 +34,7 @@ impl Config { Self { deposits_for_chain_start: as_usize(&yaml, "deposits_for_chain_start") .expect("Must specify validator count"), - epoch_length: as_u64(&yaml, "epoch_length"), + slots_per_epoch: as_u64(&yaml, "slots_per_epoch"), num_slots: as_u64(&yaml, "num_slots").expect("Must specify `config.num_slots`"), skip_slots: as_vec_u64(yaml, "skip_slots"), deposits: parse_deposits(&yaml), diff --git a/beacon_node/beacon_chain/test_harness/src/test_case/mod.rs b/beacon_node/beacon_chain/test_harness/src/test_case/mod.rs index f6d8d42e8..ae438a5d5 100644 --- a/beacon_node/beacon_chain/test_harness/src/test_case/mod.rs +++ b/beacon_node/beacon_chain/test_harness/src/test_case/mod.rs @@ -54,12 +54,12 @@ impl TestCase { /// Return a `ChainSpec::foundation()`. /// - /// If specified in `config`, returns it with a modified `epoch_length`. + /// If specified in `config`, returns it with a modified `slots_per_epoch`. fn spec(&self) -> ChainSpec { let mut spec = ChainSpec::foundation(); - if let Some(n) = self.config.epoch_length { - spec.epoch_length = n; + if let Some(n) = self.config.slots_per_epoch { + spec.slots_per_epoch = n; } spec @@ -174,7 +174,7 @@ impl TestCase { for state_check in state_checks { let adjusted_state_slot = - state.slot - spec.genesis_epoch.start_slot(spec.epoch_length); + state.slot - spec.genesis_epoch.start_slot(spec.slots_per_epoch); if state_check.slot == adjusted_state_slot { state_check.assert_valid(state, spec); diff --git a/beacon_node/beacon_chain/test_harness/src/test_case/state_check.rs b/beacon_node/beacon_chain/test_harness/src/test_case/state_check.rs index 90c622894..3126dcfea 100644 --- a/beacon_node/beacon_chain/test_harness/src/test_case/state_check.rs +++ b/beacon_node/beacon_chain/test_harness/src/test_case/state_check.rs @@ -36,13 +36,13 @@ impl StateCheck { /// /// Panics with an error message if any test fails. pub fn assert_valid(&self, state: &BeaconState, spec: &ChainSpec) { - let state_epoch = state.slot.epoch(spec.epoch_length); + let state_epoch = state.slot.epoch(spec.slots_per_epoch); info!("Running state check for slot height {}.", self.slot); assert_eq!( self.slot, - state.slot - spec.genesis_epoch.start_slot(spec.epoch_length), + state.slot - spec.genesis_epoch.start_slot(spec.slots_per_epoch), "State slot is invalid." ); diff --git a/beacon_node/beacon_chain/test_harness/tests/chain.rs b/beacon_node/beacon_chain/test_harness/tests/chain.rs index 238e567ad..e72c3a5aa 100644 --- a/beacon_node/beacon_chain/test_harness/tests/chain.rs +++ b/beacon_node/beacon_chain/test_harness/tests/chain.rs @@ -29,7 +29,7 @@ fn it_can_produce_past_first_epoch_boundary() { debug!("Harness built, tests starting.."); - let blocks = harness.spec.epoch_length * 2 + 1; + let blocks = harness.spec.slots_per_epoch * 2 + 1; for i in 0..blocks { harness.advance_chain_with_block(); diff --git a/eth2/attester/src/lib.rs b/eth2/attester/src/lib.rs index 13a1d72bb..5bffd0c50 100644 --- a/eth2/attester/src/lib.rs +++ b/eth2/attester/src/lib.rs @@ -195,9 +195,9 @@ mod tests { let beacon_node = Arc::new(SimulatedBeaconNode::default()); let signer = Arc::new(LocalSigner::new(Keypair::random())); - let mut duties = EpochMap::new(spec.epoch_length); + let mut duties = EpochMap::new(spec.slots_per_epoch); let attest_slot = Slot::new(100); - let attest_epoch = attest_slot / spec.epoch_length; + let attest_epoch = attest_slot / spec.slots_per_epoch; let attest_shard = 12; duties.insert_attestation_shard(attest_slot, attest_shard); duties.set_validator_index(Some(2)); @@ -243,7 +243,7 @@ mod tests { ); // In an epoch without known duties... - let slot = (attest_epoch + 1) * spec.epoch_length; + let slot = (attest_epoch + 1) * spec.slots_per_epoch; slot_clock.set_slot(slot.into()); assert_eq!( attester.poll(), diff --git a/eth2/attester/src/test_utils/epoch_map.rs b/eth2/attester/src/test_utils/epoch_map.rs index f0dc4312e..0b5827d64 100644 --- a/eth2/attester/src/test_utils/epoch_map.rs +++ b/eth2/attester/src/test_utils/epoch_map.rs @@ -3,22 +3,22 @@ use std::collections::HashMap; use types::{Epoch, Slot}; pub struct EpochMap { - epoch_length: u64, + slots_per_epoch: u64, validator_index: Option, map: HashMap, } impl EpochMap { - pub fn new(epoch_length: u64) -> Self { + pub fn new(slots_per_epoch: u64) -> Self { Self { - epoch_length, + slots_per_epoch, validator_index: None, map: HashMap::new(), } } pub fn insert_attestation_shard(&mut self, slot: Slot, shard: u64) { - let epoch = slot.epoch(self.epoch_length); + let epoch = slot.epoch(self.slots_per_epoch); self.map.insert(epoch, (slot, shard)); } @@ -29,7 +29,7 @@ impl EpochMap { impl DutiesReader for EpochMap { fn attestation_shard(&self, slot: Slot) -> Result, DutiesReaderError> { - let epoch = slot.epoch(self.epoch_length); + let epoch = slot.epoch(self.slots_per_epoch); match self.map.get(&epoch) { Some((attest_slot, attest_shard)) if *attest_slot == slot => Ok(Some(*attest_shard)), diff --git a/eth2/block_proposer/src/lib.rs b/eth2/block_proposer/src/lib.rs index 0e66bdc70..cea855627 100644 --- a/eth2/block_proposer/src/lib.rs +++ b/eth2/block_proposer/src/lib.rs @@ -132,7 +132,7 @@ impl BlockProducer Result { let randao_reveal = { // TODO: add domain, etc to this message. Also ensure result matches `into_to_bytes32`. - let message = int_to_bytes32(slot.epoch(self.spec.epoch_length).as_u64()); + let message = int_to_bytes32(slot.epoch(self.spec.slots_per_epoch).as_u64()); match self .signer @@ -233,9 +233,9 @@ mod tests { let beacon_node = Arc::new(SimulatedBeaconNode::default()); let signer = Arc::new(LocalSigner::new(Keypair::random())); - let mut epoch_map = EpochMap::new(spec.epoch_length); + let mut epoch_map = EpochMap::new(spec.slots_per_epoch); let produce_slot = Slot::new(100); - let produce_epoch = produce_slot.epoch(spec.epoch_length); + let produce_epoch = produce_slot.epoch(spec.slots_per_epoch); epoch_map.map.insert(produce_epoch, produce_slot); let epoch_map = Arc::new(epoch_map); @@ -280,7 +280,7 @@ mod tests { ); // In an epoch without known duties... - let slot = (produce_epoch.as_u64() + 1) * spec.epoch_length; + let slot = (produce_epoch.as_u64() + 1) * spec.slots_per_epoch; slot_clock.set_slot(slot); assert_eq!( block_proposer.poll(), diff --git a/eth2/block_proposer/src/test_utils/epoch_map.rs b/eth2/block_proposer/src/test_utils/epoch_map.rs index e9ed9b68a..f7d8dcdcb 100644 --- a/eth2/block_proposer/src/test_utils/epoch_map.rs +++ b/eth2/block_proposer/src/test_utils/epoch_map.rs @@ -3,14 +3,14 @@ use std::collections::HashMap; use types::{Epoch, Slot}; pub struct EpochMap { - epoch_length: u64, + slots_per_epoch: u64, pub map: HashMap, } impl EpochMap { - pub fn new(epoch_length: u64) -> Self { + pub fn new(slots_per_epoch: u64) -> Self { Self { - epoch_length, + slots_per_epoch, map: HashMap::new(), } } @@ -18,7 +18,7 @@ impl EpochMap { impl DutiesReader for EpochMap { fn is_block_production_slot(&self, slot: Slot) -> Result { - let epoch = slot.epoch(self.epoch_length); + let epoch = slot.epoch(self.slots_per_epoch); match self.map.get(&epoch) { Some(s) if *s == slot => Ok(true), Some(s) if *s != slot => Ok(false), diff --git a/eth2/fork_choice/src/bitwise_lmd_ghost.rs b/eth2/fork_choice/src/bitwise_lmd_ghost.rs index 60aa38fe7..c13aacbf4 100644 --- a/eth2/fork_choice/src/bitwise_lmd_ghost.rs +++ b/eth2/fork_choice/src/bitwise_lmd_ghost.rs @@ -95,7 +95,7 @@ where let active_validator_indices = get_active_validator_indices( ¤t_state.validator_registry[..], - block_slot.epoch(spec.epoch_length), + block_slot.epoch(spec.slots_per_epoch), ); for index in active_validator_indices { diff --git a/eth2/fork_choice/src/slow_lmd_ghost.rs b/eth2/fork_choice/src/slow_lmd_ghost.rs index 3aafb3924..ab4cd2ada 100644 --- a/eth2/fork_choice/src/slow_lmd_ghost.rs +++ b/eth2/fork_choice/src/slow_lmd_ghost.rs @@ -64,7 +64,7 @@ where let active_validator_indices = get_active_validator_indices( ¤t_state.validator_registry[..], - block_slot.epoch(spec.epoch_length), + block_slot.epoch(spec.slots_per_epoch), ); for index in active_validator_indices { diff --git a/eth2/state_processing/src/block_processable.rs b/eth2/state_processing/src/block_processable.rs index 32327aad3..ee8bc7c33 100644 --- a/eth2/state_processing/src/block_processable.rs +++ b/eth2/state_processing/src/block_processable.rs @@ -185,7 +185,7 @@ fn per_block_processing_signature_optional( proposer_slashing .proposal_data_1 .slot - .epoch(spec.epoch_length), + .epoch(spec.slots_per_epoch), spec.domain_proposal ) ), @@ -201,7 +201,7 @@ fn per_block_processing_signature_optional( proposer_slashing .proposal_data_2 .slot - .epoch(spec.epoch_length), + .epoch(spec.slots_per_epoch), spec.domain_proposal ) ), @@ -341,14 +341,14 @@ fn validate_attestation_signature_optional( ) -> Result<(), AttestationValidationError> { trace!( "validate_attestation_signature_optional: attestation epoch: {}", - attestation.data.slot.epoch(spec.epoch_length) + attestation.data.slot.epoch(spec.slots_per_epoch) ); ensure!( attestation.data.slot + spec.min_attestation_inclusion_delay <= state.slot, AttestationValidationError::IncludedTooEarly ); ensure!( - attestation.data.slot + spec.epoch_length >= state.slot, + attestation.data.slot + spec.slots_per_epoch >= state.slot, AttestationValidationError::IncludedTooLate ); if attestation.data.slot >= state.current_epoch_start_slot(spec) { @@ -369,7 +369,7 @@ fn validate_attestation_signature_optional( attestation .data .justified_epoch - .start_slot(spec.epoch_length), + .start_slot(spec.slots_per_epoch), &spec ) .ok_or(AttestationValidationError::NoBlockRoot)?, @@ -377,7 +377,7 @@ fn validate_attestation_signature_optional( ); let potential_crosslink = Crosslink { shard_block_root: attestation.data.shard_block_root, - epoch: attestation.data.slot.epoch(spec.epoch_length), + epoch: attestation.data.slot.epoch(spec.slots_per_epoch), }; ensure!( (attestation.data.latest_crosslink @@ -407,7 +407,7 @@ fn validate_attestation_signature_optional( PHASE_0_CUSTODY_BIT, get_domain( &state.fork, - attestation.data.slot.epoch(spec.epoch_length), + attestation.data.slot.epoch(spec.slots_per_epoch), spec.domain_attestation, ) ), diff --git a/eth2/state_processing/src/epoch_processable.rs b/eth2/state_processing/src/epoch_processable.rs index 0ecd1bbd1..47336d3f8 100644 --- a/eth2/state_processing/src/epoch_processable.rs +++ b/eth2/state_processing/src/epoch_processable.rs @@ -76,7 +76,7 @@ impl EpochProcessable for BeaconState { */ let active_validator_indices = get_active_validator_indices( &self.validator_registry, - self.slot.epoch(spec.epoch_length), + self.slot.epoch(spec.slots_per_epoch), ); let current_total_balance = self.get_total_balance(&active_validator_indices[..], spec); @@ -90,7 +90,7 @@ impl EpochProcessable for BeaconState { .latest_attestations .par_iter() .filter(|a| { - (a.data.slot / spec.epoch_length).epoch(spec.epoch_length) + (a.data.slot / spec.slots_per_epoch).epoch(spec.slots_per_epoch) == self.current_epoch(spec) }) .collect(); @@ -137,7 +137,7 @@ impl EpochProcessable for BeaconState { .par_iter() .filter(|a| { //TODO: ensure these saturating subs are correct. - (a.data.slot / spec.epoch_length).epoch(spec.epoch_length) + (a.data.slot / spec.slots_per_epoch).epoch(spec.slots_per_epoch) == self.previous_epoch(spec) }) .collect(); @@ -320,12 +320,12 @@ impl EpochProcessable for BeaconState { let mut winning_root_for_shards: HashMap> = HashMap::new(); - // for slot in self.slot.saturating_sub(2 * spec.epoch_length)..self.slot { - for slot in self.previous_epoch(spec).slot_iter(spec.epoch_length) { + // for slot in self.slot.saturating_sub(2 * spec.slots_per_epoch)..self.slot { + for slot in self.previous_epoch(spec).slot_iter(spec.slots_per_epoch) { trace!( "Finding winning root for slot: {} (epoch: {})", slot, - slot.epoch(spec.epoch_length) + slot.epoch(spec.slots_per_epoch) ); // Clone is used to remove the borrow. It becomes an issue later when trying to mutate @@ -506,7 +506,7 @@ impl EpochProcessable for BeaconState { /* * Crosslinks */ - for slot in self.previous_epoch(spec).slot_iter(spec.epoch_length) { + for slot in self.previous_epoch(spec).slot_iter(spec.slots_per_epoch) { // Clone is used to remove the borrow. It becomes an issue later when trying to mutate // `self.balances`. let crosslink_committees_at_slot = @@ -615,7 +615,7 @@ impl EpochProcessable for BeaconState { self.latest_attestations = self .latest_attestations .iter() - .filter(|a| a.data.slot.epoch(spec.epoch_length) >= current_epoch) + .filter(|a| a.data.slot.epoch(spec.slots_per_epoch) >= current_epoch) .cloned() .collect(); diff --git a/eth2/state_processing/src/slot_processable.rs b/eth2/state_processing/src/slot_processable.rs index 6017f4c0a..8d6506c36 100644 --- a/eth2/state_processing/src/slot_processable.rs +++ b/eth2/state_processing/src/slot_processable.rs @@ -24,7 +24,7 @@ where previous_block_root: Hash256, spec: &ChainSpec, ) -> Result<(), Error> { - if (self.slot + 1) % spec.epoch_length == 0 { + if (self.slot + 1) % spec.slots_per_epoch == 0 { self.per_epoch_processing(spec)?; self.advance_caches(); } diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index b420f51d5..75beb79ca 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -337,7 +337,7 @@ impl BeaconState { /// /// Spec v0.2.0 pub fn current_epoch(&self, spec: &ChainSpec) -> Epoch { - self.slot.epoch(spec.epoch_length) + self.slot.epoch(spec.slots_per_epoch) } /// The epoch prior to `self.current_epoch()`. @@ -363,14 +363,14 @@ impl BeaconState { /// /// Spec v0.2.0 pub fn current_epoch_start_slot(&self, spec: &ChainSpec) -> Slot { - self.current_epoch(spec).start_slot(spec.epoch_length) + self.current_epoch(spec).start_slot(spec.slots_per_epoch) } /// The first slot of the epoch preceeding the one corresponding to `self.slot`. /// /// Spec v0.2.0 pub fn previous_epoch_start_slot(&self, spec: &ChainSpec) -> Slot { - self.previous_epoch(spec).start_slot(spec.epoch_length) + self.previous_epoch(spec).start_slot(spec.slots_per_epoch) } /// Return the number of committees in one epoch. @@ -386,10 +386,10 @@ impl BeaconState { std::cmp::max( 1, std::cmp::min( - spec.shard_count / spec.epoch_length, - active_validator_count as u64 / spec.epoch_length / spec.target_committee_size, + spec.shard_count / spec.slots_per_epoch, + active_validator_count as u64 / spec.slots_per_epoch / spec.target_committee_size, ), - ) * spec.epoch_length + ) * spec.slots_per_epoch } /// Shuffle ``validators`` into crosslink committees seeded by ``seed`` and ``epoch``. @@ -520,11 +520,11 @@ impl BeaconState { slot: Slot, spec: &ChainSpec, ) -> Result<&CrosslinkCommittees, Error> { - let epoch = slot.epoch(spec.epoch_length); + let epoch = slot.epoch(spec.slots_per_epoch); let relative_epoch = self.relative_epoch(epoch, spec)?; let cache = self.cache(relative_epoch)?; - let slot_offset = slot - epoch.start_slot(spec.epoch_length); + let slot_offset = slot - epoch.start_slot(spec.slots_per_epoch); Ok(&cache.committees[slot_offset.as_usize()]) } @@ -565,7 +565,7 @@ impl BeaconState { registry_change: bool, spec: &ChainSpec, ) -> Result<(u64, Hash256, Epoch, u64), Error> { - let epoch = slot.epoch(spec.epoch_length); + let epoch = slot.epoch(spec.slots_per_epoch); let current_epoch = self.current_epoch(spec); let previous_epoch = self.previous_epoch(spec); let next_epoch = self.next_epoch(spec); @@ -639,8 +639,8 @@ impl BeaconState { let (committees_per_epoch, _seed, _shuffling_epoch, shuffling_start_shard) = self.get_committee_params_at_slot(slot, registry_change, spec)?; - let offset = slot.as_u64() % spec.epoch_length; - let committees_per_slot = committees_per_epoch / spec.epoch_length; + let offset = slot.as_u64() % spec.slots_per_epoch; + let committees_per_slot = committees_per_epoch / spec.slots_per_epoch; let slot_start_shard = (shuffling_start_shard + committees_per_slot * offset) % spec.shard_count; @@ -835,7 +835,7 @@ impl BeaconState { proof_of_possession.verify( &proof_of_possession_data.hash_tree_root(), self.fork - .get_domain(self.slot.epoch(spec.epoch_length), spec.domain_deposit), + .get_domain(self.slot.epoch(spec.slots_per_epoch), spec.domain_deposit), &pubkey, ) } @@ -1296,7 +1296,7 @@ impl BeaconState { bitfield: &Bitfield, spec: &ChainSpec, ) -> Result, Error> { - let epoch = attestation_data.slot.epoch(spec.epoch_length); + let epoch = attestation_data.slot.epoch(spec.slots_per_epoch); let relative_epoch = self.relative_epoch(epoch, spec)?; let cache = self.cache(relative_epoch)?; diff --git a/eth2/types/src/beacon_state/builder.rs b/eth2/types/src/beacon_state/builder.rs index 02886a86e..4acd172f5 100644 --- a/eth2/types/src/beacon_state/builder.rs +++ b/eth2/types/src/beacon_state/builder.rs @@ -137,7 +137,7 @@ impl BeaconStateBuilder { pub fn teleport_to_end_of_epoch(&mut self, epoch: Epoch) { let state = self.state.as_mut().expect("Genesis required"); - let slot = epoch.end_slot(self.spec.epoch_length); + let slot = epoch.end_slot(self.spec.slots_per_epoch); state.slot = slot; state.validator_registry_update_epoch = epoch - 1; @@ -171,11 +171,11 @@ impl BeaconStateBuilder { let current_epoch = state.current_epoch(&self.spec); let previous_epoch = state.previous_epoch(&self.spec); let current_epoch_depth = - (state.slot - current_epoch.end_slot(self.spec.epoch_length)).as_usize(); + (state.slot - current_epoch.end_slot(self.spec.slots_per_epoch)).as_usize(); - let previous_epoch_slots = previous_epoch.slot_iter(self.spec.epoch_length); + let previous_epoch_slots = previous_epoch.slot_iter(self.spec.slots_per_epoch); let current_epoch_slots = current_epoch - .slot_iter(self.spec.epoch_length) + .slot_iter(self.spec.slots_per_epoch) .take(current_epoch_depth); for slot in previous_epoch_slots.chain(current_epoch_slots) { @@ -219,7 +219,8 @@ fn committee_to_pending_attestation( custody_bitfield.set(i, true); } - let is_previous_epoch = state.slot.epoch(spec.epoch_length) != slot.epoch(spec.epoch_length); + let is_previous_epoch = + state.slot.epoch(spec.slots_per_epoch) != slot.epoch(spec.slots_per_epoch); let justified_epoch = if is_previous_epoch { state.previous_justified_epoch @@ -229,16 +230,16 @@ fn committee_to_pending_attestation( let epoch_boundary_root = if is_previous_epoch { *state - .get_block_root(previous_epoch.start_slot(spec.epoch_length), spec) + .get_block_root(previous_epoch.start_slot(spec.slots_per_epoch), spec) .unwrap() } else { *state - .get_block_root(current_epoch.start_slot(spec.epoch_length), spec) + .get_block_root(current_epoch.start_slot(spec.slots_per_epoch), spec) .unwrap() }; let justified_block_root = *state - .get_block_root(justified_epoch.start_slot(spec.epoch_length), &spec) + .get_block_root(justified_epoch.start_slot(spec.slots_per_epoch), &spec) .unwrap(); PendingAttestation { @@ -250,7 +251,7 @@ fn committee_to_pending_attestation( epoch_boundary_root, shard_block_root: Hash256::zero(), latest_crosslink: Crosslink { - epoch: slot.epoch(spec.epoch_length), + epoch: slot.epoch(spec.slots_per_epoch), shard_block_root: Hash256::zero(), }, justified_epoch, diff --git a/eth2/types/src/beacon_state/epoch_cache.rs b/eth2/types/src/beacon_state/epoch_cache.rs index ee3a67813..bbc991646 100644 --- a/eth2/types/src/beacon_state/epoch_cache.rs +++ b/eth2/types/src/beacon_state/epoch_cache.rs @@ -32,14 +32,14 @@ impl EpochCache { spec: &ChainSpec, ) -> Result { let mut epoch_committees: Vec = - Vec::with_capacity(spec.epoch_length as usize); + Vec::with_capacity(spec.slots_per_epoch as usize); let mut attestation_duty_map: AttestationDutyMap = HashMap::new(); let mut shard_committee_index_map: ShardCommitteeIndexMap = HashMap::new(); let shuffling = - state.get_shuffling_for_slot(epoch.start_slot(spec.epoch_length), false, spec)?; + state.get_shuffling_for_slot(epoch.start_slot(spec.slots_per_epoch), false, spec)?; - for (epoch_committeess_index, slot) in epoch.slot_iter(spec.epoch_length).enumerate() { + for (epoch_committeess_index, slot) in epoch.slot_iter(spec.slots_per_epoch).enumerate() { let slot_committees = state.calculate_crosslink_committees_at_slot( slot, false, diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs index bb8561511..40bfd146c 100644 --- a/eth2/types/src/beacon_state/tests.rs +++ b/eth2/types/src/beacon_state/tests.rs @@ -35,8 +35,8 @@ pub fn get_attestation_participants_consistency() { for slot in state .slot - .epoch(spec.epoch_length) - .slot_iter(spec.epoch_length) + .epoch(spec.slots_per_epoch) + .slot_iter(spec.slots_per_epoch) { let committees = state.get_crosslink_committees_at_slot(slot, &spec).unwrap(); diff --git a/eth2/types/src/proposer_slashing/builder.rs b/eth2/types/src/proposer_slashing/builder.rs index 66f74a2e2..8dc1abfbe 100644 --- a/eth2/types/src/proposer_slashing/builder.rs +++ b/eth2/types/src/proposer_slashing/builder.rs @@ -38,14 +38,14 @@ impl ProposerSlashingBuilder { proposal_1.signature = { let message = proposal_1.signed_root(); - let epoch = slot.epoch(spec.epoch_length); + let epoch = slot.epoch(spec.slots_per_epoch); let domain = spec.domain_proposal; signer(proposer_index, &message[..], epoch, domain) }; proposal_2.signature = { let message = proposal_2.signed_root(); - let epoch = slot.epoch(spec.epoch_length); + let epoch = slot.epoch(spec.slots_per_epoch); let domain = spec.domain_proposal; signer(proposer_index, &message[..], epoch, domain) }; diff --git a/eth2/types/src/slashable_attestation.rs b/eth2/types/src/slashable_attestation.rs index 9f2ccab60..20ba76cdb 100644 --- a/eth2/types/src/slashable_attestation.rs +++ b/eth2/types/src/slashable_attestation.rs @@ -24,7 +24,7 @@ impl SlashableAttestation { /// /// Spec v0.4.0 pub fn is_double_vote(&self, other: &SlashableAttestation, spec: &ChainSpec) -> bool { - self.data.slot.epoch(spec.epoch_length) == other.data.slot.epoch(spec.epoch_length) + self.data.slot.epoch(spec.slots_per_epoch) == other.data.slot.epoch(spec.slots_per_epoch) } /// Check if ``attestation_data_1`` surrounds ``attestation_data_2``. @@ -33,8 +33,8 @@ impl SlashableAttestation { pub fn is_surround_vote(&self, other: &SlashableAttestation, spec: &ChainSpec) -> bool { let source_epoch_1 = self.data.justified_epoch; let source_epoch_2 = other.data.justified_epoch; - let target_epoch_1 = self.data.slot.epoch(spec.epoch_length); - let target_epoch_2 = other.data.slot.epoch(spec.epoch_length); + let target_epoch_1 = self.data.slot.epoch(spec.slots_per_epoch); + let target_epoch_2 = other.data.slot.epoch(spec.slots_per_epoch); (source_epoch_1 < source_epoch_2) & (target_epoch_2 < target_epoch_1) } @@ -151,7 +151,7 @@ mod tests { let mut rng = XorShiftRng::from_seed([42; 16]); let mut slashable_vote = SlashableAttestation::random_for_test(&mut rng); - slashable_vote.data.slot = Slot::new(slot_factor * spec.epoch_length); + slashable_vote.data.slot = Slot::new(slot_factor * spec.slots_per_epoch); slashable_vote.data.justified_epoch = Epoch::new(justified_epoch); slashable_vote } diff --git a/eth2/types/src/slot_epoch.rs b/eth2/types/src/slot_epoch.rs index ff4fd5b9b..7753027a6 100644 --- a/eth2/types/src/slot_epoch.rs +++ b/eth2/types/src/slot_epoch.rs @@ -35,8 +35,8 @@ impl Slot { Slot(slot) } - pub fn epoch(self, epoch_length: u64) -> Epoch { - Epoch::from(self.0 / epoch_length) + pub fn epoch(self, slots_per_epoch: u64) -> Epoch { + Epoch::from(self.0 / slots_per_epoch) } pub fn height(self, genesis_slot: Slot) -> SlotHeight { @@ -57,24 +57,24 @@ impl Epoch { Epoch(u64::max_value()) } - pub fn start_slot(self, epoch_length: u64) -> Slot { - Slot::from(self.0.saturating_mul(epoch_length)) + pub fn start_slot(self, slots_per_epoch: u64) -> Slot { + Slot::from(self.0.saturating_mul(slots_per_epoch)) } - pub fn end_slot(self, epoch_length: u64) -> Slot { + pub fn end_slot(self, slots_per_epoch: u64) -> Slot { Slot::from( self.0 .saturating_add(1) - .saturating_mul(epoch_length) + .saturating_mul(slots_per_epoch) .saturating_sub(1), ) } - pub fn slot_iter(&self, epoch_length: u64) -> SlotIter { + pub fn slot_iter(&self, slots_per_epoch: u64) -> SlotIter { SlotIter { current_iteration: 0, epoch: self, - epoch_length, + slots_per_epoch, } } } @@ -82,17 +82,17 @@ impl Epoch { pub struct SlotIter<'a> { current_iteration: u64, epoch: &'a Epoch, - epoch_length: u64, + slots_per_epoch: u64, } impl<'a> Iterator for SlotIter<'a> { type Item = Slot; fn next(&mut self) -> Option { - if self.current_iteration >= self.epoch_length { + if self.current_iteration >= self.slots_per_epoch { None } else { - let start_slot = self.epoch.start_slot(self.epoch_length); + let start_slot = self.epoch.start_slot(self.slots_per_epoch); let previous = self.current_iteration; self.current_iteration += 1; Some(start_slot + previous) @@ -119,18 +119,18 @@ mod epoch_tests { #[test] fn slot_iter() { - let epoch_length = 8; + let slots_per_epoch = 8; let epoch = Epoch::new(0); let mut slots = vec![]; - for slot in epoch.slot_iter(epoch_length) { + for slot in epoch.slot_iter(slots_per_epoch) { slots.push(slot); } - assert_eq!(slots.len(), epoch_length as usize); + assert_eq!(slots.len(), slots_per_epoch as usize); - for i in 0..epoch_length { + for i in 0..slots_per_epoch { assert_eq!(Slot::from(i), slots[i as usize]) } } diff --git a/eth2/types/src/slot_height.rs b/eth2/types/src/slot_height.rs index f9370f485..1739227a4 100644 --- a/eth2/types/src/slot_height.rs +++ b/eth2/types/src/slot_height.rs @@ -23,8 +23,8 @@ impl SlotHeight { Slot::from(self.0.saturating_add(genesis_slot.as_u64())) } - pub fn epoch(self, genesis_slot: u64, epoch_length: u64) -> Epoch { - Epoch::from(self.0.saturating_add(genesis_slot) / epoch_length) + pub fn epoch(self, genesis_slot: u64, slots_per_epoch: u64) -> Epoch { + Epoch::from(self.0.saturating_add(genesis_slot) / slots_per_epoch) } pub fn max_value() -> SlotHeight { diff --git a/validator_client/src/duties/epoch_duties.rs b/validator_client/src/duties/epoch_duties.rs index 54a882f8d..d1bbfa156 100644 --- a/validator_client/src/duties/epoch_duties.rs +++ b/validator_client/src/duties/epoch_duties.rs @@ -32,14 +32,14 @@ pub enum EpochDutiesMapError { /// Maps an `epoch` to some `EpochDuties` for a single validator. pub struct EpochDutiesMap { - pub epoch_length: u64, + pub slots_per_epoch: u64, pub map: RwLock>, } impl EpochDutiesMap { - pub fn new(epoch_length: u64) -> Self { + pub fn new(slots_per_epoch: u64) -> Self { Self { - epoch_length, + slots_per_epoch, map: RwLock::new(HashMap::new()), } } @@ -67,7 +67,7 @@ impl EpochDutiesMap { impl DutiesReader for EpochDutiesMap { fn is_block_production_slot(&self, slot: Slot) -> Result { - let epoch = slot.epoch(self.epoch_length); + let epoch = slot.epoch(self.slots_per_epoch); let map = self.map.read().map_err(|_| DutiesReaderError::Poisoned)?; let duties = map diff --git a/validator_client/src/duties/mod.rs b/validator_client/src/duties/mod.rs index febab4755..29bd81d0a 100644 --- a/validator_client/src/duties/mod.rs +++ b/validator_client/src/duties/mod.rs @@ -61,7 +61,7 @@ impl DutiesManager { .map_err(|_| Error::SlotClockError)? .ok_or(Error::SlotUnknowable)?; - let epoch = slot.epoch(self.spec.epoch_length); + let epoch = slot.epoch(self.spec.slots_per_epoch); if let Some(duties) = self.beacon_node.request_shuffling(epoch, &self.pubkey)? { // If these duties were known, check to see if they're updates or identical. @@ -112,7 +112,7 @@ mod tests { #[test] pub fn polling() { let spec = Arc::new(ChainSpec::foundation()); - let duties_map = Arc::new(EpochDutiesMap::new(spec.epoch_length)); + let duties_map = Arc::new(EpochDutiesMap::new(spec.slots_per_epoch)); let keypair = Keypair::random(); let slot_clock = Arc::new(TestingSlotClock::new(0)); let beacon_node = Arc::new(TestBeaconNode::default()); diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index c835300b5..b4d8ae5db 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -111,7 +111,7 @@ fn main() { for keypair in keypairs { info!(log, "Starting validator services"; "validator" => keypair.pk.concatenated_hex_id()); - let duties_map = Arc::new(EpochDutiesMap::new(spec.epoch_length)); + let duties_map = Arc::new(EpochDutiesMap::new(spec.slots_per_epoch)); // Spawn a new thread to maintain the validator's `EpochDuties`. let duties_manager_thread = { From 0eddfa5556393a4ae689b561bea13f58fb6ba17b Mon Sep 17 00:00:00 2001 From: Luke Schoen Date: Mon, 4 Mar 2019 14:46:56 +0100 Subject: [PATCH 102/132] docs: Fix typos --- docs/onboarding.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/onboarding.md b/docs/onboarding.md index 8af3b0a83..275f95484 100644 --- a/docs/onboarding.md +++ b/docs/onboarding.md @@ -122,7 +122,7 @@ project. * **Module**: A collection of items: functions, structs, traits, and even other modules. Modules allow you to hierarchically split code into logical units and manage visibility. -* **Attribute**: Metadaata applied to some module, crate or item. +* **Attribute**: Metadata applied to some module, crate or item. * **Macros**: Macros are powerful meta-programming statements that get expanded into source code that gets compiled with the rest of the code (Unlike `C` macros that are pre-processed, Rust macros form an Abstract Syntax Tree). @@ -185,7 +185,7 @@ check your code. | Function / Method | ``snake_case`` | | Macro Names | ``snake_case`` | | Constants | ``SCREAMING_SNAKE_CASE`` | -| Forbidden name | Trialing Underscore: ``name_`` | +| Forbidden name | Trailing Underscore: ``name_`` | Other general rust docs: From 2b1549358ff2012eb5623e0e00fc090d3a9e4a3d Mon Sep 17 00:00:00 2001 From: Luke Schoen Date: Mon, 4 Mar 2019 14:53:52 +0100 Subject: [PATCH 103/132] docs: Fix typo --- docs/lighthouse.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/lighthouse.md b/docs/lighthouse.md index 8ca2387f8..16da13b56 100644 --- a/docs/lighthouse.md +++ b/docs/lighthouse.md @@ -67,7 +67,7 @@ into individual crates wherever possible. Generally, tests can be kept in the same file, as is typical in Rust. Integration tests should be placed in the `tests` directory in the crate's -root. Particularity large (line-count) tests should be placed into a separate +root. Particularly large (line-count) tests should be placed into a separate file. A function is not considered complete until a test exists for it. We produce From 262e9cf0bcc772e54f312b3d6055448dd9058145 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 5 Mar 2019 09:51:29 +1100 Subject: [PATCH 104/132] Additional spec updates --- .../state_processing/src/epoch_processable.rs | 18 ++--- eth2/types/src/beacon_state.rs | 77 ++++++++++++------- eth2/types/src/beacon_state/builder.rs | 8 +- eth2/types/src/chain_spec.rs | 4 +- 4 files changed, 63 insertions(+), 44 deletions(-) diff --git a/eth2/state_processing/src/epoch_processable.rs b/eth2/state_processing/src/epoch_processable.rs index 47336d3f8..0d6ca8038 100644 --- a/eth2/state_processing/src/epoch_processable.rs +++ b/eth2/state_processing/src/epoch_processable.rs @@ -555,15 +555,15 @@ impl EpochProcessable for BeaconState { /* * Validator Registry */ - self.previous_calculation_epoch = self.current_calculation_epoch; + self.previous_shuffling_epoch = self.current_shuffling_epoch; self.previous_epoch_start_shard = self.current_epoch_start_shard; debug!( - "setting previous_epoch_seed to : {}", - self.current_epoch_seed + "setting previous_shuffling_seed to : {}", + self.current_shuffling_seed ); - self.previous_epoch_seed = self.current_epoch_seed; + self.previous_shuffling_seed = self.current_shuffling_seed; let should_update_validator_registy = if self.finalized_epoch > self.validator_registry_update_epoch @@ -580,11 +580,11 @@ impl EpochProcessable for BeaconState { trace!("updating validator registry."); self.update_validator_registry(spec); - self.current_calculation_epoch = next_epoch; + self.current_shuffling_epoch = next_epoch; self.current_epoch_start_shard = (self.current_epoch_start_shard + self.get_current_epoch_committee_count(spec) as u64) % spec.shard_count; - self.current_epoch_seed = self.generate_seed(self.current_calculation_epoch, spec)? + self.current_shuffling_seed = self.generate_seed(self.current_shuffling_epoch, spec)? } else { trace!("not updating validator registry."); let epochs_since_last_registry_update = @@ -592,9 +592,9 @@ impl EpochProcessable for BeaconState { if (epochs_since_last_registry_update > 1) & epochs_since_last_registry_update.is_power_of_two() { - self.current_calculation_epoch = next_epoch; - self.current_epoch_seed = - self.generate_seed(self.current_calculation_epoch, spec)? + self.current_shuffling_epoch = next_epoch; + self.current_shuffling_seed = + self.generate_seed(self.current_shuffling_epoch, spec)? } } diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 75beb79ca..43fd72232 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -39,8 +39,10 @@ pub enum Error { EpochOutOfBounds, /// The supplied shard is unknown. It may be larger than the maximum shard count, or not in a /// committee for the given slot. + SlotOutOfBounds, ShardOutOfBounds, UnableToShuffle, + UnknownValidator, InsufficientRandaoMixes, InsufficientValidators, InsufficientBlockRoots, @@ -172,7 +174,7 @@ impl BeaconState { spec.zero_hash; spec.latest_active_index_roots_length as usize ], - latest_slashed_balances: vec![0; spec.latest_penalized_exit_length as usize], + latest_slashed_balances: vec![0; spec.latest_slashed_exit_length as usize], latest_attestations: vec![], batched_block_roots: vec![], @@ -711,7 +713,7 @@ impl BeaconState { /// Process the penalties and prepare the validators who are eligible to withdrawal. /// /// Spec v0.2.0 - pub fn process_penalties_and_exits(&mut self, spec: &ChainSpec) { + pub fn process_slashings(&mut self, spec: &ChainSpec) { let current_epoch = self.current_epoch(spec); let active_validator_indices = get_active_validator_indices(&self.validator_registry, current_epoch); @@ -721,13 +723,12 @@ impl BeaconState { let validator = &self.validator_registry[index]; if current_epoch - == validator.penalized_epoch + Epoch::from(spec.latest_penalized_exit_length / 2) + == validator.penalized_epoch + Epoch::from(spec.latest_slashed_exit_length / 2) { - let epoch_index: usize = - current_epoch.as_usize() % spec.latest_penalized_exit_length; + let epoch_index: usize = current_epoch.as_usize() % spec.latest_slashed_exit_length; let total_at_start = self.latest_slashed_balances - [(epoch_index + 1) % spec.latest_penalized_exit_length]; + [(epoch_index + 1) % spec.latest_slashed_exit_length]; let total_at_end = self.latest_slashed_balances[epoch_index]; let total_penalities = total_at_end.saturating_sub(total_at_start); let penalty = self.get_effective_balance(index, spec) @@ -736,12 +737,22 @@ impl BeaconState { safe_sub_assign!(self.validator_balances[index], penalty); } } + } + + /// Process the penalties and prepare the validators who are eligible to withdrawal. + /// + /// Spec v0.2.0 + pub fn process_exit_queue(&mut self, spec: &ChainSpec) { + let current_epoch = self.current_epoch(spec); + let active_validator_indices = + get_active_validator_indices(&self.validator_registry, current_epoch); + let total_balance = self.get_total_balance(&active_validator_indices[..], spec); let eligible = |index: usize| { let validator = &self.validator_registry[index]; if validator.penalized_epoch <= current_epoch { - let penalized_withdrawal_epochs = spec.latest_penalized_exit_length / 2; + let penalized_withdrawal_epochs = spec.latest_slashed_exit_length / 2; current_epoch >= validator.penalized_epoch + penalized_withdrawal_epochs as u64 } else { current_epoch >= validator.exit_epoch + spec.min_validator_withdrawal_epochs @@ -881,7 +892,7 @@ impl BeaconState { /// this hashmap, each call to `process_deposits` requires an iteration though /// `self.validator_registry`. This becomes highly inefficient at scale. /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn process_deposit( &mut self, pubkey: PublicKey, @@ -893,15 +904,9 @@ impl BeaconState { ) -> Result { // TODO: update proof of possession to function written above ( // requires bls::create_proof_of_possession to be updated + // // https://github.com/sigp/lighthouse/issues/239 - if !verify_proof_of_possession(&proof_of_possession, &pubkey) - //if !self.validate_proof_of_possession( - // pubkey.clone(), - // proof_of_possession, - // withdrawal_credentials, - // &spec, - // ) - { + if !verify_proof_of_possession(&proof_of_possession, &pubkey) { return Err(()); } @@ -926,9 +931,9 @@ impl BeaconState { withdrawal_credentials, activation_epoch: spec.far_future_epoch, exit_epoch: spec.far_future_epoch, - withdrawal_epoch: spec.far_future_epoch, - penalized_epoch: spec.far_future_epoch, - status_flags: None, + withdrawable_epoch: spec.far_future_epoch, + initiated_exit: false, + slashed: false, }; self.validator_registry.push(validator); self.validator_balances.push(amount); @@ -977,22 +982,32 @@ impl BeaconState { self.get_entry_exit_effect_epoch(current_epoch, spec); } - /// Penalize the validator of the given ``index``. + /// Slash the validator with index ``index``. /// - /// Exits the validator and assigns its effective balance to the block producer for this - /// state. - /// - /// Spec v0.2.0 - pub fn penalize_validator( + /// Spec v0.4.0 + pub fn slash_validator( &mut self, validator_index: usize, spec: &ChainSpec, ) -> Result<(), Error> { - self.exit_validator(validator_index, spec); let current_epoch = self.current_epoch(spec); - self.latest_slashed_balances - [current_epoch.as_usize() % spec.latest_penalized_exit_length] += + let validator = &self + .validator_registry + .get(validator_index) + .ok_or_else(|| Error::UnknownValidator)?; + + if self.slot + >= validator + .withdrawable_epoch + .start_slot(spec.slots_per_epoch) + { + return Err(Error::SlotOutOfBounds); + } + + self.exit_validator(validator_index, spec); + + self.latest_slashed_balances[current_epoch.as_usize() % spec.latest_slashed_exit_length] += self.get_effective_balance(validator_index, spec); let whistleblower_index = self.get_beacon_proposer_index(self.slot, spec)?; @@ -1005,11 +1020,15 @@ impl BeaconState { self.validator_balances[validator_index], whistleblower_reward ); - self.validator_registry[validator_index].penalized_epoch = current_epoch; + self.validator_registry[validator_index].slashed = true; + self.validator_registry[validator_index].withdrawable_epoch = + current_epoch + Epoch::from(spec.latest_slashed_exit_length); + debug!( "Whistleblower {} penalized validator {}.", whistleblower_index, validator_index ); + Ok(()) } diff --git a/eth2/types/src/beacon_state/builder.rs b/eth2/types/src/beacon_state/builder.rs index 4acd172f5..94cc28a6b 100644 --- a/eth2/types/src/beacon_state/builder.rs +++ b/eth2/types/src/beacon_state/builder.rs @@ -142,11 +142,11 @@ impl BeaconStateBuilder { state.slot = slot; state.validator_registry_update_epoch = epoch - 1; - state.previous_calculation_epoch = epoch - 1; - state.current_calculation_epoch = epoch; + state.previous_shuffling_epoch = epoch - 1; + state.current_shuffling_epoch = epoch; - state.previous_epoch_seed = Hash256::from(&b"previous_seed"[..]); - state.current_epoch_seed = Hash256::from(&b"current_seed"[..]); + state.previous_shuffling_seed = Hash256::from(&b"previous_seed"[..]); + state.current_shuffling_seed = Hash256::from(&b"current_seed"[..]); state.previous_justified_epoch = epoch - 2; state.justified_epoch = epoch - 1; diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs index ac5d0a815..8aef79466 100644 --- a/eth2/types/src/chain_spec.rs +++ b/eth2/types/src/chain_spec.rs @@ -61,7 +61,7 @@ pub struct ChainSpec { */ pub latest_block_roots_length: usize, pub latest_randao_mixes_length: usize, - pub latest_index_roots_length: usize, + pub latest_active_index_roots_length: usize, pub latest_slashed_exit_length: usize, /* @@ -157,7 +157,7 @@ impl ChainSpec { */ latest_block_roots_length: 8_192, latest_randao_mixes_length: 8_192, - latest_index_roots_length: 8_192, + latest_active_index_roots_length: 8_192, latest_slashed_exit_length: 8_192, /* From 0f7d2c168ca6770d9ecabcf34a2274faee61152b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 5 Mar 2019 17:19:36 +1100 Subject: [PATCH 105/132] Fix compile errors in beacon_state.rs --- eth2/types/src/beacon_state.rs | 77 +++++++++++++++++----------------- 1 file changed, 38 insertions(+), 39 deletions(-) diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 43fd72232..bf989b858 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -470,17 +470,15 @@ impl BeaconState { /// Return the index root at a recent `epoch`. /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn get_active_index_root(&self, epoch: Epoch, spec: &ChainSpec) -> Option { let current_epoch = self.current_epoch(spec); - let earliest_index_root = current_epoch - - Epoch::from(spec.latest_active_index_roots_length) - + Epoch::from(spec.entry_exit_delay) - + 1; - let latest_index_root = current_epoch + spec.entry_exit_delay; - - if (epoch >= earliest_index_root) & (epoch <= latest_index_root) { + if (current_epoch - spec.latest_active_index_roots_length as u64 + + spec.activation_exit_delay + < epoch) + & (epoch <= current_epoch + spec.activation_exit_delay) + { Some( self.latest_active_index_roots [epoch.as_usize() % spec.latest_active_index_roots_length], @@ -678,12 +676,11 @@ impl BeaconState { .and_then(|tuple| Some(*tuple))) } - /// An entry or exit triggered in the ``epoch`` given by the input takes effect at - /// the epoch given by the output. + /// Return the epoch at which an activation or exit triggered in ``epoch`` takes effect. /// - /// Spec v0.2.0 - pub fn get_entry_exit_effect_epoch(&self, epoch: Epoch, spec: &ChainSpec) -> Epoch { - epoch + 1 + spec.entry_exit_delay + /// Spec v0.4.0 + pub fn get_delayed_activation_exit_epoch(&self, epoch: Epoch, spec: &ChainSpec) -> Epoch { + epoch + 1 + spec.activation_exit_delay } /// Returns the beacon proposer index for the `slot`. @@ -710,20 +707,20 @@ impl BeaconState { }) } - /// Process the penalties and prepare the validators who are eligible to withdrawal. + /// Process the slashings. /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn process_slashings(&mut self, spec: &ChainSpec) { let current_epoch = self.current_epoch(spec); let active_validator_indices = get_active_validator_indices(&self.validator_registry, current_epoch); let total_balance = self.get_total_balance(&active_validator_indices[..], spec); - for index in 0..self.validator_balances.len() { - let validator = &self.validator_registry[index]; - - if current_epoch - == validator.penalized_epoch + Epoch::from(spec.latest_slashed_exit_length / 2) + for (index, validator) in self.validator_registry.iter().enumerate() { + if validator.slashed + && (current_epoch + == validator.withdrawable_epoch + - Epoch::from(spec.latest_slashed_exit_length / 2)) { let epoch_index: usize = current_epoch.as_usize() % spec.latest_slashed_exit_length; @@ -731,17 +728,21 @@ impl BeaconState { [(epoch_index + 1) % spec.latest_slashed_exit_length]; let total_at_end = self.latest_slashed_balances[epoch_index]; let total_penalities = total_at_end.saturating_sub(total_at_start); - let penalty = self.get_effective_balance(index, spec) - * std::cmp::min(total_penalities * 3, total_balance) - / total_balance; + let penalty = std::cmp::max( + self.get_effective_balance(index, spec) + * std::cmp::min(total_penalities * 3, total_balance) + / total_balance, + self.get_effective_balance(index, spec) / spec.min_penalty_quotient, + ); + safe_sub_assign!(self.validator_balances[index], penalty); } } } - /// Process the penalties and prepare the validators who are eligible to withdrawal. + /// Process the exit queue. /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn process_exit_queue(&mut self, spec: &ChainSpec) { let current_epoch = self.current_epoch(spec); let active_validator_indices = @@ -751,11 +752,10 @@ impl BeaconState { let eligible = |index: usize| { let validator = &self.validator_registry[index]; - if validator.penalized_epoch <= current_epoch { - let penalized_withdrawal_epochs = spec.latest_slashed_exit_length / 2; - current_epoch >= validator.penalized_epoch + penalized_withdrawal_epochs as u64 + if validator.withdrawable_epoch != spec.far_future_epoch { + false } else { - current_epoch >= validator.exit_epoch + spec.min_validator_withdrawal_epochs + current_epoch >= validator.exit_epoch + spec.min_validator_withdrawability_delay } }; @@ -763,11 +763,12 @@ impl BeaconState { .filter(|i| eligible(*i)) .collect(); eligable_indices.sort_by_key(|i| self.validator_registry[*i].exit_epoch); + for (withdrawn_so_far, index) in eligable_indices.iter().enumerate() { - self.prepare_validator_for_withdrawal(*index); - if withdrawn_so_far as u64 >= spec.max_withdrawals_per_epoch { + if withdrawn_so_far as u64 >= spec.max_exit_dequeues_per_epoch { break; } + self.prepare_validator_for_withdrawal(*index, spec); } } @@ -943,7 +944,7 @@ impl BeaconState { /// Activate the validator of the given ``index``. /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn activate_validator( &mut self, validator_index: usize, @@ -955,7 +956,7 @@ impl BeaconState { self.validator_registry[validator_index].activation_epoch = if is_genesis { spec.genesis_epoch } else { - self.get_entry_exit_effect_epoch(current_epoch, spec) + self.get_delayed_activation_exit_epoch(current_epoch, spec) } } @@ -968,18 +969,16 @@ impl BeaconState { /// Exit the validator of the given `index`. /// - /// Spec v0.2.0 + /// Spec v0.4.0 fn exit_validator(&mut self, validator_index: usize, spec: &ChainSpec) { let current_epoch = self.current_epoch(spec); + let delayed_epoch = self.get_delayed_activation_exit_epoch(current_epoch, spec); - if self.validator_registry[validator_index].exit_epoch - <= self.get_entry_exit_effect_epoch(current_epoch, spec) - { + if self.validator_registry[validator_index].exit_epoch <= delayed_epoch { return; } - self.validator_registry[validator_index].exit_epoch = - self.get_entry_exit_effect_epoch(current_epoch, spec); + self.validator_registry[validator_index].exit_epoch = delayed_epoch; } /// Slash the validator with index ``index``. From d519bc1388505371df464dd1b169e8176cea96d0 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 5 Mar 2019 17:18:41 +1100 Subject: [PATCH 106/132] Use cfg(test) for test macros --- beacon_node/db/src/stores/macros.rs | 2 +- eth2/types/src/slot_epoch_macros.rs | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/beacon_node/db/src/stores/macros.rs b/beacon_node/db/src/stores/macros.rs index 36b8aef8e..e26e101c9 100644 --- a/beacon_node/db/src/stores/macros.rs +++ b/beacon_node/db/src/stores/macros.rs @@ -20,7 +20,7 @@ macro_rules! impl_crud_for_store { }; } -#[allow(unused_macros)] +#[cfg(test)] macro_rules! test_crud_for_store { ($store: ident, $db_column: expr) => { #[test] diff --git a/eth2/types/src/slot_epoch_macros.rs b/eth2/types/src/slot_epoch_macros.rs index b0550f2f8..22355fefe 100644 --- a/eth2/types/src/slot_epoch_macros.rs +++ b/eth2/types/src/slot_epoch_macros.rs @@ -21,6 +21,7 @@ macro_rules! impl_from_into_u64 { } // need to truncate for some fork-choice algorithms +#[allow(unused_macros)] macro_rules! impl_into_u32 { ($main: ident) => { impl Into for $main { @@ -267,7 +268,7 @@ macro_rules! impl_common { } // test macros -#[allow(unused_macros)] +#[cfg(test)] macro_rules! new_tests { ($type: ident) => { #[test] @@ -279,7 +280,7 @@ macro_rules! new_tests { }; } -#[allow(unused_macros)] +#[cfg(test)] macro_rules! from_into_tests { ($type: ident, $other: ident) => { #[test] @@ -305,7 +306,7 @@ macro_rules! from_into_tests { }; } -#[allow(unused_macros)] +#[cfg(test)] macro_rules! math_between_tests { ($type: ident, $other: ident) => { #[test] @@ -453,7 +454,7 @@ macro_rules! math_between_tests { }; } -#[allow(unused_macros)] +#[cfg(test)] macro_rules! math_tests { ($type: ident) => { #[test] @@ -575,7 +576,7 @@ macro_rules! ssz_tests { }; } -#[allow(unused_macros)] +#[cfg(test)] macro_rules! all_tests { ($type: ident) => { new_tests!($type); From 33a3161905561c9ae52914ab0b95775f49fc4051 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 5 Mar 2019 17:28:51 +1100 Subject: [PATCH 107/132] Remove SSZ round-trip test duplication Closes #244 --- eth2/types/src/attestation.rs | 25 +------------- eth2/types/src/attestation_data.rs | 25 +------------- .../src/attestation_data_and_custody_bit.rs | 27 +-------------- eth2/types/src/attester_slashing.rs | 25 +------------- eth2/types/src/beacon_block.rs | 25 +------------- eth2/types/src/beacon_block_body.rs | 25 +------------- eth2/types/src/beacon_state/tests.rs | 24 +------------ eth2/types/src/casper_slashing.rs | 25 +------------- eth2/types/src/crosslink.rs | 25 +------------- eth2/types/src/deposit.rs | 25 +------------- eth2/types/src/deposit_data.rs | 25 +------------- eth2/types/src/deposit_input.rs | 25 +------------- eth2/types/src/eth1_data.rs | 25 +------------- eth2/types/src/eth1_data_vote.rs | 25 +------------- eth2/types/src/exit.rs | 25 +------------- eth2/types/src/fork.rs | 25 +------------- eth2/types/src/lib.rs | 1 + eth2/types/src/pending_attestation.rs | 25 +------------- eth2/types/src/proposal_signed_data.rs | 25 +------------- eth2/types/src/proposer_slashing.rs | 25 +------------- eth2/types/src/shard_reassignment_record.rs | 25 +------------- eth2/types/src/slashable_attestation.rs | 24 +------------ eth2/types/src/slashable_vote_data.rs | 24 +------------ eth2/types/src/slot_epoch.rs | 4 --- eth2/types/src/slot_epoch_macros.rs | 28 --------------- eth2/types/src/slot_height.rs | 3 -- eth2/types/src/test_utils/macros.rs | 34 +++++++++++++++++++ eth2/types/src/test_utils/mod.rs | 2 ++ eth2/types/src/validator.rs | 24 +------------ .../src/validator_registry_delta_block.rs | 25 +------------- 30 files changed, 61 insertions(+), 609 deletions(-) create mode 100644 eth2/types/src/test_utils/macros.rs diff --git a/eth2/types/src/attestation.rs b/eth2/types/src/attestation.rs index a0c8505b8..4ac81bb4c 100644 --- a/eth2/types/src/attestation.rs +++ b/eth2/types/src/attestation.rs @@ -40,29 +40,6 @@ impl Attestation { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Attestation::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Attestation::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(Attestation); } diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data.rs index e23cdab46..73b5facfa 100644 --- a/eth2/types/src/attestation_data.rs +++ b/eth2/types/src/attestation_data.rs @@ -50,29 +50,6 @@ impl AttestationData { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = AttestationData::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = AttestationData::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(AttestationData); } diff --git a/eth2/types/src/attestation_data_and_custody_bit.rs b/eth2/types/src/attestation_data_and_custody_bit.rs index 9175863ae..89a795292 100644 --- a/eth2/types/src/attestation_data_and_custody_bit.rs +++ b/eth2/types/src/attestation_data_and_custody_bit.rs @@ -23,31 +23,6 @@ impl TestRandom for AttestationDataAndCustodyBit { #[cfg(test)] mod test { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - - let original = AttestationDataAndCustodyBit::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = AttestationDataAndCustodyBit::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(AttestationDataAndCustodyBit); } diff --git a/eth2/types/src/attester_slashing.rs b/eth2/types/src/attester_slashing.rs index ac75a2562..cfa96a166 100644 --- a/eth2/types/src/attester_slashing.rs +++ b/eth2/types/src/attester_slashing.rs @@ -17,29 +17,6 @@ pub struct AttesterSlashing { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = AttesterSlashing::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = AttesterSlashing::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(AttesterSlashing); } diff --git a/eth2/types/src/beacon_block.rs b/eth2/types/src/beacon_block.rs index cb4e6668b..a10fd2a6e 100644 --- a/eth2/types/src/beacon_block.rs +++ b/eth2/types/src/beacon_block.rs @@ -64,29 +64,6 @@ impl BeaconBlock { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = BeaconBlock::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = BeaconBlock::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(BeaconBlock); } diff --git a/eth2/types/src/beacon_block_body.rs b/eth2/types/src/beacon_block_body.rs index 2b343b970..72733789c 100644 --- a/eth2/types/src/beacon_block_body.rs +++ b/eth2/types/src/beacon_block_body.rs @@ -17,29 +17,6 @@ pub struct BeaconBlockBody { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = BeaconBlockBody::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = BeaconBlockBody::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(BeaconBlockBody); } diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs index bb8561511..71e98e9b9 100644 --- a/eth2/types/src/beacon_state/tests.rs +++ b/eth2/types/src/beacon_state/tests.rs @@ -3,7 +3,6 @@ use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; use crate::{BeaconState, ChainSpec}; -use ssz::{ssz_encode, Decodable}; #[test] pub fn can_produce_genesis_block() { @@ -60,25 +59,4 @@ pub fn get_attestation_participants_consistency() { } } -#[test] -pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = BeaconState::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); -} - -#[test] -pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = BeaconState::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 -} +ssz_tests!(BeaconState); diff --git a/eth2/types/src/casper_slashing.rs b/eth2/types/src/casper_slashing.rs index cb1e46ee5..d6a6ac9fe 100644 --- a/eth2/types/src/casper_slashing.rs +++ b/eth2/types/src/casper_slashing.rs @@ -14,29 +14,6 @@ pub struct CasperSlashing { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = CasperSlashing::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = CasperSlashing::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(CasperSlashing); } diff --git a/eth2/types/src/crosslink.rs b/eth2/types/src/crosslink.rs index 11fb3386d..198a039e9 100644 --- a/eth2/types/src/crosslink.rs +++ b/eth2/types/src/crosslink.rs @@ -26,29 +26,6 @@ impl Crosslink { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Crosslink::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Crosslink::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(Crosslink); } diff --git a/eth2/types/src/deposit.rs b/eth2/types/src/deposit.rs index 02da32cfe..23772f222 100644 --- a/eth2/types/src/deposit.rs +++ b/eth2/types/src/deposit.rs @@ -15,29 +15,6 @@ pub struct Deposit { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Deposit::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Deposit::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(Deposit); } diff --git a/eth2/types/src/deposit_data.rs b/eth2/types/src/deposit_data.rs index 349207791..ba380378a 100644 --- a/eth2/types/src/deposit_data.rs +++ b/eth2/types/src/deposit_data.rs @@ -15,29 +15,6 @@ pub struct DepositData { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = DepositData::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = DepositData::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(DepositData); } diff --git a/eth2/types/src/deposit_input.rs b/eth2/types/src/deposit_input.rs index 1f3b22779..47a803359 100644 --- a/eth2/types/src/deposit_input.rs +++ b/eth2/types/src/deposit_input.rs @@ -16,29 +16,6 @@ pub struct DepositInput { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = DepositInput::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = DepositInput::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(DepositInput); } diff --git a/eth2/types/src/eth1_data.rs b/eth2/types/src/eth1_data.rs index 8eabbabc7..c68e73aa8 100644 --- a/eth2/types/src/eth1_data.rs +++ b/eth2/types/src/eth1_data.rs @@ -15,29 +15,6 @@ pub struct Eth1Data { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Eth1Data::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Eth1Data::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(Eth1Data); } diff --git a/eth2/types/src/eth1_data_vote.rs b/eth2/types/src/eth1_data_vote.rs index fa30b9052..8e4cbb4e2 100644 --- a/eth2/types/src/eth1_data_vote.rs +++ b/eth2/types/src/eth1_data_vote.rs @@ -15,29 +15,6 @@ pub struct Eth1DataVote { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Eth1DataVote::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Eth1DataVote::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(Eth1DataVote); } diff --git a/eth2/types/src/exit.rs b/eth2/types/src/exit.rs index 5b41fcc7a..3beaa2a22 100644 --- a/eth2/types/src/exit.rs +++ b/eth2/types/src/exit.rs @@ -15,29 +15,6 @@ pub struct Exit { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Exit::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Exit::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(Exit); } diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs index 5b13a2388..9da7c29c2 100644 --- a/eth2/types/src/fork.rs +++ b/eth2/types/src/fork.rs @@ -30,29 +30,6 @@ impl Fork { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Fork::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Fork::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(Fork); } diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index 4f196b9e9..32927ad1f 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -1,3 +1,4 @@ +#[macro_use] pub mod test_utils; pub mod attestation; diff --git a/eth2/types/src/pending_attestation.rs b/eth2/types/src/pending_attestation.rs index 84eb59207..5afdf1f2c 100644 --- a/eth2/types/src/pending_attestation.rs +++ b/eth2/types/src/pending_attestation.rs @@ -16,29 +16,6 @@ pub struct PendingAttestation { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = PendingAttestation::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = PendingAttestation::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(PendingAttestation); } diff --git a/eth2/types/src/proposal_signed_data.rs b/eth2/types/src/proposal_signed_data.rs index 6f6048ffc..58f45a41d 100644 --- a/eth2/types/src/proposal_signed_data.rs +++ b/eth2/types/src/proposal_signed_data.rs @@ -15,29 +15,6 @@ pub struct ProposalSignedData { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = ProposalSignedData::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = ProposalSignedData::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(ProposalSignedData); } diff --git a/eth2/types/src/proposer_slashing.rs b/eth2/types/src/proposer_slashing.rs index ea30d46ec..93629c588 100644 --- a/eth2/types/src/proposer_slashing.rs +++ b/eth2/types/src/proposer_slashing.rs @@ -22,29 +22,6 @@ pub struct ProposerSlashing { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = ProposerSlashing::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = ProposerSlashing::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(ProposerSlashing); } diff --git a/eth2/types/src/shard_reassignment_record.rs b/eth2/types/src/shard_reassignment_record.rs index f5dfa8676..9f1705f16 100644 --- a/eth2/types/src/shard_reassignment_record.rs +++ b/eth2/types/src/shard_reassignment_record.rs @@ -14,29 +14,6 @@ pub struct ShardReassignmentRecord { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = ShardReassignmentRecord::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = ShardReassignmentRecord::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(ShardReassignmentRecord); } diff --git a/eth2/types/src/slashable_attestation.rs b/eth2/types/src/slashable_attestation.rs index 8ad582ce6..0aff549ac 100644 --- a/eth2/types/src/slashable_attestation.rs +++ b/eth2/types/src/slashable_attestation.rs @@ -39,7 +39,6 @@ mod tests { use crate::chain_spec::ChainSpec; use crate::slot_epoch::{Epoch, Slot}; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_is_double_vote_true() { @@ -113,28 +112,7 @@ mod tests { ); } - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = SlashableAttestation::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = SlashableAttestation::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(SlashableAttestation); fn create_slashable_attestation( slot_factor: u64, diff --git a/eth2/types/src/slashable_vote_data.rs b/eth2/types/src/slashable_vote_data.rs index 31dd9e0a8..73cf91c61 100644 --- a/eth2/types/src/slashable_vote_data.rs +++ b/eth2/types/src/slashable_vote_data.rs @@ -42,7 +42,6 @@ mod tests { use crate::chain_spec::ChainSpec; use crate::slot_epoch::{Epoch, Slot}; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; #[test] pub fn test_is_double_vote_true() { @@ -116,28 +115,7 @@ mod tests { ); } - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = SlashableVoteData::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = SlashableVoteData::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(SlashableVoteData); fn create_slashable_vote_data( slot_factor: u64, diff --git a/eth2/types/src/slot_epoch.rs b/eth2/types/src/slot_epoch.rs index ff4fd5b9b..0c9c37e9a 100644 --- a/eth2/types/src/slot_epoch.rs +++ b/eth2/types/src/slot_epoch.rs @@ -103,8 +103,6 @@ impl<'a> Iterator for SlotIter<'a> { #[cfg(test)] mod slot_tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; all_tests!(Slot); } @@ -112,8 +110,6 @@ mod slot_tests { #[cfg(test)] mod epoch_tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; all_tests!(Epoch); diff --git a/eth2/types/src/slot_epoch_macros.rs b/eth2/types/src/slot_epoch_macros.rs index 22355fefe..b8af53ae1 100644 --- a/eth2/types/src/slot_epoch_macros.rs +++ b/eth2/types/src/slot_epoch_macros.rs @@ -548,34 +548,6 @@ macro_rules! math_tests { }; } -#[allow(unused_macros)] -macro_rules! ssz_tests { - ($type: ident) => { - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = $type::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = $type::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = $type::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } - }; -} - #[cfg(test)] macro_rules! all_tests { ($type: ident) => { diff --git a/eth2/types/src/slot_height.rs b/eth2/types/src/slot_height.rs index f9370f485..e3a7b449a 100644 --- a/eth2/types/src/slot_height.rs +++ b/eth2/types/src/slot_height.rs @@ -33,11 +33,8 @@ impl SlotHeight { } #[cfg(test)] - mod slot_height_tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; all_tests!(SlotHeight); } diff --git a/eth2/types/src/test_utils/macros.rs b/eth2/types/src/test_utils/macros.rs new file mode 100644 index 000000000..b7c0a6522 --- /dev/null +++ b/eth2/types/src/test_utils/macros.rs @@ -0,0 +1,34 @@ +#[cfg(test)] +#[macro_export] +macro_rules! ssz_tests { + ($type: ident) => { + #[test] + pub fn test_ssz_round_trip() { + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::{ssz_encode, Decodable}; + + let mut rng = XorShiftRng::from_seed([42; 16]); + let original = $type::random_for_test(&mut rng); + + let bytes = ssz_encode(&original); + let (decoded, _) = $type::ssz_decode(&bytes, 0).unwrap(); + + assert_eq!(original, decoded); + } + + #[test] + pub fn test_hash_tree_root_internal() { + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::TreeHash; + + let mut rng = XorShiftRng::from_seed([42; 16]); + let original = $type::random_for_test(&mut rng); + + let result = original.hash_tree_root_internal(); + + assert_eq!(result.len(), 32); + // TODO: Add further tests + // https://github.com/sigp/lighthouse/issues/170 + } + }; +} diff --git a/eth2/types/src/test_utils/mod.rs b/eth2/types/src/test_utils/mod.rs index eb54f2a53..2203d40d6 100644 --- a/eth2/types/src/test_utils/mod.rs +++ b/eth2/types/src/test_utils/mod.rs @@ -6,6 +6,8 @@ pub mod address; pub mod aggregate_signature; pub mod bitfield; pub mod hash256; +#[macro_use] +mod macros; pub mod public_key; pub mod secret_key; pub mod signature; diff --git a/eth2/types/src/validator.rs b/eth2/types/src/validator.rs index 42a2b31f2..6c0e7506d 100644 --- a/eth2/types/src/validator.rs +++ b/eth2/types/src/validator.rs @@ -170,18 +170,6 @@ impl TestRandom for Validator { mod tests { use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; - - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Validator::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } #[test] fn test_validator_can_be_active() { @@ -206,15 +194,5 @@ mod tests { } } - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = Validator::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(Validator); } diff --git a/eth2/types/src/validator_registry_delta_block.rs b/eth2/types/src/validator_registry_delta_block.rs index 0746875f0..e9a075052 100644 --- a/eth2/types/src/validator_registry_delta_block.rs +++ b/eth2/types/src/validator_registry_delta_block.rs @@ -31,29 +31,6 @@ impl Default for ValidatorRegistryDeltaBlock { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::{ssz_encode, Decodable, TreeHash}; - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = ValidatorRegistryDeltaBlock::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root_internal() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = ValidatorRegistryDeltaBlock::random_for_test(&mut rng); - - let result = original.hash_tree_root_internal(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } + ssz_tests!(ValidatorRegistryDeltaBlock); } From e0ccde1ce3f71fe4b5bcfba03676a8bd285d3c23 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 5 Mar 2019 17:37:09 +1100 Subject: [PATCH 108/132] Remove unused function from @agemanning --- eth2/types/src/slot_epoch_macros.rs | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/eth2/types/src/slot_epoch_macros.rs b/eth2/types/src/slot_epoch_macros.rs index b0550f2f8..2148b6cc2 100644 --- a/eth2/types/src/slot_epoch_macros.rs +++ b/eth2/types/src/slot_epoch_macros.rs @@ -20,25 +20,6 @@ macro_rules! impl_from_into_u64 { }; } -// need to truncate for some fork-choice algorithms -macro_rules! impl_into_u32 { - ($main: ident) => { - impl Into for $main { - fn into(self) -> u32 { - assert!(self.0 < u64::from(std::u32::MAX), "Lossy conversion to u32"); - self.0 as u32 - } - } - - impl $main { - pub fn as_u32(&self) -> u32 { - assert!(self.0 < u64::from(std::u32::MAX), "Lossy conversion to u32"); - self.0 as u32 - } - } - }; -} - macro_rules! impl_from_into_usize { ($main: ident) => { impl From for $main { From 38a1b94f61ad22aeedee22aa91ae5f413dc1aff4 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 5 Mar 2019 17:38:17 +1100 Subject: [PATCH 109/132] Fix compile issues in `types` crate --- eth2/types/src/attester_slashing/builder.rs | 4 ++-- eth2/types/src/beacon_block.rs | 3 ++- eth2/types/src/beacon_state.rs | 3 --- eth2/types/src/beacon_state/builder.rs | 2 +- eth2/types/src/readers/block_reader.rs | 5 ----- 5 files changed, 5 insertions(+), 12 deletions(-) diff --git a/eth2/types/src/attester_slashing/builder.rs b/eth2/types/src/attester_slashing/builder.rs index ed203d6e1..54dfae959 100644 --- a/eth2/types/src/attester_slashing/builder.rs +++ b/eth2/types/src/attester_slashing/builder.rs @@ -37,7 +37,7 @@ impl AttesterSlashingBuilder { shard, beacon_block_root: hash_1, epoch_boundary_root: hash_1, - shard_block_root: hash_1, + crosslink_data_root: hash_1, latest_crosslink: Crosslink { epoch, shard_block_root: hash_1, @@ -56,7 +56,7 @@ impl AttesterSlashingBuilder { shard, beacon_block_root: hash_2, epoch_boundary_root: hash_2, - shard_block_root: hash_2, + crosslink_data_root: hash_2, latest_crosslink: Crosslink { epoch, shard_block_root: hash_2, diff --git a/eth2/types/src/beacon_block.rs b/eth2/types/src/beacon_block.rs index d8e50d9ca..111ca1b4b 100644 --- a/eth2/types/src/beacon_block.rs +++ b/eth2/types/src/beacon_block.rs @@ -39,7 +39,8 @@ impl BeaconBlock { attester_slashings: vec![], attestations: vec![], deposits: vec![], - exits: vec![], + voluntary_exits: vec![], + transfers: vec![], }, } } diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index bf989b858..d62e29127 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -745,9 +745,6 @@ impl BeaconState { /// Spec v0.4.0 pub fn process_exit_queue(&mut self, spec: &ChainSpec) { let current_epoch = self.current_epoch(spec); - let active_validator_indices = - get_active_validator_indices(&self.validator_registry, current_epoch); - let total_balance = self.get_total_balance(&active_validator_indices[..], spec); let eligible = |index: usize| { let validator = &self.validator_registry[index]; diff --git a/eth2/types/src/beacon_state/builder.rs b/eth2/types/src/beacon_state/builder.rs index 94cc28a6b..36b6468fc 100644 --- a/eth2/types/src/beacon_state/builder.rs +++ b/eth2/types/src/beacon_state/builder.rs @@ -249,7 +249,7 @@ fn committee_to_pending_attestation( shard, beacon_block_root: *state.get_block_root(slot, spec).unwrap(), epoch_boundary_root, - shard_block_root: Hash256::zero(), + crosslink_data_root: Hash256::zero(), latest_crosslink: Crosslink { epoch: slot.epoch(spec.slots_per_epoch), shard_block_root: Hash256::zero(), diff --git a/eth2/types/src/readers/block_reader.rs b/eth2/types/src/readers/block_reader.rs index bcb2d0e63..93157a1a3 100644 --- a/eth2/types/src/readers/block_reader.rs +++ b/eth2/types/src/readers/block_reader.rs @@ -13,7 +13,6 @@ pub trait BeaconBlockReader: Debug + PartialEq { fn slot(&self) -> Slot; fn parent_root(&self) -> Hash256; fn state_root(&self) -> Hash256; - fn canonical_root(&self) -> Hash256; fn into_beacon_block(self) -> Option; } @@ -30,10 +29,6 @@ impl BeaconBlockReader for BeaconBlock { self.state_root } - fn canonical_root(&self) -> Hash256 { - self.canonical_root() - } - fn into_beacon_block(self) -> Option { Some(self) } From 96ec53c6a83f01c6e2f4826269fa85846f35b604 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 5 Mar 2019 18:22:37 +1100 Subject: [PATCH 110/132] Update beacon_state.rs to spec v0.4.0 --- eth2/types/Cargo.toml | 1 + eth2/types/src/beacon_state.rs | 1131 ++++++++++++------------ eth2/types/src/beacon_state/helpers.rs | 0 eth2/types/src/chain_spec.rs | 16 +- eth2/types/src/fork.rs | 4 + eth2/types/src/readers/state_reader.rs | 7 +- eth2/types/src/validator_registry.rs | 2 + 7 files changed, 575 insertions(+), 586 deletions(-) create mode 100644 eth2/types/src/beacon_state/helpers.rs diff --git a/eth2/types/Cargo.toml b/eth2/types/Cargo.toml index f70e8b490..f611aa3c1 100644 --- a/eth2/types/Cargo.toml +++ b/eth2/types/Cargo.toml @@ -10,6 +10,7 @@ boolean-bitfield = { path = "../utils/boolean-bitfield" } ethereum-types = "0.4.0" hashing = { path = "../utils/hashing" } honey-badger-split = { path = "../utils/honey-badger-split" } +int_to_bytes = { path = "../utils/int_to_bytes" } log = "0.4" rayon = "1.0" rand = "0.5.5" diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index d62e29127..d876adc62 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -3,6 +3,7 @@ use crate::test_utils::TestRandom; use crate::{validator_registry::get_active_validator_indices, *}; use bls::verify_proof_of_possession; use honey_badger_split::SplitExt; +use int_to_bytes::int_to_bytes32; use log::{debug, error, trace}; use rand::RngCore; use rayon::prelude::*; @@ -191,6 +192,7 @@ impl BeaconState { caches: vec![EpochCache::empty(); CACHED_EPOCHS], }) } + /// Produce the first state of the Beacon Chain. pub fn genesis( genesis_time: u64, @@ -328,144 +330,116 @@ impl BeaconState { } } - /// Return the tree hash root for this `BeaconState`. - /// - /// Spec v0.2.0 - pub fn canonical_root(&self) -> Hash256 { - Hash256::from(&self.hash_tree_root()[..]) - } - /// The epoch corresponding to `self.slot`. /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn current_epoch(&self, spec: &ChainSpec) -> Epoch { self.slot.epoch(spec.slots_per_epoch) } /// The epoch prior to `self.current_epoch()`. /// - /// Spec v0.2.0 + /// If the current epoch is the genesis epoch, the genesis_epoch is returned. + /// + /// Spec v0.4.0 pub fn previous_epoch(&self, spec: &ChainSpec) -> Epoch { let current_epoch = self.current_epoch(&spec); - if current_epoch == spec.genesis_epoch { - current_epoch - } else { - current_epoch - 1 - } + std::cmp::max(current_epoch - 1, spec.genesis_epoch) } /// The epoch following `self.current_epoch()`. /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn next_epoch(&self, spec: &ChainSpec) -> Epoch { self.current_epoch(spec).saturating_add(1_u64) } /// The first slot of the epoch corresponding to `self.slot`. /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn current_epoch_start_slot(&self, spec: &ChainSpec) -> Slot { self.current_epoch(spec).start_slot(spec.slots_per_epoch) } - /// The first slot of the epoch preceeding the one corresponding to `self.slot`. + /// The first slot of the epoch preceding the one corresponding to `self.slot`. /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn previous_epoch_start_slot(&self, spec: &ChainSpec) -> Slot { self.previous_epoch(spec).start_slot(spec.slots_per_epoch) } - /// Return the number of committees in one epoch. - /// - /// TODO: this should probably be a method on `ChainSpec`. - /// - /// Spec v0.2.0 - pub fn get_epoch_committee_count( - &self, - active_validator_count: usize, - spec: &ChainSpec, - ) -> u64 { - std::cmp::max( - 1, - std::cmp::min( - spec.shard_count / spec.slots_per_epoch, - active_validator_count as u64 / spec.slots_per_epoch / spec.target_committee_size, - ), - ) * spec.slots_per_epoch - } - - /// Shuffle ``validators`` into crosslink committees seeded by ``seed`` and ``epoch``. - /// - /// Return a list of ``committees_per_epoch`` committees where each - /// committee is itself a list of validator indices. - /// - /// Spec v0.2.0 - pub(crate) fn get_shuffling( - &self, - seed: Hash256, - epoch: Epoch, - spec: &ChainSpec, - ) -> Result>, Error> { - let active_validator_indices = - get_active_validator_indices(&self.validator_registry, epoch); - - if active_validator_indices.is_empty() { - error!("get_shuffling: no validators."); - return Err(Error::InsufficientValidators); - } - - debug!("Shuffling {} validators...", active_validator_indices.len()); - - let committees_per_epoch = - self.get_epoch_committee_count(active_validator_indices.len(), spec); - - trace!( - "get_shuffling: active_validator_indices.len() == {}, committees_per_epoch: {}", - active_validator_indices.len(), - committees_per_epoch - ); - - let active_validator_indices: Vec = active_validator_indices.to_vec(); - - let shuffled_active_validator_indices = shuffle_list( - active_validator_indices, - spec.shuffle_round_count, - &seed[..], - true, - ) - .ok_or_else(|| Error::UnableToShuffle)?; - - Ok(shuffled_active_validator_indices - .honey_badger_split(committees_per_epoch as usize) - .map(|slice: &[usize]| slice.to_vec()) - .collect()) - } - /// Return the number of committees in the previous epoch. /// - /// Spec v0.2.0 + /// Spec v0.4.0 fn get_previous_epoch_committee_count(&self, spec: &ChainSpec) -> u64 { let previous_active_validators = get_active_validator_indices(&self.validator_registry, self.previous_shuffling_epoch); - self.get_epoch_committee_count(previous_active_validators.len(), spec) + spec.get_epoch_committee_count(previous_active_validators.len()) } /// Return the number of committees in the current epoch. /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn get_current_epoch_committee_count(&self, spec: &ChainSpec) -> u64 { let current_active_validators = get_active_validator_indices(&self.validator_registry, self.current_shuffling_epoch); - self.get_epoch_committee_count(current_active_validators.len(), spec) + spec.get_epoch_committee_count(current_active_validators.len()) } /// Return the number of committees in the next epoch. /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn get_next_epoch_committee_count(&self, spec: &ChainSpec) -> u64 { - let current_active_validators = + let next_active_validators = get_active_validator_indices(&self.validator_registry, self.next_epoch(spec)); - self.get_epoch_committee_count(current_active_validators.len(), spec) + spec.get_epoch_committee_count(next_active_validators.len()) + } + + /// Returns the crosslink committees for some slot. + /// + /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. + /// + /// Spec v0.4.0 + pub fn get_crosslink_committees_at_slot( + &self, + slot: Slot, + spec: &ChainSpec, + ) -> Result<&CrosslinkCommittees, Error> { + let epoch = slot.epoch(spec.slots_per_epoch); + let relative_epoch = self.relative_epoch(epoch, spec)?; + let cache = self.cache(relative_epoch)?; + + let slot_offset = slot - epoch.start_slot(spec.slots_per_epoch); + + Ok(&cache.committees[slot_offset.as_usize()]) + } + + /// Return the block root at a recent `slot`. + /// + /// Spec v0.4.0 + pub fn get_block_root(&self, slot: Slot, spec: &ChainSpec) -> Option<&Hash256> { + if (self.slot <= slot + spec.latest_block_roots_length as u64) && (slot < self.slot) { + self.latest_block_roots + .get(slot.as_usize() % spec.latest_block_roots_length) + } else { + None + } + } + + /// Return the randao mix at a recent ``epoch``. + /// + /// Spec v0.4.0 + pub fn get_randao_mix(&self, epoch: Epoch, spec: &ChainSpec) -> Option<&Hash256> { + let current_epoch = self.current_epoch(spec); + + if (current_epoch - (spec.latest_randao_mixes_length as u64) < epoch) + & (epoch <= current_epoch) + { + self.latest_randao_mixes + .get(epoch.as_usize() % spec.latest_randao_mixes_length) + } else { + None + } } /// Return the index root at a recent `epoch`. @@ -490,10 +464,10 @@ impl BeaconState { /// Generate a seed for the given `epoch`. /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn generate_seed(&self, epoch: Epoch, spec: &ChainSpec) -> Result { let mut input = self - .get_randao_mix(epoch, spec) + .get_randao_mix(epoch - spec.min_seed_lookahead, spec) .ok_or_else(|| Error::InsufficientRandaoMixes)? .to_vec(); @@ -504,190 +478,16 @@ impl BeaconState { .to_vec(), ); - // TODO: ensure `Hash256::from(u64)` == `int_to_bytes32`. - input.append(&mut Hash256::from(epoch.as_u64()).to_vec()); + input.append(&mut int_to_bytes32(epoch.as_u64())); Ok(Hash256::from(&hash(&input[..])[..])) } - /// Returns the crosslink committees for some slot. - /// - /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. - /// - /// Spec v0.2.0 - pub fn get_crosslink_committees_at_slot( - &self, - slot: Slot, - spec: &ChainSpec, - ) -> Result<&CrosslinkCommittees, Error> { - let epoch = slot.epoch(spec.slots_per_epoch); - let relative_epoch = self.relative_epoch(epoch, spec)?; - let cache = self.cache(relative_epoch)?; - - let slot_offset = slot - epoch.start_slot(spec.slots_per_epoch); - - Ok(&cache.committees[slot_offset.as_usize()]) - } - - /// Returns the crosslink committees for some slot. - /// - /// Utilizes the cache and will fail if the appropriate cache is not initialized. - /// - /// Spec v0.2.0 - pub(crate) fn get_shuffling_for_slot( - &self, - slot: Slot, - registry_change: bool, - - spec: &ChainSpec, - ) -> Result>, Error> { - let (_committees_per_epoch, seed, shuffling_epoch, _shuffling_start_shard) = - self.get_committee_params_at_slot(slot, registry_change, spec)?; - - self.get_shuffling(seed, shuffling_epoch, spec) - } - - /// Returns the following params for the given slot: - /// - /// - epoch committee count - /// - epoch seed - /// - calculation epoch - /// - start shard - /// - /// In the spec, this functionality is included in the `get_crosslink_committees_at_slot(..)` - /// function. It is separated here to allow the division of shuffling and committee building, - /// as is required for efficient operations. - /// - /// Spec v0.2.0 - pub(crate) fn get_committee_params_at_slot( - &self, - slot: Slot, - registry_change: bool, - spec: &ChainSpec, - ) -> Result<(u64, Hash256, Epoch, u64), Error> { - let epoch = slot.epoch(spec.slots_per_epoch); - let current_epoch = self.current_epoch(spec); - let previous_epoch = self.previous_epoch(spec); - let next_epoch = self.next_epoch(spec); - - if epoch == current_epoch { - trace!("get_committee_params_at_slot: current_epoch"); - Ok(( - self.get_current_epoch_committee_count(spec), - self.current_shuffling_seed, - self.current_shuffling_epoch, - self.current_shuffling_start_shard, - )) - } else if epoch == previous_epoch { - trace!("get_committee_params_at_slot: previous_epoch"); - Ok(( - self.get_previous_epoch_committee_count(spec), - self.previous_shuffling_seed, - self.previous_shuffling_epoch, - self.previous_shuffling_start_shard, - )) - } else if epoch == next_epoch { - trace!("get_committee_params_at_slot: next_epoch"); - let current_committees_per_epoch = self.get_current_epoch_committee_count(spec); - let epochs_since_last_registry_update = - current_epoch - self.validator_registry_update_epoch; - let (seed, shuffling_start_shard) = if registry_change { - let next_seed = self.generate_seed(next_epoch, spec)?; - ( - next_seed, - (self.current_shuffling_start_shard + current_committees_per_epoch) - % spec.shard_count, - ) - } else if (epochs_since_last_registry_update > 1) - & epochs_since_last_registry_update.is_power_of_two() - { - let next_seed = self.generate_seed(next_epoch, spec)?; - (next_seed, self.current_shuffling_start_shard) - } else { - ( - self.current_shuffling_seed, - self.current_shuffling_start_shard, - ) - }; - Ok(( - self.get_next_epoch_committee_count(spec), - seed, - next_epoch, - shuffling_start_shard, - )) - } else { - Err(Error::EpochOutOfBounds) - } - } - - /// Return the list of ``(committee, shard)`` tuples for the ``slot``. - /// - /// Note: There are two possible shufflings for crosslink committees for a - /// `slot` in the next epoch: with and without a `registry_change` - /// - /// Note: does not utilize the cache, `get_crosslink_committees_at_slot` is an equivalent - /// function which uses the cache. - /// - /// Spec v0.2.0 - pub(crate) fn calculate_crosslink_committees_at_slot( - &self, - slot: Slot, - registry_change: bool, - shuffling: Vec>, - spec: &ChainSpec, - ) -> Result, u64)>, Error> { - let (committees_per_epoch, _seed, _shuffling_epoch, shuffling_start_shard) = - self.get_committee_params_at_slot(slot, registry_change, spec)?; - - let offset = slot.as_u64() % spec.slots_per_epoch; - let committees_per_slot = committees_per_epoch / spec.slots_per_epoch; - let slot_start_shard = - (shuffling_start_shard + committees_per_slot * offset) % spec.shard_count; - - let mut crosslinks_at_slot = vec![]; - for i in 0..committees_per_slot { - let tuple = ( - shuffling[(committees_per_slot * offset + i) as usize].clone(), - (slot_start_shard + i) % spec.shard_count, - ); - crosslinks_at_slot.push(tuple) - } - Ok(crosslinks_at_slot) - } - - /// Returns the `slot`, `shard` and `committee_index` for which a validator must produce an - /// attestation. - /// - /// Only reads the current epoch. - /// - /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. - /// - /// Spec v0.2.0 - pub fn attestation_slot_and_shard_for_validator( - &self, - validator_index: usize, - _spec: &ChainSpec, - ) -> Result, Error> { - let cache = self.cache(RelativeEpoch::Current)?; - - Ok(cache - .attestation_duty_map - .get(&(validator_index as u64)) - .and_then(|tuple| Some(*tuple))) - } - - /// Return the epoch at which an activation or exit triggered in ``epoch`` takes effect. - /// - /// Spec v0.4.0 - pub fn get_delayed_activation_exit_epoch(&self, epoch: Epoch, spec: &ChainSpec) -> Epoch { - epoch + 1 + spec.activation_exit_delay - } - /// Returns the beacon proposer index for the `slot`. /// /// If the state does not contain an index for a beacon proposer at the requested `slot`, then `None` is returned. /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn get_beacon_proposer_index(&self, slot: Slot, spec: &ChainSpec) -> Result { let committees = self.get_crosslink_committees_at_slot(slot, spec)?; trace!( @@ -707,155 +507,175 @@ impl BeaconState { }) } - /// Process the slashings. + /// Returns the list of validator indices which participiated in the attestation. + /// + /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. /// /// Spec v0.4.0 - pub fn process_slashings(&mut self, spec: &ChainSpec) { - let current_epoch = self.current_epoch(spec); - let active_validator_indices = - get_active_validator_indices(&self.validator_registry, current_epoch); - let total_balance = self.get_total_balance(&active_validator_indices[..], spec); - - for (index, validator) in self.validator_registry.iter().enumerate() { - if validator.slashed - && (current_epoch - == validator.withdrawable_epoch - - Epoch::from(spec.latest_slashed_exit_length / 2)) - { - let epoch_index: usize = current_epoch.as_usize() % spec.latest_slashed_exit_length; - - let total_at_start = self.latest_slashed_balances - [(epoch_index + 1) % spec.latest_slashed_exit_length]; - let total_at_end = self.latest_slashed_balances[epoch_index]; - let total_penalities = total_at_end.saturating_sub(total_at_start); - let penalty = std::cmp::max( - self.get_effective_balance(index, spec) - * std::cmp::min(total_penalities * 3, total_balance) - / total_balance, - self.get_effective_balance(index, spec) / spec.min_penalty_quotient, - ); - - safe_sub_assign!(self.validator_balances[index], penalty); - } - } - } - - /// Process the exit queue. - /// - /// Spec v0.4.0 - pub fn process_exit_queue(&mut self, spec: &ChainSpec) { - let current_epoch = self.current_epoch(spec); - - let eligible = |index: usize| { - let validator = &self.validator_registry[index]; - - if validator.withdrawable_epoch != spec.far_future_epoch { - false - } else { - current_epoch >= validator.exit_epoch + spec.min_validator_withdrawability_delay - } - }; - - let mut eligable_indices: Vec = (0..self.validator_registry.len()) - .filter(|i| eligible(*i)) - .collect(); - eligable_indices.sort_by_key(|i| self.validator_registry[*i].exit_epoch); - - for (withdrawn_so_far, index) in eligable_indices.iter().enumerate() { - if withdrawn_so_far as u64 >= spec.max_exit_dequeues_per_epoch { - break; - } - self.prepare_validator_for_withdrawal(*index, spec); - } - } - - /// Return the randao mix at a recent ``epoch``. - /// - /// Returns `None` if the epoch is out-of-bounds of `self.latest_randao_mixes`. - /// - /// Spec v0.2.0 - pub fn get_randao_mix(&self, epoch: Epoch, spec: &ChainSpec) -> Option<&Hash256> { - self.latest_randao_mixes - .get(epoch.as_usize() % spec.latest_randao_mixes_length) - } - - /// Update validator registry, activating/exiting validators if possible. - /// - /// Spec v0.4.0 - pub fn update_validator_registry(&mut self, spec: &ChainSpec) { - let current_epoch = self.current_epoch(spec); - let active_validator_indices = - get_active_validator_indices(&self.validator_registry, current_epoch); - let total_balance = self.get_total_balance(&active_validator_indices[..], spec); - - let max_balance_churn = std::cmp::max( - spec.max_deposit_amount, - total_balance / (2 * spec.max_balance_churn_quotient), - ); - - let mut balance_churn = 0; - for index in 0..self.validator_registry.len() { - let validator = &self.validator_registry[index]; - - if (validator.activation_epoch == spec.far_future_epoch) - & (self.validator_balances[index] == spec.max_deposit_amount) - { - balance_churn += self.get_effective_balance(index, spec); - if balance_churn > max_balance_churn { - break; - } - self.activate_validator(index, false, spec); - } - } - - let mut balance_churn = 0; - for index in 0..self.validator_registry.len() { - let validator = &self.validator_registry[index]; - - if (validator.exit_epoch == spec.far_future_epoch) & (validator.initiated_exit) { - balance_churn += self.get_effective_balance(index, spec); - if balance_churn > max_balance_churn { - break; - } - - self.exit_validator(index, spec); - } - } - - self.validator_registry_update_epoch = current_epoch; - } - - /// Confirm validator owns PublicKey - /// - /// Spec v0.2.0 - pub fn validate_proof_of_possession( + pub fn get_attestation_participants( &self, - pubkey: PublicKey, - proof_of_possession: Signature, - withdrawal_credentials: Hash256, + attestation_data: &AttestationData, + bitfield: &Bitfield, + spec: &ChainSpec, + ) -> Result, Error> { + let epoch = attestation_data.slot.epoch(spec.slots_per_epoch); + let relative_epoch = self.relative_epoch(epoch, spec)?; + let cache = self.cache(relative_epoch)?; + + let (committee_slot_index, committee_index) = cache + .shard_committee_index_map + .get(&attestation_data.shard) + .ok_or_else(|| Error::ShardOutOfBounds)?; + let (committee, shard) = &cache.committees[*committee_slot_index][*committee_index]; + + assert_eq!(*shard, attestation_data.shard, "Bad epoch cache build."); + + let mut participants = vec![]; + for (i, validator_index) in committee.iter().enumerate() { + if bitfield.get(i).unwrap() { + participants.push(*validator_index); + } + } + + Ok(participants) + } + + /// Return the effective balance (also known as "balance at stake") for a validator with the given ``index``. + /// + /// Spec v0.4.0 + pub fn get_effective_balance(&self, validator_index: usize, spec: &ChainSpec) -> u64 { + std::cmp::min( + self.validator_balances[validator_index], + spec.max_deposit_amount, + ) + } + + /// Return the combined effective balance of an array of validators. + /// + /// Spec v0.4.0 + pub fn get_total_balance(&self, validator_indices: &[usize], spec: &ChainSpec) -> u64 { + validator_indices + .iter() + .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)) + } + + /// Verify ``bitfield`` against the ``committee_size``. + /// + /// Spec v0.4.0 + pub fn verify_bitfield(&self, bitfield: &Bitfield, committee_size: usize) -> bool { + if bitfield.num_bytes() != ((committee_size + 7) / 8) { + return false; + } + + for i in committee_size..(bitfield.num_bytes() * 8) { + if bitfield.get(i).expect("Impossible due to previous check.") { + return false; + } + } + + true + } + + /// Verify validity of ``slashable_attestation`` fields. + /// + /// Spec v0.4.0 + pub fn verify_slashable_attestation( + &self, + slashable_attestation: &SlashableAttestation, spec: &ChainSpec, ) -> bool { - let proof_of_possession_data = DepositInput { - pubkey: pubkey.clone(), - withdrawal_credentials, - proof_of_possession: Signature::empty_signature(), - }; + if slashable_attestation.custody_bitfield.num_set_bits() > 0 { + return false; + } - proof_of_possession.verify( - &proof_of_possession_data.hash_tree_root(), - self.fork - .get_domain(self.slot.epoch(spec.slots_per_epoch), spec.domain_deposit), - &pubkey, + if slashable_attestation.validator_indices.is_empty() { + return false; + } + + for i in 0..(slashable_attestation.validator_indices.len() - 1) { + if slashable_attestation.validator_indices[i] + >= slashable_attestation.validator_indices[i + 1] + { + return false; + } + } + + if !self.verify_bitfield( + &slashable_attestation.custody_bitfield, + slashable_attestation.validator_indices.len(), + ) { + return false; + } + + if slashable_attestation.validator_indices.len() + > spec.max_indices_per_slashable_vote as usize + { + return false; + } + + let mut aggregate_pubs = vec![AggregatePublicKey::new(); 2]; + let mut message_exists = vec![false; 2]; + + for (i, v) in slashable_attestation.validator_indices.iter().enumerate() { + let custody_bit = match slashable_attestation.custody_bitfield.get(i) { + Ok(bit) => bit, + Err(_) => unreachable!(), + }; + + message_exists[custody_bit as usize] = true; + + match self.validator_registry.get(*v as usize) { + Some(validator) => { + aggregate_pubs[custody_bit as usize].add(&validator.pubkey); + } + None => return false, + }; + } + + let message_0 = AttestationDataAndCustodyBit { + data: slashable_attestation.data.clone(), + custody_bit: false, + } + .hash_tree_root(); + let message_1 = AttestationDataAndCustodyBit { + data: slashable_attestation.data.clone(), + custody_bit: true, + } + .hash_tree_root(); + + let mut messages = vec![]; + let mut keys = vec![]; + + if message_exists[0] { + messages.push(&message_0[..]); + keys.push(&aggregate_pubs[0]); + } + if message_exists[1] { + messages.push(&message_1[..]); + keys.push(&aggregate_pubs[1]); + } + + slashable_attestation.aggregate_signature.verify_multiple( + &messages[..], + spec.domain_attestation, + &keys[..], ) } + /// Return the epoch at which an activation or exit triggered in ``epoch`` takes effect. + /// + /// Spec v0.4.0 + pub fn get_delayed_activation_exit_epoch(&self, epoch: Epoch, spec: &ChainSpec) -> Epoch { + epoch + 1 + spec.activation_exit_delay + } + /// Process multiple deposits in sequence. /// /// Builds a hashmap of validator pubkeys to validator index and passes it to each successive /// call to `process_deposit(..)`. This requires much less computation than successive calls to /// `process_deposits(..)` without the hashmap. /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn process_deposits( &mut self, deposits: Vec<&DepositData>, @@ -1030,17 +850,339 @@ impl BeaconState { /// Initiate an exit for the validator of the given `index`. /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn prepare_validator_for_withdrawal(&mut self, validator_index: usize, spec: &ChainSpec) { //TODO: we're not ANDing here, we're setting. Potentially wrong. self.validator_registry[validator_index].withdrawable_epoch = self.current_epoch(spec) + spec.min_validator_withdrawability_delay; } + /// Returns the crosslink committees for some slot. + /// + /// Utilizes the cache and will fail if the appropriate cache is not initialized. + /// + /// Spec v0.4.0 + pub(crate) fn get_shuffling_for_slot( + &self, + slot: Slot, + registry_change: bool, + + spec: &ChainSpec, + ) -> Result>, Error> { + let (_committees_per_epoch, seed, shuffling_epoch, _shuffling_start_shard) = + self.get_committee_params_at_slot(slot, registry_change, spec)?; + + self.get_shuffling(seed, shuffling_epoch, spec) + } + + /// Shuffle ``validators`` into crosslink committees seeded by ``seed`` and ``epoch``. + /// + /// Return a list of ``committees_per_epoch`` committees where each + /// committee is itself a list of validator indices. + /// + /// Spec v0.4.0 + pub(crate) fn get_shuffling( + &self, + seed: Hash256, + epoch: Epoch, + spec: &ChainSpec, + ) -> Result>, Error> { + let active_validator_indices = + get_active_validator_indices(&self.validator_registry, epoch); + + if active_validator_indices.is_empty() { + error!("get_shuffling: no validators."); + return Err(Error::InsufficientValidators); + } + + debug!("Shuffling {} validators...", active_validator_indices.len()); + + let committees_per_epoch = spec.get_epoch_committee_count(active_validator_indices.len()); + + trace!( + "get_shuffling: active_validator_indices.len() == {}, committees_per_epoch: {}", + active_validator_indices.len(), + committees_per_epoch + ); + + let active_validator_indices: Vec = active_validator_indices.to_vec(); + + let shuffled_active_validator_indices = shuffle_list( + active_validator_indices, + spec.shuffle_round_count, + &seed[..], + true, + ) + .ok_or_else(|| Error::UnableToShuffle)?; + + Ok(shuffled_active_validator_indices + .honey_badger_split(committees_per_epoch as usize) + .map(|slice: &[usize]| slice.to_vec()) + .collect()) + } + + /// Returns the following params for the given slot: + /// + /// - epoch committee count + /// - epoch seed + /// - calculation epoch + /// - start shard + /// + /// In the spec, this functionality is included in the `get_crosslink_committees_at_slot(..)` + /// function. It is separated here to allow the division of shuffling and committee building, + /// as is required for efficient operations. + /// + /// Spec v0.4.0 + pub(crate) fn get_committee_params_at_slot( + &self, + slot: Slot, + registry_change: bool, + spec: &ChainSpec, + ) -> Result<(u64, Hash256, Epoch, u64), Error> { + let epoch = slot.epoch(spec.slots_per_epoch); + let current_epoch = self.current_epoch(spec); + let previous_epoch = self.previous_epoch(spec); + let next_epoch = self.next_epoch(spec); + + if epoch == current_epoch { + Ok(( + self.get_current_epoch_committee_count(spec), + self.current_shuffling_seed, + self.current_shuffling_epoch, + self.current_shuffling_start_shard, + )) + } else if epoch == previous_epoch { + Ok(( + self.get_previous_epoch_committee_count(spec), + self.previous_shuffling_seed, + self.previous_shuffling_epoch, + self.previous_shuffling_start_shard, + )) + } else if epoch == next_epoch { + let current_committees_per_epoch = self.get_current_epoch_committee_count(spec); + let epochs_since_last_registry_update = + current_epoch - self.validator_registry_update_epoch; + let (seed, shuffling_start_shard) = if registry_change { + let next_seed = self.generate_seed(next_epoch, spec)?; + ( + next_seed, + (self.current_shuffling_start_shard + current_committees_per_epoch) + % spec.shard_count, + ) + } else if (epochs_since_last_registry_update > 1) + & epochs_since_last_registry_update.is_power_of_two() + { + let next_seed = self.generate_seed(next_epoch, spec)?; + (next_seed, self.current_shuffling_start_shard) + } else { + ( + self.current_shuffling_seed, + self.current_shuffling_start_shard, + ) + }; + Ok(( + self.get_next_epoch_committee_count(spec), + seed, + next_epoch, + shuffling_start_shard, + )) + } else { + Err(Error::EpochOutOfBounds) + } + } + + /// Return the list of ``(committee, shard)`` tuples for the ``slot``. + /// + /// Note: There are two possible shufflings for crosslink committees for a + /// `slot` in the next epoch: with and without a `registry_change` + /// + /// Note: does not utilize the cache, `get_crosslink_committees_at_slot` is an equivalent + /// function which uses the cache. + /// + /// Spec v0.4.0 + pub(crate) fn calculate_crosslink_committees_at_slot( + &self, + slot: Slot, + registry_change: bool, + shuffling: Vec>, + spec: &ChainSpec, + ) -> Result, u64)>, Error> { + let (committees_per_epoch, _seed, _shuffling_epoch, shuffling_start_shard) = + self.get_committee_params_at_slot(slot, registry_change, spec)?; + + let offset = slot.as_u64() % spec.slots_per_epoch; + let committees_per_slot = committees_per_epoch / spec.slots_per_epoch; + let slot_start_shard = + (shuffling_start_shard + committees_per_slot * offset) % spec.shard_count; + + let mut crosslinks_at_slot = vec![]; + for i in 0..committees_per_slot { + let tuple = ( + shuffling[(committees_per_slot * offset + i) as usize].clone(), + (slot_start_shard + i) % spec.shard_count, + ); + crosslinks_at_slot.push(tuple) + } + Ok(crosslinks_at_slot) + } + + /// Returns the `slot`, `shard` and `committee_index` for which a validator must produce an + /// attestation. + /// + /// Only reads the current epoch. + /// + /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. + /// + /// Spec v0.4.0 + pub fn attestation_slot_and_shard_for_validator( + &self, + validator_index: usize, + _spec: &ChainSpec, + ) -> Result, Error> { + let cache = self.cache(RelativeEpoch::Current)?; + + Ok(cache + .attestation_duty_map + .get(&(validator_index as u64)) + .and_then(|tuple| Some(*tuple))) + } + + /// Process the slashings. + /// + /// Spec v0.4.0 + pub fn process_slashings(&mut self, spec: &ChainSpec) { + let current_epoch = self.current_epoch(spec); + let active_validator_indices = + get_active_validator_indices(&self.validator_registry, current_epoch); + let total_balance = self.get_total_balance(&active_validator_indices[..], spec); + + for (index, validator) in self.validator_registry.iter().enumerate() { + if validator.slashed + && (current_epoch + == validator.withdrawable_epoch + - Epoch::from(spec.latest_slashed_exit_length / 2)) + { + let epoch_index: usize = current_epoch.as_usize() % spec.latest_slashed_exit_length; + + let total_at_start = self.latest_slashed_balances + [(epoch_index + 1) % spec.latest_slashed_exit_length]; + let total_at_end = self.latest_slashed_balances[epoch_index]; + let total_penalities = total_at_end.saturating_sub(total_at_start); + let penalty = std::cmp::max( + self.get_effective_balance(index, spec) + * std::cmp::min(total_penalities * 3, total_balance) + / total_balance, + self.get_effective_balance(index, spec) / spec.min_penalty_quotient, + ); + + safe_sub_assign!(self.validator_balances[index], penalty); + } + } + } + + /// Process the exit queue. + /// + /// Spec v0.4.0 + pub fn process_exit_queue(&mut self, spec: &ChainSpec) { + let current_epoch = self.current_epoch(spec); + + let eligible = |index: usize| { + let validator = &self.validator_registry[index]; + + if validator.withdrawable_epoch != spec.far_future_epoch { + false + } else { + current_epoch >= validator.exit_epoch + spec.min_validator_withdrawability_delay + } + }; + + let mut eligable_indices: Vec = (0..self.validator_registry.len()) + .filter(|i| eligible(*i)) + .collect(); + eligable_indices.sort_by_key(|i| self.validator_registry[*i].exit_epoch); + + for (withdrawn_so_far, index) in eligable_indices.iter().enumerate() { + if withdrawn_so_far as u64 >= spec.max_exit_dequeues_per_epoch { + break; + } + self.prepare_validator_for_withdrawal(*index, spec); + } + } + + /// Update validator registry, activating/exiting validators if possible. + /// + /// Spec v0.4.0 + pub fn update_validator_registry(&mut self, spec: &ChainSpec) { + let current_epoch = self.current_epoch(spec); + let active_validator_indices = + get_active_validator_indices(&self.validator_registry, current_epoch); + let total_balance = self.get_total_balance(&active_validator_indices[..], spec); + + let max_balance_churn = std::cmp::max( + spec.max_deposit_amount, + total_balance / (2 * spec.max_balance_churn_quotient), + ); + + let mut balance_churn = 0; + for index in 0..self.validator_registry.len() { + let validator = &self.validator_registry[index]; + + if (validator.activation_epoch == spec.far_future_epoch) + & (self.validator_balances[index] == spec.max_deposit_amount) + { + balance_churn += self.get_effective_balance(index, spec); + if balance_churn > max_balance_churn { + break; + } + self.activate_validator(index, false, spec); + } + } + + let mut balance_churn = 0; + for index in 0..self.validator_registry.len() { + let validator = &self.validator_registry[index]; + + if (validator.exit_epoch == spec.far_future_epoch) & (validator.initiated_exit) { + balance_churn += self.get_effective_balance(index, spec); + if balance_churn > max_balance_churn { + break; + } + + self.exit_validator(index, spec); + } + } + + self.validator_registry_update_epoch = current_epoch; + } + + /// Confirm validator owns PublicKey + /// + /// Spec v0.4.0 + pub fn validate_proof_of_possession( + &self, + pubkey: PublicKey, + proof_of_possession: Signature, + withdrawal_credentials: Hash256, + spec: &ChainSpec, + ) -> bool { + let proof_of_possession_data = DepositInput { + pubkey: pubkey.clone(), + withdrawal_credentials, + proof_of_possession: Signature::empty_signature(), + }; + + proof_of_possession.verify( + &proof_of_possession_data.hash_tree_root(), + self.fork + .get_domain(self.slot.epoch(spec.slots_per_epoch), spec.domain_deposit), + &pubkey, + ) + } + /// Iterate through the validator registry and eject active validators with balance below /// ``EJECTION_BALANCE``. /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn process_ejections(&mut self, spec: &ChainSpec) { for validator_index in get_active_validator_indices(&self.validator_registry, self.current_epoch(spec)) @@ -1055,7 +1197,7 @@ impl BeaconState { /// /// Note: this is defined "inline" in the spec, not as a helper function. /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn inactivity_penalty( &self, validator_index: usize, @@ -1075,7 +1217,7 @@ impl BeaconState { /// /// Note: In the spec this is defined "inline", not as a helper function. /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn inclusion_distance( &self, attestations: &[&PendingAttestation], @@ -1091,7 +1233,7 @@ impl BeaconState { /// /// Note: In the spec this is defined "inline", not as a helper function. /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn inclusion_slot( &self, attestations: &[&PendingAttestation], @@ -1107,7 +1249,7 @@ impl BeaconState { /// /// Note: In the spec this is defined "inline", not as a helper function. /// - /// Spec v0.2.0 + /// Spec v0.4.0 fn earliest_included_attestation( &self, attestations: &[&PendingAttestation], @@ -1135,7 +1277,7 @@ impl BeaconState { /// /// Note: In the spec this is defined "inline", not as a helper function. /// - /// Spec v0.2.0 + /// Spec v0.4.0 pub fn base_reward( &self, validator_index: usize, @@ -1145,141 +1287,9 @@ impl BeaconState { self.get_effective_balance(validator_index, spec) / base_reward_quotient / 5 } - /// Return the combined effective balance of an array of validators. + /// Returns the union of all participants in the provided attestations /// - /// Spec v0.2.0 - pub fn get_total_balance(&self, validator_indices: &[usize], spec: &ChainSpec) -> u64 { - validator_indices - .iter() - .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)) - } - - /// Return the effective balance (also known as "balance at stake") for a validator with the given ``index``. - /// - /// Spec v0.2.0 - pub fn get_effective_balance(&self, validator_index: usize, spec: &ChainSpec) -> u64 { - std::cmp::min( - self.validator_balances[validator_index], - spec.max_deposit_amount, - ) - } - - /// Verify ``bitfield`` against the ``committee_size``. - /// - /// Spec v0.2.0 - pub fn verify_bitfield(&self, bitfield: &Bitfield, committee_size: usize) -> bool { - if bitfield.num_bytes() != ((committee_size + 7) / 8) { - return false; - } - - for i in committee_size..(bitfield.num_bytes() * 8) { - match bitfield.get(i) { - Ok(bit) => { - if bit { - return false; - } - } - Err(_) => unreachable!(), - } - } - - true - } - - /// Verify validity of ``slashable_attestation`` fields. - /// - /// Spec v0.2.0 - pub fn verify_slashable_attestation( - &self, - slashable_attestation: &SlashableAttestation, - spec: &ChainSpec, - ) -> bool { - if slashable_attestation.custody_bitfield.num_set_bits() > 0 { - return false; - } - - if slashable_attestation.validator_indices.is_empty() { - return false; - } - - for i in 0..(slashable_attestation.validator_indices.len() - 1) { - if slashable_attestation.validator_indices[i] - >= slashable_attestation.validator_indices[i + 1] - { - return false; - } - } - - if !self.verify_bitfield( - &slashable_attestation.custody_bitfield, - slashable_attestation.validator_indices.len(), - ) { - return false; - } - - if slashable_attestation.validator_indices.len() - > spec.max_indices_per_slashable_vote as usize - { - return false; - } - - let mut aggregate_pubs = vec![AggregatePublicKey::new(); 2]; - let mut message_exists = vec![false; 2]; - - for (i, v) in slashable_attestation.validator_indices.iter().enumerate() { - let custody_bit = match slashable_attestation.custody_bitfield.get(i) { - Ok(bit) => bit, - Err(_) => unreachable!(), - }; - - message_exists[custody_bit as usize] = true; - - match self.validator_registry.get(*v as usize) { - Some(validator) => { - aggregate_pubs[custody_bit as usize].add(&validator.pubkey); - } - None => return false, - }; - } - - let message_0 = AttestationDataAndCustodyBit { - data: slashable_attestation.data.clone(), - custody_bit: false, - } - .hash_tree_root(); - let message_1 = AttestationDataAndCustodyBit { - data: slashable_attestation.data.clone(), - custody_bit: true, - } - .hash_tree_root(); - - let mut messages = vec![]; - let mut keys = vec![]; - - if message_exists[0] { - messages.push(&message_0[..]); - keys.push(&aggregate_pubs[0]); - } - if message_exists[1] { - messages.push(&message_1[..]); - keys.push(&aggregate_pubs[1]); - } - - slashable_attestation.aggregate_signature.verify_multiple( - &messages[..], - spec.domain_attestation, - &keys[..], - ) - } - - /// Return the block root at a recent `slot`. - /// - /// Spec v0.2.0 - pub fn get_block_root(&self, slot: Slot, spec: &ChainSpec) -> Option<&Hash256> { - self.latest_block_roots - .get(slot.as_usize() % spec.latest_block_roots_length) - } - + /// Spec v0.4.0 pub fn get_attestation_participants_union( &self, attestations: &[&PendingAttestation], @@ -1299,39 +1309,6 @@ impl BeaconState { all_participants.dedup(); Ok(all_participants) } - - /// Returns the list of validator indices which participiated in the attestation. - /// - /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. - /// - /// Spec v0.2.0 - pub fn get_attestation_participants( - &self, - attestation_data: &AttestationData, - bitfield: &Bitfield, - spec: &ChainSpec, - ) -> Result, Error> { - let epoch = attestation_data.slot.epoch(spec.slots_per_epoch); - let relative_epoch = self.relative_epoch(epoch, spec)?; - let cache = self.cache(relative_epoch)?; - - let (committee_slot_index, committee_index) = cache - .shard_committee_index_map - .get(&attestation_data.shard) - .ok_or_else(|| Error::ShardOutOfBounds)?; - let (committee, shard) = &cache.committees[*committee_slot_index][*committee_index]; - - assert_eq!(*shard, attestation_data.shard, "Bad epoch cache build."); - - let mut participants = vec![]; - for (i, validator_index) in committee.iter().enumerate() { - if bitfield.get(i).unwrap() { - participants.push(*validator_index); - } - } - - Ok(participants) - } } fn hash_tree_root(input: Vec) -> Hash256 { diff --git a/eth2/types/src/beacon_state/helpers.rs b/eth2/types/src/beacon_state/helpers.rs new file mode 100644 index 000000000..e69de29bb diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs index 8aef79466..dea079755 100644 --- a/eth2/types/src/chain_spec.rs +++ b/eth2/types/src/chain_spec.rs @@ -93,8 +93,20 @@ pub struct ChainSpec { pub domain_randao: u64, pub domain_transfer: u64, } - impl ChainSpec { + /// Return the number of committees in one epoch. + /// + /// Spec v0.4.0 + pub fn get_epoch_committee_count(&self, active_validator_count: usize) -> u64 { + std::cmp::max( + 1, + std::cmp::min( + self.shard_count / self.slots_per_epoch, + active_validator_count as u64 / self.slots_per_epoch / self.target_committee_size, + ), + ) * self.slots_per_epoch + } + /// Returns a `ChainSpec` compatible with the Ethereum Foundation specification. /// /// Spec v0.4.0 @@ -190,9 +202,7 @@ impl ChainSpec { domain_transfer: 5, } } -} -impl ChainSpec { /// Returns a `ChainSpec` compatible with the specification suitable for 8 validators. /// /// Spec v0.4.0 diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs index 783a65302..6c524a2d9 100644 --- a/eth2/types/src/fork.rs +++ b/eth2/types/src/fork.rs @@ -16,6 +16,8 @@ pub struct Fork { impl Fork { /// Return the fork version of the given ``epoch``. + /// + /// Spec v0.4.0 pub fn get_fork_version(&self, epoch: Epoch) -> u64 { if epoch < self.epoch { return self.previous_version; @@ -24,6 +26,8 @@ impl Fork { } /// Get the domain number that represents the fork meta and signature domain. + /// + /// Spec v0.4.0 pub fn get_domain(&self, epoch: Epoch, domain_type: u64) -> u64 { let fork_version = self.get_fork_version(epoch); fork_version * u64::pow(2, 32) + domain_type diff --git a/eth2/types/src/readers/state_reader.rs b/eth2/types/src/readers/state_reader.rs index 92a870855..e469bee57 100644 --- a/eth2/types/src/readers/state_reader.rs +++ b/eth2/types/src/readers/state_reader.rs @@ -1,4 +1,4 @@ -use crate::{BeaconState, Hash256, Slot}; +use crate::{BeaconState, Slot}; use std::fmt::Debug; /// The `BeaconStateReader` provides interfaces for reading a subset of fields of a `BeaconState`. @@ -11,7 +11,6 @@ use std::fmt::Debug; /// "future proofing". pub trait BeaconStateReader: Debug + PartialEq { fn slot(&self) -> Slot; - fn canonical_root(&self) -> Hash256; fn into_beacon_state(self) -> Option; } @@ -20,10 +19,6 @@ impl BeaconStateReader for BeaconState { self.slot } - fn canonical_root(&self) -> Hash256 { - self.canonical_root() - } - fn into_beacon_state(self) -> Option { Some(self) } diff --git a/eth2/types/src/validator_registry.rs b/eth2/types/src/validator_registry.rs index 20863dd72..7b55e78cb 100644 --- a/eth2/types/src/validator_registry.rs +++ b/eth2/types/src/validator_registry.rs @@ -4,6 +4,8 @@ use super::validator::*; use crate::Epoch; /// Given an indexed sequence of `validators`, return the indices corresponding to validators that are active at `epoch`. +/// +/// Spec v0.4.0 pub fn get_active_validator_indices(validators: &[Validator], epoch: Epoch) -> Vec { validators .iter() From 6253167cacfb34a317ce4915920af3db2d1055f8 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 4 Mar 2019 17:19:25 +1100 Subject: [PATCH 111/132] Update ethereum-types to 0.5 --- .../db/src/stores/beacon_block_store.rs | 51 +++++++++---------- beacon_node/db/src/stores/macros.rs | 40 +++++++-------- beacon_node/db/src/stores/pow_chain_store.rs | 8 +-- eth2/fork_choice/src/bitwise_lmd_ghost.rs | 2 +- .../state_processing/src/block_processable.rs | 3 +- .../state_processing/src/epoch_processable.rs | 2 +- eth2/types/Cargo.toml | 3 +- eth2/types/src/attestation.rs | 2 +- eth2/types/src/attestation_data.rs | 2 +- eth2/types/src/attester_slashing/builder.rs | 4 +- eth2/types/src/beacon_block.rs | 4 +- eth2/types/src/beacon_state.rs | 12 +++-- eth2/types/src/beacon_state/builder.rs | 4 +- eth2/types/src/proposer_slashing/builder.rs | 4 +- eth2/types/src/test_utils/address.rs | 2 +- eth2/types/src/test_utils/hash256.rs | 2 +- eth2/utils/ssz/Cargo.toml | 2 +- eth2/utils/ssz/src/impl_decode.rs | 8 +-- eth2/utils/ssz/src/impl_encode.rs | 4 +- 19 files changed, 80 insertions(+), 79 deletions(-) diff --git a/beacon_node/db/src/stores/beacon_block_store.rs b/beacon_node/db/src/stores/beacon_block_store.rs index 8a1fc2b0d..bd5149cfd 100644 --- a/beacon_node/db/src/stores/beacon_block_store.rs +++ b/beacon_node/db/src/stores/beacon_block_store.rs @@ -134,9 +134,9 @@ mod tests { let store = BeaconBlockStore::new(db.clone()); let ssz = "definitly not a valid block".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()); + let hash = &Hash256::from([0xAA; 32]); - db.put(DB_COLUMN, hash, ssz).unwrap(); + db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); assert_eq!( store.block_at_slot(hash, Slot::from(42_u64)), Err(BeaconBlockAtSlotError::DBError( @@ -151,10 +151,10 @@ mod tests { let store = BeaconBlockStore::new(db.clone()); let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()); - let other_hash = &Hash256::from("another hash".as_bytes()); + let hash = &Hash256::from([0xAA; 32]); + let other_hash = &Hash256::from([0xBB; 32]); - db.put(DB_COLUMN, hash, ssz).unwrap(); + db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); assert_eq!( store.block_at_slot(other_hash, Slot::from(42_u64)), Err(BeaconBlockAtSlotError::UnknownBeaconBlock(*other_hash)) @@ -169,18 +169,15 @@ mod tests { let thread_count = 10; let write_count = 10; - // We're expecting the product of these numbers to fit in one byte. - assert!(thread_count * write_count <= 255); - let mut handles = vec![]; for t in 0..thread_count { let wc = write_count; let bs = bs.clone(); let handle = thread::spawn(move || { for w in 0..wc { - let key = (t * w) as u8; + let key = t * w; let val = 42; - bs.put(&[key][..].into(), &vec![val]).unwrap(); + bs.put(&Hash256::from_low_u64_le(key), &vec![val]).unwrap(); } }); handles.push(handle); @@ -192,9 +189,9 @@ mod tests { for t in 0..thread_count { for w in 0..write_count { - let key = (t * w) as u8; - assert!(bs.exists(&[key][..].into()).unwrap()); - let val = bs.get(&[key][..].into()).unwrap().unwrap(); + let key = t * w; + assert!(bs.exists(&Hash256::from_low_u64_le(key)).unwrap()); + let val = bs.get(&Hash256::from_low_u64_le(key)).unwrap().unwrap(); assert_eq!(vec![42], val); } } @@ -208,19 +205,20 @@ mod tests { // Specify test block parameters. let hashes = [ - Hash256::from(&[0; 32][..]), - Hash256::from(&[1; 32][..]), - Hash256::from(&[2; 32][..]), - Hash256::from(&[3; 32][..]), - Hash256::from(&[4; 32][..]), + Hash256::from([0; 32]), + Hash256::from([1; 32]), + Hash256::from([2; 32]), + Hash256::from([3; 32]), + Hash256::from([4; 32]), ]; let parent_hashes = [ - Hash256::from(&[255; 32][..]), // Genesis block. - Hash256::from(&[0; 32][..]), - Hash256::from(&[1; 32][..]), - Hash256::from(&[2; 32][..]), - Hash256::from(&[3; 32][..]), + Hash256::from([255; 32]), // Genesis block. + Hash256::from([0; 32]), + Hash256::from([1; 32]), + Hash256::from([2; 32]), + Hash256::from([3; 32]), ]; + let unknown_hash = Hash256::from([101; 32]); // different from all above let slots: Vec = vec![0, 1, 3, 4, 5].iter().map(|x| Slot::new(*x)).collect(); // Generate a vec of random blocks and store them in the DB. @@ -233,7 +231,7 @@ mod tests { block.slot = slots[i]; let ssz = ssz_encode(&block); - db.put(DB_COLUMN, &hashes[i], &ssz).unwrap(); + db.put(DB_COLUMN, hashes[i].as_bytes(), &ssz).unwrap(); blocks.push(block); } @@ -255,11 +253,10 @@ mod tests { let ssz = bs.block_at_slot(&hashes[4], Slot::new(6)).unwrap(); assert_eq!(ssz, None); - let bad_hash = &Hash256::from("unknown".as_bytes()); - let ssz = bs.block_at_slot(bad_hash, Slot::new(2)); + let ssz = bs.block_at_slot(&unknown_hash, Slot::new(2)); assert_eq!( ssz, - Err(BeaconBlockAtSlotError::UnknownBeaconBlock(*bad_hash)) + Err(BeaconBlockAtSlotError::UnknownBeaconBlock(unknown_hash)) ); } } diff --git a/beacon_node/db/src/stores/macros.rs b/beacon_node/db/src/stores/macros.rs index e26e101c9..6c53e40ee 100644 --- a/beacon_node/db/src/stores/macros.rs +++ b/beacon_node/db/src/stores/macros.rs @@ -2,19 +2,19 @@ macro_rules! impl_crud_for_store { ($store: ident, $db_column: expr) => { impl $store { pub fn put(&self, hash: &Hash256, ssz: &[u8]) -> Result<(), DBError> { - self.db.put($db_column, hash, ssz) + self.db.put($db_column, hash.as_bytes(), ssz) } pub fn get(&self, hash: &Hash256) -> Result>, DBError> { - self.db.get($db_column, hash) + self.db.get($db_column, hash.as_bytes()) } pub fn exists(&self, hash: &Hash256) -> Result { - self.db.exists($db_column, hash) + self.db.exists($db_column, hash.as_bytes()) } pub fn delete(&self, hash: &Hash256) -> Result<(), DBError> { - self.db.delete($db_column, hash) + self.db.delete($db_column, hash.as_bytes()) } } }; @@ -29,10 +29,10 @@ macro_rules! test_crud_for_store { let store = $store::new(db.clone()); let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()); + let hash = &Hash256::from([0xAA; 32]); store.put(hash, ssz).unwrap(); - assert_eq!(db.get(DB_COLUMN, hash).unwrap().unwrap(), ssz); + assert_eq!(db.get(DB_COLUMN, hash.as_bytes()).unwrap().unwrap(), ssz); } #[test] @@ -41,9 +41,9 @@ macro_rules! test_crud_for_store { let store = $store::new(db.clone()); let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()); + let hash = &Hash256::from([0xAA; 32]); - db.put(DB_COLUMN, hash, ssz).unwrap(); + db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); assert_eq!(store.get(hash).unwrap().unwrap(), ssz); } @@ -53,10 +53,10 @@ macro_rules! test_crud_for_store { let store = $store::new(db.clone()); let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()); - let other_hash = &Hash256::from("another hash".as_bytes()); + let hash = &Hash256::from([0xAA; 32]); + let other_hash = &Hash256::from([0xBB; 32]); - db.put(DB_COLUMN, other_hash, ssz).unwrap(); + db.put(DB_COLUMN, other_hash.as_bytes(), ssz).unwrap(); assert_eq!(store.get(hash).unwrap(), None); } @@ -66,9 +66,9 @@ macro_rules! test_crud_for_store { let store = $store::new(db.clone()); let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()); + let hash = &Hash256::from([0xAA; 32]); - db.put(DB_COLUMN, hash, ssz).unwrap(); + db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); assert!(store.exists(hash).unwrap()); } @@ -78,10 +78,10 @@ macro_rules! test_crud_for_store { let store = $store::new(db.clone()); let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()); - let other_hash = &Hash256::from("another hash".as_bytes()); + let hash = &Hash256::from([0xAA; 32]); + let other_hash = &Hash256::from([0xBB; 32]); - db.put(DB_COLUMN, hash, ssz).unwrap(); + db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); assert!(!store.exists(other_hash).unwrap()); } @@ -91,13 +91,13 @@ macro_rules! test_crud_for_store { let store = $store::new(db.clone()); let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()); + let hash = &Hash256::from([0xAA; 32]); - db.put(DB_COLUMN, hash, ssz).unwrap(); - assert!(db.exists(DB_COLUMN, hash).unwrap()); + db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); + assert!(db.exists(DB_COLUMN, hash.as_bytes()).unwrap()); store.delete(hash).unwrap(); - assert!(!db.exists(DB_COLUMN, hash).unwrap()); + assert!(!db.exists(DB_COLUMN, hash.as_bytes()).unwrap()); } }; } diff --git a/beacon_node/db/src/stores/pow_chain_store.rs b/beacon_node/db/src/stores/pow_chain_store.rs index a7c77bab5..5c8b97907 100644 --- a/beacon_node/db/src/stores/pow_chain_store.rs +++ b/beacon_node/db/src/stores/pow_chain_store.rs @@ -37,7 +37,7 @@ mod tests { let db = Arc::new(MemoryDB::open()); let store = PoWChainStore::new(db.clone()); - let hash = &Hash256::from("some hash".as_bytes()).to_vec(); + let hash = &Hash256::from([0xAA; 32]).as_bytes().to_vec(); store.put_block_hash(hash).unwrap(); assert!(db.exists(DB_COLUMN, hash).unwrap()); @@ -48,7 +48,7 @@ mod tests { let db = Arc::new(MemoryDB::open()); let store = PoWChainStore::new(db.clone()); - let hash = &Hash256::from("some hash".as_bytes()).to_vec(); + let hash = &Hash256::from([0xAA; 32]).as_bytes().to_vec(); db.put(DB_COLUMN, hash, &[0]).unwrap(); assert!(store.block_hash_exists(hash).unwrap()); @@ -59,8 +59,8 @@ mod tests { let db = Arc::new(MemoryDB::open()); let store = PoWChainStore::new(db.clone()); - let hash = &Hash256::from("some hash".as_bytes()).to_vec(); - let other_hash = &Hash256::from("another hash".as_bytes()).to_vec(); + let hash = &Hash256::from([0xAA; 32]).as_bytes().to_vec(); + let other_hash = &Hash256::from([0xBB; 32]).as_bytes().to_vec(); db.put(DB_COLUMN, hash, &[0]).unwrap(); assert!(!store.block_hash_exists(other_hash).unwrap()); diff --git a/eth2/fork_choice/src/bitwise_lmd_ghost.rs b/eth2/fork_choice/src/bitwise_lmd_ghost.rs index 60aa38fe7..8a26b3e3c 100644 --- a/eth2/fork_choice/src/bitwise_lmd_ghost.rs +++ b/eth2/fork_choice/src/bitwise_lmd_ghost.rs @@ -210,7 +210,7 @@ where trace!("Child vote length: {}", votes.len()); for (candidate, votes) in votes.iter() { - let candidate_bit: BitVec = BitVec::from_bytes(&candidate); + let candidate_bit: BitVec = BitVec::from_bytes(candidate.as_bytes()); // if the bitmasks don't match, exclude candidate if !bitmask.iter().eq(candidate_bit.iter().take(bit)) { diff --git a/eth2/state_processing/src/block_processable.rs b/eth2/state_processing/src/block_processable.rs index 32327aad3..a54bb96d4 100644 --- a/eth2/state_processing/src/block_processable.rs +++ b/eth2/state_processing/src/block_processable.rs @@ -134,9 +134,10 @@ fn per_block_processing_signature_optional( let new_mix = { let mut mix = state.latest_randao_mixes [state.slot.as_usize() % spec.latest_randao_mixes_length] + .as_bytes() .to_vec(); mix.append(&mut ssz_encode(&block.randao_reveal)); - Hash256::from(&hash(&mix)[..]) + Hash256::from_slice(&hash(&mix)[..]) }; state.latest_randao_mixes[state.slot.as_usize() % spec.latest_randao_mixes_length] = new_mix; diff --git a/eth2/state_processing/src/epoch_processable.rs b/eth2/state_processing/src/epoch_processable.rs index 0ecd1bbd1..ff6f18113 100644 --- a/eth2/state_processing/src/epoch_processable.rs +++ b/eth2/state_processing/src/epoch_processable.rs @@ -626,7 +626,7 @@ impl EpochProcessable for BeaconState { } fn hash_tree_root(input: Vec) -> Hash256 { - Hash256::from(&input.hash_tree_root()[..]) + Hash256::from_slice(&input.hash_tree_root()[..]) } fn winning_root( diff --git a/eth2/types/Cargo.toml b/eth2/types/Cargo.toml index f70e8b490..1b9eed9b8 100644 --- a/eth2/types/Cargo.toml +++ b/eth2/types/Cargo.toml @@ -7,7 +7,7 @@ edition = "2018" [dependencies] bls = { path = "../utils/bls" } boolean-bitfield = { path = "../utils/boolean-bitfield" } -ethereum-types = "0.4.0" +ethereum-types = "0.5" hashing = { path = "../utils/hashing" } honey-badger-split = { path = "../utils/honey-badger-split" } log = "0.4" @@ -21,6 +21,7 @@ ssz = { path = "../utils/ssz" } ssz_derive = { path = "../utils/ssz_derive" } swap_or_not_shuffle = { path = "../utils/swap_or_not_shuffle" } test_random_derive = { path = "../utils/test_random_derive" } +int_to_bytes = { path = "../utils/int_to_bytes" } [dev-dependencies] env_logger = "0.6.0" diff --git a/eth2/types/src/attestation.rs b/eth2/types/src/attestation.rs index 4ac81bb4c..eff9512ab 100644 --- a/eth2/types/src/attestation.rs +++ b/eth2/types/src/attestation.rs @@ -16,7 +16,7 @@ pub struct Attestation { impl Attestation { pub fn canonical_root(&self) -> Hash256 { - Hash256::from(&self.hash_tree_root()[..]) + Hash256::from_slice(&self.hash_tree_root()[..]) } pub fn signable_message(&self, custody_bit: bool) -> Vec { diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data.rs index 73b5facfa..a7351125f 100644 --- a/eth2/types/src/attestation_data.rs +++ b/eth2/types/src/attestation_data.rs @@ -35,7 +35,7 @@ impl Eq for AttestationData {} impl AttestationData { pub fn canonical_root(&self) -> Hash256 { - Hash256::from(&self.hash_tree_root()[..]) + Hash256::from_slice(&self.hash_tree_root()[..]) } pub fn signable_message(&self, custody_bit: bool) -> Vec { diff --git a/eth2/types/src/attester_slashing/builder.rs b/eth2/types/src/attester_slashing/builder.rs index ed203d6e1..abade77c9 100644 --- a/eth2/types/src/attester_slashing/builder.rs +++ b/eth2/types/src/attester_slashing/builder.rs @@ -27,8 +27,8 @@ impl AttesterSlashingBuilder { let shard = 0; let justified_epoch = Epoch::new(0); let epoch = Epoch::new(0); - let hash_1 = Hash256::from(&[1][..]); - let hash_2 = Hash256::from(&[2][..]); + let hash_1 = Hash256::from_low_u64_le(1); + let hash_2 = Hash256::from_low_u64_le(2); let mut slashable_attestation_1 = SlashableAttestation { validator_indices: validator_indices.to_vec(), diff --git a/eth2/types/src/beacon_block.rs b/eth2/types/src/beacon_block.rs index a10fd2a6e..f69cdaec9 100644 --- a/eth2/types/src/beacon_block.rs +++ b/eth2/types/src/beacon_block.rs @@ -42,7 +42,7 @@ impl BeaconBlock { } pub fn canonical_root(&self) -> Hash256 { - Hash256::from(&self.hash_tree_root()[..]) + Hash256::from_slice(&self.hash_tree_root()[..]) } pub fn proposal_root(&self, spec: &ChainSpec) -> Hash256 { @@ -57,7 +57,7 @@ impl BeaconBlock { shard: spec.beacon_chain_shard_number, block_root: block_without_signature_root, }; - Hash256::from(&proposal.hash_tree_root()[..]) + Hash256::from_slice(&proposal.hash_tree_root()[..]) } } diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index a9e2f2673..5c87d274a 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -3,6 +3,7 @@ use crate::test_utils::TestRandom; use crate::{validator::StatusFlags, validator_registry::get_active_validator_indices, *}; use bls::verify_proof_of_possession; use honey_badger_split::SplitExt; +use int_to_bytes::int_to_bytes32; use log::{debug, error, trace}; use rand::RngCore; use rayon::prelude::*; @@ -326,7 +327,7 @@ impl BeaconState { /// /// Spec v0.2.0 pub fn canonical_root(&self) -> Hash256 { - Hash256::from(&self.hash_tree_root()[..]) + Hash256::from_slice(&self.hash_tree_root()[..]) } /// The epoch corresponding to `self.slot`. @@ -487,19 +488,20 @@ impl BeaconState { let mut input = self .get_randao_mix(epoch, spec) .ok_or_else(|| Error::InsufficientRandaoMixes)? + .as_bytes() .to_vec(); input.append( &mut self .get_active_index_root(epoch, spec) .ok_or_else(|| Error::InsufficientIndexRoots)? + .as_bytes() .to_vec(), ); - // TODO: ensure `Hash256::from(u64)` == `int_to_bytes32`. - input.append(&mut Hash256::from(epoch.as_u64()).to_vec()); + input.append(&mut int_to_bytes32(epoch.as_u64())); - Ok(Hash256::from(&hash(&input[..])[..])) + Ok(Hash256::from_slice(&hash(&input[..])[..])) } /// Returns the crosslink committees for some slot. @@ -1311,7 +1313,7 @@ impl BeaconState { } fn hash_tree_root(input: Vec) -> Hash256 { - Hash256::from(&input.hash_tree_root()[..]) + Hash256::from_slice(&input.hash_tree_root()[..]) } impl From for InclusionError { diff --git a/eth2/types/src/beacon_state/builder.rs b/eth2/types/src/beacon_state/builder.rs index 02886a86e..4889b5b5a 100644 --- a/eth2/types/src/beacon_state/builder.rs +++ b/eth2/types/src/beacon_state/builder.rs @@ -145,8 +145,8 @@ impl BeaconStateBuilder { state.previous_calculation_epoch = epoch - 1; state.current_calculation_epoch = epoch; - state.previous_epoch_seed = Hash256::from(&b"previous_seed"[..]); - state.current_epoch_seed = Hash256::from(&b"current_seed"[..]); + state.previous_epoch_seed = Hash256::from([0x01; 32]); + state.current_epoch_seed = Hash256::from([0x02; 32]); state.previous_justified_epoch = epoch - 2; state.justified_epoch = epoch - 1; diff --git a/eth2/types/src/proposer_slashing/builder.rs b/eth2/types/src/proposer_slashing/builder.rs index 7923ff74d..c9d2ff272 100644 --- a/eth2/types/src/proposer_slashing/builder.rs +++ b/eth2/types/src/proposer_slashing/builder.rs @@ -25,13 +25,13 @@ impl ProposerSlashingBuilder { let proposal_data_1 = ProposalSignedData { slot, shard, - block_root: Hash256::from(&[1][..]), + block_root: Hash256::from_low_u64_le(1), }; let proposal_data_2 = ProposalSignedData { slot, shard, - block_root: Hash256::from(&[2][..]), + block_root: Hash256::from_low_u64_le(2), }; let proposal_signature_1 = { diff --git a/eth2/types/src/test_utils/address.rs b/eth2/types/src/test_utils/address.rs index 2d60b72da..13de2dec9 100644 --- a/eth2/types/src/test_utils/address.rs +++ b/eth2/types/src/test_utils/address.rs @@ -6,6 +6,6 @@ impl TestRandom for Address { fn random_for_test(rng: &mut T) -> Self { let mut key_bytes = vec![0; 20]; rng.fill_bytes(&mut key_bytes); - Address::from(&key_bytes[..]) + Address::from_slice(&key_bytes[..]) } } diff --git a/eth2/types/src/test_utils/hash256.rs b/eth2/types/src/test_utils/hash256.rs index 98f5e7899..a227679da 100644 --- a/eth2/types/src/test_utils/hash256.rs +++ b/eth2/types/src/test_utils/hash256.rs @@ -6,6 +6,6 @@ impl TestRandom for Hash256 { fn random_for_test(rng: &mut T) -> Self { let mut key_bytes = vec![0; 32]; rng.fill_bytes(&mut key_bytes); - Hash256::from(&key_bytes[..]) + Hash256::from_slice(&key_bytes[..]) } } diff --git a/eth2/utils/ssz/Cargo.toml b/eth2/utils/ssz/Cargo.toml index 25326cb5b..f13db5def 100644 --- a/eth2/utils/ssz/Cargo.toml +++ b/eth2/utils/ssz/Cargo.toml @@ -6,5 +6,5 @@ edition = "2018" [dependencies] bytes = "0.4.9" -ethereum-types = "0.4.0" +ethereum-types = "0.5" hashing = { path = "../hashing" } diff --git a/eth2/utils/ssz/src/impl_decode.rs b/eth2/utils/ssz/src/impl_decode.rs index b9ca48f9b..b13cbeb5d 100644 --- a/eth2/utils/ssz/src/impl_decode.rs +++ b/eth2/utils/ssz/src/impl_decode.rs @@ -59,7 +59,7 @@ impl Decodable for H256 { if bytes.len() < 32 || bytes.len() - 32 < index { Err(DecodeError::TooShort) } else { - Ok((H256::from(&bytes[index..(index + 32)]), index + 32)) + Ok((H256::from_slice(&bytes[index..(index + 32)]), index + 32)) } } } @@ -69,7 +69,7 @@ impl Decodable for Address { if bytes.len() < 20 || bytes.len() - 20 < index { Err(DecodeError::TooShort) } else { - Ok((Address::from(&bytes[index..(index + 20)]), index + 20)) + Ok((Address::from_slice(&bytes[index..(index + 20)]), index + 20)) } } } @@ -95,7 +95,7 @@ mod tests { */ let input = vec![42_u8; 32]; let (decoded, i) = H256::ssz_decode(&input, 0).unwrap(); - assert_eq!(decoded.to_vec(), input); + assert_eq!(decoded.as_bytes(), &input[..]); assert_eq!(i, 32); /* @@ -104,7 +104,7 @@ mod tests { let mut input = vec![42_u8; 32]; input.push(12); let (decoded, i) = H256::ssz_decode(&input, 0).unwrap(); - assert_eq!(decoded.to_vec()[..], input[0..32]); + assert_eq!(decoded.as_bytes(), &input[0..32]); assert_eq!(i, 32); /* diff --git a/eth2/utils/ssz/src/impl_encode.rs b/eth2/utils/ssz/src/impl_encode.rs index 5f73b8483..bb1ec42d5 100644 --- a/eth2/utils/ssz/src/impl_encode.rs +++ b/eth2/utils/ssz/src/impl_encode.rs @@ -55,13 +55,13 @@ impl Encodable for bool { impl Encodable for H256 { fn ssz_append(&self, s: &mut SszStream) { - s.append_encoded_raw(&self.to_vec()); + s.append_encoded_raw(self.as_bytes()); } } impl Encodable for Address { fn ssz_append(&self, s: &mut SszStream) { - s.append_encoded_raw(&self.to_vec()); + s.append_encoded_raw(self.as_bytes()); } } From 0be8e57fd3bf916ca8dee426fc5a5a4fdae8336c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 6 Mar 2019 10:21:55 +1100 Subject: [PATCH 112/132] Further v0.4.0 updates to `types` crate --- eth2/types/src/attester_slashing/builder.rs | 4 +-- eth2/types/src/beacon_block_body.rs | 4 +-- eth2/types/src/beacon_state.rs | 28 +++++++-------------- eth2/types/src/beacon_state/builder.rs | 2 +- eth2/types/src/beacon_state/helpers.rs | 20 +++++++++++++++ eth2/types/src/crosslink.rs | 2 +- eth2/types/src/lib.rs | 2 +- eth2/types/src/voluntary_exit.rs | 6 ++--- 8 files changed, 39 insertions(+), 29 deletions(-) diff --git a/eth2/types/src/attester_slashing/builder.rs b/eth2/types/src/attester_slashing/builder.rs index 54dfae959..6638eb2a5 100644 --- a/eth2/types/src/attester_slashing/builder.rs +++ b/eth2/types/src/attester_slashing/builder.rs @@ -40,7 +40,7 @@ impl AttesterSlashingBuilder { crosslink_data_root: hash_1, latest_crosslink: Crosslink { epoch, - shard_block_root: hash_1, + crosslink_data_root: hash_1, }, justified_epoch, justified_block_root: hash_1, @@ -59,7 +59,7 @@ impl AttesterSlashingBuilder { crosslink_data_root: hash_2, latest_crosslink: Crosslink { epoch, - shard_block_root: hash_2, + crosslink_data_root: hash_2, }, justified_epoch, justified_block_root: hash_2, diff --git a/eth2/types/src/beacon_block_body.rs b/eth2/types/src/beacon_block_body.rs index 13c82e42f..e7dec2e4b 100644 --- a/eth2/types/src/beacon_block_body.rs +++ b/eth2/types/src/beacon_block_body.rs @@ -1,4 +1,4 @@ -use super::{Attestation, AttesterSlashing, Deposit, ProposerSlashing, Transfer, VolutaryExit}; +use super::{Attestation, AttesterSlashing, Deposit, ProposerSlashing, Transfer, VoluntaryExit}; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; @@ -14,7 +14,7 @@ pub struct BeaconBlockBody { pub attester_slashings: Vec, pub attestations: Vec, pub deposits: Vec, - pub voluntary_exits: Vec, + pub voluntary_exits: Vec, pub transfers: Vec, } diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index d876adc62..2394c2c0d 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -2,6 +2,7 @@ use self::epoch_cache::EpochCache; use crate::test_utils::TestRandom; use crate::{validator_registry::get_active_validator_indices, *}; use bls::verify_proof_of_possession; +use helpers::*; use honey_badger_split::SplitExt; use int_to_bytes::int_to_bytes32; use log::{debug, error, trace}; @@ -16,6 +17,7 @@ pub use builder::BeaconStateBuilder; mod builder; mod epoch_cache; +pub mod helpers; mod tests; pub type Committee = Vec; @@ -44,6 +46,7 @@ pub enum Error { ShardOutOfBounds, UnableToShuffle, UnknownValidator, + InvalidBitfield, InsufficientRandaoMixes, InsufficientValidators, InsufficientBlockRoots, @@ -125,7 +128,7 @@ impl BeaconState { debug!("Creating genesis state (without validator processing)."); let initial_crosslink = Crosslink { epoch: spec.genesis_epoch, - shard_block_root: spec.zero_hash, + crosslink_data_root: spec.zero_hash, }; Ok(BeaconState { @@ -530,6 +533,10 @@ impl BeaconState { assert_eq!(*shard, attestation_data.shard, "Bad epoch cache build."); + if !verify_bitfield_length(&bitfield, committee.len()) { + return Err(Error::InvalidBitfield); + } + let mut participants = vec![]; for (i, validator_index) in committee.iter().enumerate() { if bitfield.get(i).unwrap() { @@ -559,23 +566,6 @@ impl BeaconState { .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)) } - /// Verify ``bitfield`` against the ``committee_size``. - /// - /// Spec v0.4.0 - pub fn verify_bitfield(&self, bitfield: &Bitfield, committee_size: usize) -> bool { - if bitfield.num_bytes() != ((committee_size + 7) / 8) { - return false; - } - - for i in committee_size..(bitfield.num_bytes() * 8) { - if bitfield.get(i).expect("Impossible due to previous check.") { - return false; - } - } - - true - } - /// Verify validity of ``slashable_attestation`` fields. /// /// Spec v0.4.0 @@ -600,7 +590,7 @@ impl BeaconState { } } - if !self.verify_bitfield( + if !verify_bitfield_length( &slashable_attestation.custody_bitfield, slashable_attestation.validator_indices.len(), ) { diff --git a/eth2/types/src/beacon_state/builder.rs b/eth2/types/src/beacon_state/builder.rs index 36b6468fc..dadce9cac 100644 --- a/eth2/types/src/beacon_state/builder.rs +++ b/eth2/types/src/beacon_state/builder.rs @@ -252,7 +252,7 @@ fn committee_to_pending_attestation( crosslink_data_root: Hash256::zero(), latest_crosslink: Crosslink { epoch: slot.epoch(spec.slots_per_epoch), - shard_block_root: Hash256::zero(), + crosslink_data_root: Hash256::zero(), }, justified_epoch, justified_block_root, diff --git a/eth2/types/src/beacon_state/helpers.rs b/eth2/types/src/beacon_state/helpers.rs index e69de29bb..c93b16f76 100644 --- a/eth2/types/src/beacon_state/helpers.rs +++ b/eth2/types/src/beacon_state/helpers.rs @@ -0,0 +1,20 @@ +use crate::*; + +/// Verify ``bitfield`` against the ``committee_size``. +/// +/// Is title `verify_bitfield` in spec. +/// +/// Spec v0.4.0 +pub fn verify_bitfield_length(bitfield: &Bitfield, committee_size: usize) -> bool { + if bitfield.num_bytes() != ((committee_size + 7) / 8) { + return false; + } + + for i in committee_size..(bitfield.num_bytes() * 8) { + if bitfield.get(i).expect("Impossible due to previous check.") { + return false; + } + } + + true +} diff --git a/eth2/types/src/crosslink.rs b/eth2/types/src/crosslink.rs index 142d0f894..f49195a75 100644 --- a/eth2/types/src/crosslink.rs +++ b/eth2/types/src/crosslink.rs @@ -13,7 +13,7 @@ use test_random_derive::TestRandom; )] pub struct Crosslink { pub epoch: Epoch, - pub shard_block_root: Hash256, + pub crosslink_data_root: Hash256, } #[cfg(test)] diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index f54a24b01..f88cc6a17 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -60,7 +60,7 @@ pub use crate::slot_epoch::{Epoch, Slot}; pub use crate::slot_height::SlotHeight; pub use crate::transfer::Transfer; pub use crate::validator::Validator; -pub use crate::voluntary_exit::VolutaryExit; +pub use crate::voluntary_exit::VoluntaryExit; pub type Hash256 = H256; pub type Address = H160; diff --git a/eth2/types/src/voluntary_exit.rs b/eth2/types/src/voluntary_exit.rs index 3cb0a85b7..58c3ae4c2 100644 --- a/eth2/types/src/voluntary_exit.rs +++ b/eth2/types/src/voluntary_exit.rs @@ -10,7 +10,7 @@ use test_random_derive::TestRandom; /// /// Spec v0.4.0 #[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)] -pub struct VolutaryExit { +pub struct VoluntaryExit { pub epoch: Epoch, pub validator_index: u64, pub signature: Signature, @@ -25,7 +25,7 @@ mod tests { #[test] pub fn test_ssz_round_trip() { let mut rng = XorShiftRng::from_seed([42; 16]); - let original = VolutaryExit::random_for_test(&mut rng); + let original = VoluntaryExit::random_for_test(&mut rng); let bytes = ssz_encode(&original); let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); @@ -36,7 +36,7 @@ mod tests { #[test] pub fn test_hash_tree_root_internal() { let mut rng = XorShiftRng::from_seed([42; 16]); - let original = VolutaryExit::random_for_test(&mut rng); + let original = VoluntaryExit::random_for_test(&mut rng); let result = original.hash_tree_root_internal(); From a15ed0acd3663fbae42e91edb22a89ad53b8323d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 6 Mar 2019 10:22:19 +1100 Subject: [PATCH 113/132] Start new Error structure in `state_processing` --- .../state_processing/src/block_processable.rs | 124 +-------- .../block_processable/validate_attestation.rs | 238 ++++++++++++++++++ eth2/state_processing/src/errors.rs | 30 +++ eth2/state_processing/src/lib.rs | 11 +- eth2/state_processing/src/macros.rs | 8 + 5 files changed, 290 insertions(+), 121 deletions(-) create mode 100644 eth2/state_processing/src/block_processable/validate_attestation.rs create mode 100644 eth2/state_processing/src/errors.rs create mode 100644 eth2/state_processing/src/macros.rs diff --git a/eth2/state_processing/src/block_processable.rs b/eth2/state_processing/src/block_processable.rs index ee8bc7c33..6f4d7268e 100644 --- a/eth2/state_processing/src/block_processable.rs +++ b/eth2/state_processing/src/block_processable.rs @@ -5,7 +5,11 @@ use int_to_bytes::int_to_bytes32; use log::{debug, trace}; use ssz::{ssz_encode, TreeHash}; use types::*; +use validate_attestation::validate_attestations; +pub use validate_attestation::{validate_attestation, validate_attestation_without_signature}; + +mod validate_attestation; mod verify_slashable_attestation; const PHASE_0_CUSTODY_BIT: bool = false; @@ -41,7 +45,6 @@ pub enum Error { BeaconStateError(BeaconStateError), SlotProcessingError(SlotProcessingError), } - #[derive(Debug, PartialEq)] pub enum AttestationValidationError { IncludedTooEarly, @@ -221,19 +224,10 @@ fn per_block_processing_signature_optional( verify_slashable_attestation(&mut state, &attester_slashing, spec)?; } - /* - * Attestations - */ - ensure!( - block.body.attestations.len() as u64 <= spec.max_attestations, - Error::MaxAttestationsExceeded - ); - - debug!("Verifying {} attestations.", block.body.attestations.len()); + validate_attestations(&mut state, &block, spec); + // Convert each attestation into a `PendingAttestation` and insert into the state. for attestation in &block.body.attestations { - validate_attestation(&state, attestation, spec)?; - let pending_attestation = PendingAttestation { data: attestation.data.clone(), aggregation_bitfield: attestation.aggregation_bitfield.clone(), @@ -293,7 +287,7 @@ fn per_block_processing_signature_optional( ); ensure!(state.current_epoch(spec) >= exit.epoch, Error::BadExit); let exit_message = { - let exit_struct = Exit { + let exit_struct = VoluntaryExit { epoch: exit.epoch, validator_index: exit.validator_index, signature: spec.empty_signature.clone(), @@ -317,110 +311,6 @@ fn per_block_processing_signature_optional( Ok(()) } -pub fn validate_attestation( - state: &BeaconState, - attestation: &Attestation, - spec: &ChainSpec, -) -> Result<(), AttestationValidationError> { - validate_attestation_signature_optional(state, attestation, spec, true) -} - -pub fn validate_attestation_without_signature( - state: &BeaconState, - attestation: &Attestation, - spec: &ChainSpec, -) -> Result<(), AttestationValidationError> { - validate_attestation_signature_optional(state, attestation, spec, false) -} - -fn validate_attestation_signature_optional( - state: &BeaconState, - attestation: &Attestation, - spec: &ChainSpec, - verify_signature: bool, -) -> Result<(), AttestationValidationError> { - trace!( - "validate_attestation_signature_optional: attestation epoch: {}", - attestation.data.slot.epoch(spec.slots_per_epoch) - ); - ensure!( - attestation.data.slot + spec.min_attestation_inclusion_delay <= state.slot, - AttestationValidationError::IncludedTooEarly - ); - ensure!( - attestation.data.slot + spec.slots_per_epoch >= state.slot, - AttestationValidationError::IncludedTooLate - ); - if attestation.data.slot >= state.current_epoch_start_slot(spec) { - ensure!( - attestation.data.justified_epoch == state.justified_epoch, - AttestationValidationError::WrongJustifiedSlot - ); - } else { - ensure!( - attestation.data.justified_epoch == state.previous_justified_epoch, - AttestationValidationError::WrongJustifiedSlot - ); - } - ensure!( - attestation.data.justified_block_root - == *state - .get_block_root( - attestation - .data - .justified_epoch - .start_slot(spec.slots_per_epoch), - &spec - ) - .ok_or(AttestationValidationError::NoBlockRoot)?, - AttestationValidationError::WrongJustifiedRoot - ); - let potential_crosslink = Crosslink { - shard_block_root: attestation.data.shard_block_root, - epoch: attestation.data.slot.epoch(spec.slots_per_epoch), - }; - ensure!( - (attestation.data.latest_crosslink - == state.latest_crosslinks[attestation.data.shard as usize]) - | (attestation.data.latest_crosslink == potential_crosslink), - AttestationValidationError::BadLatestCrosslinkRoot - ); - if verify_signature { - let participants = state.get_attestation_participants( - &attestation.data, - &attestation.aggregation_bitfield, - spec, - )?; - trace!( - "slot: {}, shard: {}, participants: {:?}", - attestation.data.slot, - attestation.data.shard, - participants - ); - let mut group_public_key = AggregatePublicKey::new(); - for participant in participants { - group_public_key.add(&state.validator_registry[participant as usize].pubkey) - } - ensure!( - attestation.verify_signature( - &group_public_key, - PHASE_0_CUSTODY_BIT, - get_domain( - &state.fork, - attestation.data.slot.epoch(spec.slots_per_epoch), - spec.domain_attestation, - ) - ), - AttestationValidationError::BadSignature - ); - } - ensure!( - attestation.data.shard_block_root == spec.zero_hash, - AttestationValidationError::ShardBlockRootNotZero - ); - Ok(()) -} - fn get_domain(fork: &Fork, epoch: Epoch, domain_type: u64) -> u64 { fork.get_domain(epoch, domain_type) } diff --git a/eth2/state_processing/src/block_processable/validate_attestation.rs b/eth2/state_processing/src/block_processable/validate_attestation.rs new file mode 100644 index 000000000..1b422711c --- /dev/null +++ b/eth2/state_processing/src/block_processable/validate_attestation.rs @@ -0,0 +1,238 @@ +use crate::errors::{AttestationInvalid as Invalid, AttestationValidationError as Error}; +use ssz::TreeHash; +use types::beacon_state::helpers::*; +use types::*; + +/// Validate the attestations in some block, converting each into a `PendingAttestation` which is +/// then added to `state.latest_attestations`. +/// +/// Spec v0.4.0 +pub fn validate_attestations( + state: &BeaconState, + block: &BeaconBlock, + spec: &ChainSpec, +) -> Result<(), Error> { + ensure!( + block.body.attestations.len() as u64 <= spec.max_attestations, + MaxAttestationsExceeded + ); + + for attestation in &block.body.attestations { + validate_attestation(&state, attestation, spec)?; + } + + Ok(()) +} + +/// Validate an attestation, checking the aggregate signature. +/// +/// Spec v0.4.0 +pub fn validate_attestation( + state: &BeaconState, + attestation: &Attestation, + spec: &ChainSpec, +) -> Result<(), Error> { + validate_attestation_signature_optional(state, attestation, spec, true) +} + +/// Validate an attestation, without checking the aggregate signature. +/// +/// Spec v0.4.0 +pub fn validate_attestation_without_signature( + state: &BeaconState, + attestation: &Attestation, + spec: &ChainSpec, +) -> Result<(), Error> { + validate_attestation_signature_optional(state, attestation, spec, false) +} + +/// Validate an attestation, optionally checking the aggregate signature. +/// +/// Spec v0.2.0 +fn validate_attestation_signature_optional( + state: &BeaconState, + attestation: &Attestation, + spec: &ChainSpec, + verify_signature: bool, +) -> Result<(), Error> { + // Verify that `attestation.data.slot >= GENESIS_SLOT`. + ensure!(attestation.data.slot >= spec.genesis_slot, PreGenesis); + + // Verify that `attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot`. + ensure!( + attestation.data.slot + spec.min_attestation_inclusion_delay <= state.slot, + IncludedTooEarly + ); + + // Verify that `state.slot < attestation.data.slot + SLOTS_PER_EPOCH`. + ensure!( + state.slot < attestation.data.slot + spec.slots_per_epoch, + IncludedTooLate + ); + + // Verify that `attestation.data.justified_epoch` is equal to `state.justified_epoch` if + // `slot_to_epoch(attestation.data.slot + 1) >= get_current_epoch(state) else + // state.previous_justified_epoch`. + if (attestation.data.slot + 1).epoch(spec.slots_per_epoch) >= state.current_epoch(spec) { + ensure!( + attestation.data.justified_epoch == state.justified_epoch, + WrongJustifiedSlot + ); + } else { + ensure!( + attestation.data.justified_epoch == state.previous_justified_epoch, + WrongJustifiedSlot + ); + } + + // Verify that `attestation.data.justified_block_root` is equal to `get_block_root(state, + // get_epoch_start_slot(attestation.data.justified_epoch))`. + ensure!( + attestation.data.justified_block_root + == *state + .get_block_root( + attestation + .data + .justified_epoch + .start_slot(spec.slots_per_epoch), + &spec + ) + .ok_or(BeaconStateError::InsufficientBlockRoots)?, + WrongJustifiedRoot + ); + + // Verify that either: + // + // (i)`state.latest_crosslinks[attestation.data.shard] == attestation.data.latest_crosslink`, + // + // (ii) `state.latest_crosslinks[attestation.data.shard] == + // Crosslink(crosslink_data_root=attestation.data.crosslink_data_root, + // epoch=slot_to_epoch(attestation.data.slot))`. + let potential_crosslink = Crosslink { + crosslink_data_root: attestation.data.crosslink_data_root, + epoch: attestation.data.slot.epoch(spec.slots_per_epoch), + }; + ensure!( + (attestation.data.latest_crosslink + == state.latest_crosslinks[attestation.data.shard as usize]) + | (state.latest_crosslinks[attestation.data.shard as usize] == potential_crosslink), + BadLatestCrosslinkRoot + ); + + // Get the committee for this attestation + let (committee, _shard) = state + .get_crosslink_committees_at_slot(attestation.data.slot, spec)? + .iter() + .find(|(committee, shard)| *shard == attestation.data.shard) + .ok_or_else(|| Error::Invalid(Invalid::NoCommitteeForShard))?; + + // Custody bitfield is all zeros (phase 0 requirement). + ensure!( + attestation.custody_bitfield.num_set_bits() == 0, + CustodyBitfieldHasSetBits + ); + // Custody bitfield length is correct. + ensure!( + verify_bitfield_length(&attestation.aggregation_bitfield, committee.len()), + BadCustodyBitfieldLength + ); + // Aggregation bitfield isn't empty. + ensure!( + attestation.aggregation_bitfield.num_set_bits() != 0, + AggregationBitfieldIsEmpty + ); + // Aggregation bitfield length is correct. + ensure!( + verify_bitfield_length(&attestation.aggregation_bitfield, committee.len()), + BadAggregationBitfieldLength + ); + + if verify_signature { + ensure!( + verify_attestation_signature( + state, + committee, + &attestation.custody_bitfield, + &attestation.data, + &attestation.aggregate_signature, + spec + ), + BadSignature + ); + } + + // [TO BE REMOVED IN PHASE 1] Verify that `attestation.data.crosslink_data_root == ZERO_HASH`. + ensure!( + attestation.data.crosslink_data_root == spec.zero_hash, + ShardBlockRootNotZero + ); + + Ok(()) +} + +/// Verifies an aggregate signature for some given `AttestationData`, returning `true` if the +/// `aggregate_signature` is valid. +/// +/// Returns `false` if: +/// - `aggregate_signature` was not signed correctly. +/// - `custody_bitfield` does not have a bit for each index of `committee`. +/// - A `validator_index` in `committee` is not in `state.validator_registry`. +fn verify_attestation_signature( + state: &BeaconState, + committee: &[usize], + custody_bitfield: &Bitfield, + attestation_data: &AttestationData, + aggregate_signature: &AggregateSignature, + spec: &ChainSpec, +) -> bool { + let mut aggregate_pubs = vec![AggregatePublicKey::new(); 2]; + let mut message_exists = vec![false; 2]; + + for (i, v) in committee.iter().enumerate() { + let custody_bit = match custody_bitfield.get(i) { + Ok(bit) => bit, + // Invalidate signature if custody_bitfield.len() < committee + Err(_) => return false, + }; + + message_exists[custody_bit as usize] = true; + + match state.validator_registry.get(*v as usize) { + Some(validator) => { + aggregate_pubs[custody_bit as usize].add(&validator.pubkey); + } + // Invalidate signature if validator index is unknown. + None => return false, + }; + } + + // Message when custody bitfield is `false` + let message_0 = AttestationDataAndCustodyBit { + data: attestation_data.clone(), + custody_bit: false, + } + .hash_tree_root(); + + // Message when custody bitfield is `true` + let message_1 = AttestationDataAndCustodyBit { + data: attestation_data.clone(), + custody_bit: true, + } + .hash_tree_root(); + + let mut messages = vec![]; + let mut keys = vec![]; + + // If any validator signed a message with a `false` custody bit. + if message_exists[0] { + messages.push(&message_0[..]); + keys.push(&aggregate_pubs[0]); + } + // If any validator signed a message with a `true` custody bit. + if message_exists[1] { + messages.push(&message_1[..]); + keys.push(&aggregate_pubs[1]); + } + + aggregate_signature.verify_multiple(&messages[..], spec.domain_attestation, &keys[..]) +} diff --git a/eth2/state_processing/src/errors.rs b/eth2/state_processing/src/errors.rs new file mode 100644 index 000000000..5eceb0be0 --- /dev/null +++ b/eth2/state_processing/src/errors.rs @@ -0,0 +1,30 @@ +use types::BeaconStateError; + +#[derive(Debug, PartialEq)] +pub enum AttestationValidationError { + Invalid(AttestationInvalid), + ProcessingError(BeaconStateError), +} + +#[derive(Debug, PartialEq)] +pub enum AttestationInvalid { + PreGenesis, + IncludedTooEarly, + IncludedTooLate, + WrongJustifiedSlot, + WrongJustifiedRoot, + BadLatestCrosslinkRoot, + CustodyBitfieldHasSetBits, + AggregationBitfieldIsEmpty, + BadAggregationBitfieldLength, + BadCustodyBitfieldLength, + NoCommitteeForShard, + BadSignature, + ShardBlockRootNotZero, +} + +impl From for AttestationValidationError { + fn from(e: BeaconStateError) -> AttestationValidationError { + AttestationValidationError::ProcessingError(e) + } +} diff --git a/eth2/state_processing/src/lib.rs b/eth2/state_processing/src/lib.rs index 18d1f7554..0a4343e00 100644 --- a/eth2/state_processing/src/lib.rs +++ b/eth2/state_processing/src/lib.rs @@ -1,10 +1,13 @@ +#[macro_use] +mod macros; mod block_processable; -mod epoch_processable; -mod slot_processable; +// mod epoch_processable; +mod errors; +// mod slot_processable; pub use block_processable::{ validate_attestation, validate_attestation_without_signature, BlockProcessable, Error as BlockProcessingError, }; -pub use epoch_processable::{EpochProcessable, Error as EpochProcessingError}; -pub use slot_processable::{Error as SlotProcessingError, SlotProcessable}; +// pub use epoch_processable::{EpochProcessable, Error as EpochProcessingError}; +// pub use slot_processable::{Error as SlotProcessingError, SlotProcessable}; diff --git a/eth2/state_processing/src/macros.rs b/eth2/state_processing/src/macros.rs new file mode 100644 index 000000000..6ccc54ba9 --- /dev/null +++ b/eth2/state_processing/src/macros.rs @@ -0,0 +1,8 @@ +#[macro_use] +macro_rules! ensure { + ($condition: expr, $result: ident) => { + if !$condition { + return Err(Error::Invalid(Invalid::$result)); + } + }; +} From 40f74c9b2622fb1e23af7325e67d6218dae22f0c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 6 Mar 2019 15:22:45 +1100 Subject: [PATCH 114/132] First compiling version of per-block-proc refactor --- eth2/state_processing/Cargo.toml | 1 - .../state_processing/src/block_processable.rs | 344 ------------------ .../verify_slashable_attestation.rs | 61 ---- eth2/state_processing/src/errors.rs | 221 ++++++++++- eth2/state_processing/src/lib.rs | 10 +- eth2/state_processing/src/macros.rs | 13 +- .../src/per_block_processing.rs | 268 ++++++++++++++ .../validate_attestation.rs | 87 ++--- .../verify_attester_slashing.rs | 49 +++ .../per_block_processing/verify_deposit.rs | 24 ++ .../src/per_block_processing/verify_exit.rs | 41 +++ .../verify_proposer_slashing.rs | 71 ++++ .../verify_slashable_attestation.rs | 105 ++++++ .../per_block_processing/verify_transfer.rs | 19 + eth2/types/src/attester_slashing/builder.rs | 11 +- eth2/types/src/beacon_state.rs | 102 +----- eth2/types/src/chain_spec.rs | 46 ++- eth2/types/src/fork.rs | 8 - eth2/types/src/lib.rs | 2 +- eth2/types/src/proposer_slashing/builder.rs | 10 +- 20 files changed, 907 insertions(+), 586 deletions(-) delete mode 100644 eth2/state_processing/src/block_processable.rs delete mode 100644 eth2/state_processing/src/block_processable/verify_slashable_attestation.rs create mode 100644 eth2/state_processing/src/per_block_processing.rs rename eth2/state_processing/src/{block_processable => per_block_processing}/validate_attestation.rs (83%) create mode 100644 eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs create mode 100644 eth2/state_processing/src/per_block_processing/verify_deposit.rs create mode 100644 eth2/state_processing/src/per_block_processing/verify_exit.rs create mode 100644 eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs create mode 100644 eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs create mode 100644 eth2/state_processing/src/per_block_processing/verify_transfer.rs diff --git a/eth2/state_processing/Cargo.toml b/eth2/state_processing/Cargo.toml index 85ac8b42a..bdc14c1d7 100644 --- a/eth2/state_processing/Cargo.toml +++ b/eth2/state_processing/Cargo.toml @@ -10,7 +10,6 @@ harness = false [dev-dependencies] criterion = "0.2" -test_harness = { path = "../../beacon_node/beacon_chain/test_harness" } env_logger = "0.6.0" [dependencies] diff --git a/eth2/state_processing/src/block_processable.rs b/eth2/state_processing/src/block_processable.rs deleted file mode 100644 index 6f4d7268e..000000000 --- a/eth2/state_processing/src/block_processable.rs +++ /dev/null @@ -1,344 +0,0 @@ -use self::verify_slashable_attestation::verify_slashable_attestation; -use crate::SlotProcessingError; -use hashing::hash; -use int_to_bytes::int_to_bytes32; -use log::{debug, trace}; -use ssz::{ssz_encode, TreeHash}; -use types::*; -use validate_attestation::validate_attestations; - -pub use validate_attestation::{validate_attestation, validate_attestation_without_signature}; - -mod validate_attestation; -mod verify_slashable_attestation; - -const PHASE_0_CUSTODY_BIT: bool = false; - -#[derive(Debug, PartialEq)] -pub enum Error { - DBError(String), - StateAlreadyTransitioned, - PresentSlotIsNone, - UnableToDecodeBlock, - MissingParentState(Hash256), - InvalidParentState(Hash256), - MissingBeaconBlock(Hash256), - InvalidBeaconBlock(Hash256), - MissingParentBlock(Hash256), - StateSlotMismatch, - BadBlockSignature, - BadRandaoSignature, - MaxProposerSlashingsExceeded, - BadProposerSlashing, - MaxAttesterSlashingsExceed, - MaxAttestationsExceeded, - BadAttesterSlashing, - InvalidAttestation(AttestationValidationError), - NoBlockRoot, - MaxDepositsExceeded, - BadDeposit, - MaxExitsExceeded, - BadExit, - BadCustodyReseeds, - BadCustodyChallenges, - BadCustodyResponses, - BeaconStateError(BeaconStateError), - SlotProcessingError(SlotProcessingError), -} -#[derive(Debug, PartialEq)] -pub enum AttestationValidationError { - IncludedTooEarly, - IncludedTooLate, - WrongJustifiedSlot, - WrongJustifiedRoot, - BadLatestCrosslinkRoot, - BadSignature, - ShardBlockRootNotZero, - NoBlockRoot, - BeaconStateError(BeaconStateError), -} - -macro_rules! ensure { - ($condition: expr, $result: expr) => { - if !$condition { - return Err($result); - } - }; -} - -pub trait BlockProcessable { - fn per_block_processing(&mut self, block: &BeaconBlock, spec: &ChainSpec) -> Result<(), Error>; - fn per_block_processing_without_verifying_block_signature( - &mut self, - block: &BeaconBlock, - spec: &ChainSpec, - ) -> Result<(), Error>; -} - -impl BlockProcessable for BeaconState { - fn per_block_processing(&mut self, block: &BeaconBlock, spec: &ChainSpec) -> Result<(), Error> { - per_block_processing_signature_optional(self, block, true, spec) - } - - fn per_block_processing_without_verifying_block_signature( - &mut self, - block: &BeaconBlock, - spec: &ChainSpec, - ) -> Result<(), Error> { - per_block_processing_signature_optional(self, block, false, spec) - } -} - -fn per_block_processing_signature_optional( - mut state: &mut BeaconState, - block: &BeaconBlock, - verify_block_signature: bool, - spec: &ChainSpec, -) -> Result<(), Error> { - ensure!(block.slot == state.slot, Error::StateSlotMismatch); - - // Building the previous epoch could be delayed until an attestation from a previous epoch is - // included. This is left for future optimisation. - state.build_epoch_cache(RelativeEpoch::Previous, spec)?; - state.build_epoch_cache(RelativeEpoch::Current, spec)?; - - /* - * Proposer Signature - */ - let block_proposer_index = state.get_beacon_proposer_index(block.slot, spec)?; - let block_proposer = &state.validator_registry[block_proposer_index]; - - if verify_block_signature { - ensure!( - bls_verify( - &block_proposer.pubkey, - &block.proposal_root(spec)[..], - &block.signature, - get_domain(&state.fork, state.current_epoch(spec), spec.domain_proposal) - ), - Error::BadBlockSignature - ); - } - - /* - * RANDAO - */ - ensure!( - bls_verify( - &block_proposer.pubkey, - &int_to_bytes32(state.current_epoch(spec).as_u64()), - &block.randao_reveal, - get_domain(&state.fork, state.current_epoch(spec), spec.domain_randao) - ), - Error::BadRandaoSignature - ); - - // TODO: check this is correct. - let new_mix = { - let mut mix = state.latest_randao_mixes - [state.slot.as_usize() % spec.latest_randao_mixes_length] - .to_vec(); - mix.append(&mut ssz_encode(&block.randao_reveal)); - Hash256::from(&hash(&mix)[..]) - }; - - state.latest_randao_mixes[state.slot.as_usize() % spec.latest_randao_mixes_length] = new_mix; - - /* - * Eth1 data - */ - // TODO: Eth1 data processing. - - /* - * Proposer slashings - */ - ensure!( - block.body.proposer_slashings.len() as u64 <= spec.max_proposer_slashings, - Error::MaxProposerSlashingsExceeded - ); - for proposer_slashing in &block.body.proposer_slashings { - let proposer = state - .validator_registry - .get(proposer_slashing.proposer_index as usize) - .ok_or(Error::BadProposerSlashing)?; - ensure!( - proposer_slashing.proposal_data_1.slot == proposer_slashing.proposal_data_2.slot, - Error::BadProposerSlashing - ); - ensure!( - proposer_slashing.proposal_data_1.shard == proposer_slashing.proposal_data_2.shard, - Error::BadProposerSlashing - ); - ensure!( - proposer_slashing.proposal_data_1.block_root - != proposer_slashing.proposal_data_2.block_root, - Error::BadProposerSlashing - ); - ensure!( - proposer.penalized_epoch > state.current_epoch(spec), - Error::BadProposerSlashing - ); - ensure!( - bls_verify( - &proposer.pubkey, - &proposer_slashing.proposal_data_1.hash_tree_root(), - &proposer_slashing.proposal_signature_1, - get_domain( - &state.fork, - proposer_slashing - .proposal_data_1 - .slot - .epoch(spec.slots_per_epoch), - spec.domain_proposal - ) - ), - Error::BadProposerSlashing - ); - ensure!( - bls_verify( - &proposer.pubkey, - &proposer_slashing.proposal_data_2.hash_tree_root(), - &proposer_slashing.proposal_signature_2, - get_domain( - &state.fork, - proposer_slashing - .proposal_data_2 - .slot - .epoch(spec.slots_per_epoch), - spec.domain_proposal - ) - ), - Error::BadProposerSlashing - ); - state.penalize_validator(proposer_slashing.proposer_index as usize, spec)?; - } - - /* - * Attester slashings - */ - ensure!( - block.body.attester_slashings.len() as u64 <= spec.max_attester_slashings, - Error::MaxAttesterSlashingsExceed - ); - for attester_slashing in &block.body.attester_slashings { - verify_slashable_attestation(&mut state, &attester_slashing, spec)?; - } - - validate_attestations(&mut state, &block, spec); - - // Convert each attestation into a `PendingAttestation` and insert into the state. - for attestation in &block.body.attestations { - let pending_attestation = PendingAttestation { - data: attestation.data.clone(), - aggregation_bitfield: attestation.aggregation_bitfield.clone(), - custody_bitfield: attestation.custody_bitfield.clone(), - inclusion_slot: state.slot, - }; - state.latest_attestations.push(pending_attestation); - } - - /* - * Deposits - */ - ensure!( - block.body.deposits.len() as u64 <= spec.max_deposits, - Error::MaxDepositsExceeded - ); - - // TODO: verify deposit merkle branches. - for deposit in &block.body.deposits { - debug!( - "Processing deposit for pubkey {:?}", - deposit.deposit_data.deposit_input.pubkey - ); - state - .process_deposit( - deposit.deposit_data.deposit_input.pubkey.clone(), - deposit.deposit_data.amount, - deposit - .deposit_data - .deposit_input - .proof_of_possession - .clone(), - deposit.deposit_data.deposit_input.withdrawal_credentials, - None, - spec, - ) - .map_err(|_| Error::BadDeposit)?; - } - - /* - * Exits - */ - ensure!( - block.body.exits.len() as u64 <= spec.max_exits, - Error::MaxExitsExceeded - ); - - for exit in &block.body.exits { - let validator = state - .validator_registry - .get(exit.validator_index as usize) - .ok_or(Error::BadExit)?; - ensure!( - validator.exit_epoch - > state.get_entry_exit_effect_epoch(state.current_epoch(spec), spec), - Error::BadExit - ); - ensure!(state.current_epoch(spec) >= exit.epoch, Error::BadExit); - let exit_message = { - let exit_struct = VoluntaryExit { - epoch: exit.epoch, - validator_index: exit.validator_index, - signature: spec.empty_signature.clone(), - }; - exit_struct.hash_tree_root() - }; - ensure!( - bls_verify( - &validator.pubkey, - &exit_message, - &exit.signature, - get_domain(&state.fork, exit.epoch, spec.domain_exit) - ), - Error::BadProposerSlashing - ); - state.initiate_validator_exit(exit.validator_index as usize); - } - - debug!("State transition complete."); - - Ok(()) -} - -fn get_domain(fork: &Fork, epoch: Epoch, domain_type: u64) -> u64 { - fork.get_domain(epoch, domain_type) -} - -fn bls_verify(pubkey: &PublicKey, message: &[u8], signature: &Signature, domain: u64) -> bool { - signature.verify(message, domain, pubkey) -} - -impl From for Error { - fn from(e: AttestationValidationError) -> Error { - Error::InvalidAttestation(e) - } -} - -impl From for Error { - fn from(e: BeaconStateError) -> Error { - Error::BeaconStateError(e) - } -} - -impl From for Error { - fn from(e: SlotProcessingError) -> Error { - Error::SlotProcessingError(e) - } -} - -impl From for AttestationValidationError { - fn from(e: BeaconStateError) -> AttestationValidationError { - AttestationValidationError::BeaconStateError(e) - } -} diff --git a/eth2/state_processing/src/block_processable/verify_slashable_attestation.rs b/eth2/state_processing/src/block_processable/verify_slashable_attestation.rs deleted file mode 100644 index a406af24e..000000000 --- a/eth2/state_processing/src/block_processable/verify_slashable_attestation.rs +++ /dev/null @@ -1,61 +0,0 @@ -use super::Error; -use types::*; - -macro_rules! ensure { - ($condition: expr, $result: expr) => { - if !$condition { - return Err($result); - } - }; -} - -/// Returns `Ok(())` if some `AttesterSlashing` is valid to be included in some `BeaconState`, -/// otherwise returns an `Err`. -pub fn verify_slashable_attestation( - state: &mut BeaconState, - attester_slashing: &AttesterSlashing, - spec: &ChainSpec, -) -> Result<(), Error> { - let slashable_attestation_1 = &attester_slashing.slashable_attestation_1; - let slashable_attestation_2 = &attester_slashing.slashable_attestation_2; - - ensure!( - slashable_attestation_1.data != slashable_attestation_2.data, - Error::BadAttesterSlashing - ); - ensure!( - slashable_attestation_1.is_double_vote(slashable_attestation_2, spec) - | slashable_attestation_1.is_surround_vote(slashable_attestation_2, spec), - Error::BadAttesterSlashing - ); - ensure!( - state.verify_slashable_attestation(&slashable_attestation_1, spec), - Error::BadAttesterSlashing - ); - ensure!( - state.verify_slashable_attestation(&slashable_attestation_2, spec), - Error::BadAttesterSlashing - ); - - let mut slashable_indices = vec![]; - for i in &slashable_attestation_1.validator_indices { - let validator = state - .validator_registry - .get(*i as usize) - .ok_or_else(|| Error::BadAttesterSlashing)?; - - if slashable_attestation_1.validator_indices.contains(&i) - & !validator.is_penalized_at(state.current_epoch(spec)) - { - slashable_indices.push(i); - } - } - - ensure!(!slashable_indices.is_empty(), Error::BadAttesterSlashing); - - for i in slashable_indices { - state.penalize_validator(*i as usize, spec)?; - } - - Ok(()) -} diff --git a/eth2/state_processing/src/errors.rs b/eth2/state_processing/src/errors.rs index 5eceb0be0..89cd82b24 100644 --- a/eth2/state_processing/src/errors.rs +++ b/eth2/state_processing/src/errors.rs @@ -1,9 +1,98 @@ use types::BeaconStateError; +macro_rules! impl_from_beacon_state_error { + ($type: ident) => { + impl From for $type { + fn from(e: BeaconStateError) -> $type { + $type::BeaconStateError(e) + } + } + }; +} + +macro_rules! impl_into_with_index_with_beacon_error { + ($error_type: ident, $invalid_type: ident) => { + impl IntoWithIndex for $error_type { + fn into_with_index(self, i: usize) -> BlockProcessingError { + match self { + $error_type::Invalid(e) => { + BlockProcessingError::Invalid(BlockInvalid::$invalid_type(i, e)) + } + $error_type::BeaconStateError(e) => BlockProcessingError::BeaconStateError(e), + } + } + } + }; +} + +macro_rules! impl_into_with_index_without_beacon_error { + ($error_type: ident, $invalid_type: ident) => { + impl IntoWithIndex for $error_type { + fn into_with_index(self, i: usize) -> BlockProcessingError { + match self { + $error_type::Invalid(e) => { + BlockProcessingError::Invalid(BlockInvalid::$invalid_type(i, e)) + } + } + } + } + }; +} + +pub trait IntoWithIndex: Sized { + fn into_with_index(self, i: usize) -> T; +} + +/* + * Block Validation + */ + +#[derive(Debug, PartialEq)] +pub enum BlockProcessingError { + /// The `BeaconBlock` is invalid. + Invalid(BlockInvalid), + BeaconStateError(BeaconStateError), +} + +impl_from_beacon_state_error!(BlockProcessingError); + +#[derive(Debug, PartialEq)] +pub enum BlockInvalid { + StateSlotMismatch, + BadSignature, + BadRandaoSignature, + MaxAttestationsExceeded, + MaxAttesterSlashingsExceed, + MaxProposerSlashingsExceeded, + MaxDepositsExceeded, + MaxExitsExceeded, + MaxTransfersExceed, + AttestationInvalid(usize, AttestationInvalid), + AttesterSlashingInvalid(usize, AttesterSlashingInvalid), + ProposerSlashingInvalid(usize, ProposerSlashingInvalid), + DepositInvalid(usize, DepositInvalid), + // TODO: merge this into the `DepositInvalid` error. + DepositProcessingFailed(usize), + ExitInvalid(usize, ExitInvalid), + TransferInvalid(usize, TransferInvalid), +} + +impl Into for BlockInvalid { + fn into(self) -> BlockProcessingError { + BlockProcessingError::Invalid(self) + } +} + +/* + * Attestation Validation + */ + #[derive(Debug, PartialEq)] pub enum AttestationValidationError { + /// The `Attestation` is invalid. Invalid(AttestationInvalid), - ProcessingError(BeaconStateError), + /// Encountered a `BeaconStateError` whilst attempting to determine validity. + BeaconStateError(BeaconStateError), } #[derive(Debug, PartialEq)] @@ -23,8 +112,132 @@ pub enum AttestationInvalid { ShardBlockRootNotZero, } -impl From for AttestationValidationError { - fn from(e: BeaconStateError) -> AttestationValidationError { - AttestationValidationError::ProcessingError(e) +impl_from_beacon_state_error!(AttestationValidationError); +impl_into_with_index_with_beacon_error!(AttestationValidationError, AttestationInvalid); + +/* + * `AttesterSlashing` Validation + */ + +#[derive(Debug, PartialEq)] +pub enum AttesterSlashingValidationError { + /// The `SlashableAttestation` is invalid. + Invalid(AttesterSlashingInvalid), + /// Encountered a `BeaconStateError` whilst attempting to determine validity. + BeaconStateError(BeaconStateError), +} + +#[derive(Debug, PartialEq)] +pub enum AttesterSlashingInvalid { + AttestationDataIdentical, + NotSlashable, + SlashableAttestation1Invalid(SlashableAttestationInvalid), + SlashableAttestation2Invalid(SlashableAttestationInvalid), + UnknownValidator, + NoSlashableIndices, +} + +impl_from_beacon_state_error!(AttesterSlashingValidationError); +impl_into_with_index_with_beacon_error!(AttesterSlashingValidationError, AttesterSlashingInvalid); + +/* + * `SlashableAttestation` Validation + */ + +#[derive(Debug, PartialEq)] +pub enum SlashableAttestationValidationError { + Invalid(SlashableAttestationInvalid), +} + +#[derive(Debug, PartialEq)] +pub enum SlashableAttestationInvalid { + CustodyBitfieldHasSetBits, + NoValidatorIndices, + BadValidatorIndicesOrdering, + BadCustodyBitfieldLength, + MaxIndicesExceed, + UnknownValidator, + BadSignature, +} + +impl Into for SlashableAttestationValidationError { + fn into(self) -> SlashableAttestationInvalid { + match self { + SlashableAttestationValidationError::Invalid(e) => e, + } } } + +/* + * `ProposerSlashing` Validation + */ + +#[derive(Debug, PartialEq)] +pub enum ProposerSlashingValidationError { + Invalid(ProposerSlashingInvalid), +} + +#[derive(Debug, PartialEq)] +pub enum ProposerSlashingInvalid { + ProposerUnknown, + ProposalSlotMismatch, + ProposalShardMismatch, + ProposalBlockRootMismatch, + ProposerAlreadySlashed, + BadProposal1Signature, + BadProposal2Signature, +} + +impl_into_with_index_without_beacon_error!( + ProposerSlashingValidationError, + ProposerSlashingInvalid +); + +/* + * `Deposit` Validation + */ + +#[derive(Debug, PartialEq)] +pub enum DepositValidationError { + Invalid(DepositInvalid), +} + +#[derive(Debug, PartialEq)] +pub enum DepositInvalid { + BadIndex, +} + +impl_into_with_index_without_beacon_error!(DepositValidationError, DepositInvalid); + +/* + * `Exit` Validation + */ + +#[derive(Debug, PartialEq)] +pub enum ExitValidationError { + Invalid(ExitInvalid), +} + +#[derive(Debug, PartialEq)] +pub enum ExitInvalid { + ValidatorUnknown, + AlreadyExited, + FutureEpoch, + BadSignature, +} + +impl_into_with_index_without_beacon_error!(ExitValidationError, ExitInvalid); + +/* + * `Transfer` Validation + */ + +#[derive(Debug, PartialEq)] +pub enum TransferValidationError { + Invalid(TransferInvalid), +} + +#[derive(Debug, PartialEq)] +pub enum TransferInvalid {} + +impl_into_with_index_without_beacon_error!(TransferValidationError, TransferInvalid); diff --git a/eth2/state_processing/src/lib.rs b/eth2/state_processing/src/lib.rs index 0a4343e00..ec883ddca 100644 --- a/eth2/state_processing/src/lib.rs +++ b/eth2/state_processing/src/lib.rs @@ -1,13 +1,13 @@ #[macro_use] mod macros; -mod block_processable; +pub mod per_block_processing; // mod epoch_processable; -mod errors; +pub mod errors; // mod slot_processable; -pub use block_processable::{ - validate_attestation, validate_attestation_without_signature, BlockProcessable, - Error as BlockProcessingError, +pub use errors::{BlockInvalid, BlockProcessingError}; +pub use per_block_processing::{ + per_block_processing, per_block_processing_without_verifying_block_signature, }; // pub use epoch_processable::{EpochProcessable, Error as EpochProcessingError}; // pub use slot_processable::{Error as SlotProcessingError, SlotProcessable}; diff --git a/eth2/state_processing/src/macros.rs b/eth2/state_processing/src/macros.rs index 6ccc54ba9..5a0d0dc11 100644 --- a/eth2/state_processing/src/macros.rs +++ b/eth2/state_processing/src/macros.rs @@ -1,8 +1,13 @@ -#[macro_use] -macro_rules! ensure { - ($condition: expr, $result: ident) => { +macro_rules! verify { + ($condition: expr, $result: expr) => { if !$condition { - return Err(Error::Invalid(Invalid::$result)); + return Err(Error::Invalid($result)); } }; } + +macro_rules! invalid { + ($result: expr) => { + return Err(Error::Invalid($result)); + }; +} diff --git a/eth2/state_processing/src/per_block_processing.rs b/eth2/state_processing/src/per_block_processing.rs new file mode 100644 index 000000000..06fe3b556 --- /dev/null +++ b/eth2/state_processing/src/per_block_processing.rs @@ -0,0 +1,268 @@ +use self::verify_proposer_slashing::verify_proposer_slashing; +use crate::errors::{BlockInvalid as Invalid, BlockProcessingError as Error, IntoWithIndex}; +use hashing::hash; +use log::debug; +use ssz::{ssz_encode, SignedRoot, TreeHash}; +use types::*; + +pub use self::verify_attester_slashing::verify_attester_slashing; +pub use validate_attestation::{validate_attestation, validate_attestation_without_signature}; +pub use verify_deposit::verify_deposit; +pub use verify_exit::verify_exit; +pub use verify_transfer::verify_transfer; + +mod validate_attestation; +mod verify_attester_slashing; +mod verify_deposit; +mod verify_exit; +mod verify_proposer_slashing; +mod verify_slashable_attestation; +mod verify_transfer; + +pub fn per_block_processing( + state: &mut BeaconState, + block: &BeaconBlock, + spec: &ChainSpec, +) -> Result<(), Error> { + per_block_processing_signature_optional(state, block, true, spec) +} + +pub fn per_block_processing_without_verifying_block_signature( + state: &mut BeaconState, + block: &BeaconBlock, + spec: &ChainSpec, +) -> Result<(), Error> { + per_block_processing_signature_optional(state, block, false, spec) +} + +fn per_block_processing_signature_optional( + mut state: &mut BeaconState, + block: &BeaconBlock, + should_verify_block_signature: bool, + spec: &ChainSpec, +) -> Result<(), Error> { + // Verify that `block.slot == state.slot`. + verify!(block.slot == state.slot, Invalid::StateSlotMismatch); + + // Get the epoch for future ergonomics. + let epoch = block.slot.epoch(spec.slots_per_epoch); + + // Ensure the current epoch cache is built. + state.build_epoch_cache(RelativeEpoch::Current, spec)?; + + // Let `proposer = state.validator_registry[get_beacon_proposer_index(state, state.slot)]`. + let block_proposer = + &state.validator_registry[state.get_beacon_proposer_index(block.slot, spec)?]; + + // Block signature + if should_verify_block_signature { + verify!( + verify_block_signature(&block, &block_proposer, &state.fork, spec,), + Invalid::BadSignature + ); + } + + // Randao + + // Verify that `bls_verify(pubkey=proposer.pubkey, + // message_hash=hash_tree_root(get_current_epoch(state)), signature=block.randao_reveal, + // domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_RANDAO))`. + verify!( + block.randao_reveal.verify( + &state.current_epoch(spec).hash_tree_root()[..], + spec.get_domain(epoch, Domain::Randao, &state.fork), + &block_proposer.pubkey + ), + Invalid::BadRandaoSignature + ); + + // Update the state's RANDAO mix with the one revealed in the block. + update_randao(&mut state, &block.randao_reveal, spec)?; + + // Eth1 Data + + // Either increment the eth1_data vote count, or add a new eth1_data. + let matching_eth1_vote_index = state + .eth1_data_votes + .iter() + .position(|vote| vote.eth1_data == block.eth1_data); + if let Some(index) = matching_eth1_vote_index { + state.eth1_data_votes[index].vote_count += 1; + } else { + state.eth1_data_votes.push(Eth1DataVote { + eth1_data: block.eth1_data.clone(), + vote_count: 1, + }); + } + + //Proposer slashings + + verify!( + block.body.proposer_slashings.len() as u64 <= spec.max_proposer_slashings, + Invalid::MaxProposerSlashingsExceeded + ); + for (i, proposer_slashing) in block.body.proposer_slashings.iter().enumerate() { + verify_proposer_slashing(proposer_slashing, &state, spec) + .map_err(|e| e.into_with_index(i))?; + state.slash_validator(proposer_slashing.proposer_index as usize, spec)?; + } + + // Attester Slashings + + verify!( + block.body.attester_slashings.len() as u64 <= spec.max_attester_slashings, + Invalid::MaxAttesterSlashingsExceed + ); + for (i, attester_slashing) in block.body.attester_slashings.iter().enumerate() { + let slashable_indices = verify_attester_slashing(&state, &attester_slashing, spec) + .map_err(|e| e.into_with_index(i))?; + for i in slashable_indices { + state.slash_validator(i as usize, spec)?; + } + } + + // Attestations + + verify!( + block.body.attestations.len() as u64 <= spec.max_attestations, + Invalid::MaxAttestationsExceeded + ); + for (i, attestation) in block.body.attestations.iter().enumerate() { + // Build the previous epoch cache only if required by an attestation. + if attestation.data.slot.epoch(spec.slots_per_epoch) == state.previous_epoch(spec) { + state.build_epoch_cache(RelativeEpoch::Previous, spec)?; + } + + validate_attestation(&mut state, attestation, spec).map_err(|e| e.into_with_index(i))?; + + let pending_attestation = PendingAttestation { + data: attestation.data.clone(), + aggregation_bitfield: attestation.aggregation_bitfield.clone(), + custody_bitfield: attestation.custody_bitfield.clone(), + inclusion_slot: state.slot, + }; + state.latest_attestations.push(pending_attestation); + } + + // Deposits + + verify!( + block.body.deposits.len() as u64 <= spec.max_deposits, + Invalid::MaxDepositsExceeded + ); + for (i, deposit) in block.body.deposits.iter().enumerate() { + verify_deposit(&mut state, deposit, spec).map_err(|e| e.into_with_index(i))?; + + state + .process_deposit( + deposit.deposit_data.deposit_input.pubkey.clone(), + deposit.deposit_data.amount, + deposit + .deposit_data + .deposit_input + .proof_of_possession + .clone(), + deposit.deposit_data.deposit_input.withdrawal_credentials, + None, + spec, + ) + .map_err(|_| Error::Invalid(Invalid::DepositProcessingFailed(i)))?; + + state.deposit_index += 1; + } + + // Exits + + verify!( + block.body.voluntary_exits.len() as u64 <= spec.max_voluntary_exits, + Invalid::MaxExitsExceeded + ); + for (i, exit) in block.body.voluntary_exits.iter().enumerate() { + verify_exit(&state, exit, spec).map_err(|e| e.into_with_index(i))?; + + state.initiate_validator_exit(exit.validator_index as usize); + } + + // Transfers + verify!( + block.body.transfers.len() as u64 <= spec.max_transfers, + Invalid::MaxTransfersExceed + ); + for (i, transfer) in block.body.transfers.iter().enumerate() { + verify_transfer(&state, transfer, spec).map_err(|e| e.into_with_index(i))?; + + let block_proposer = state.get_beacon_proposer_index(state.slot, spec)?; + + state.validator_balances[transfer.from as usize] -= transfer.amount + transfer.fee; + state.validator_balances[transfer.to as usize] += transfer.amount + transfer.fee; + state.validator_balances[block_proposer as usize] += transfer.fee; + } + + debug!("State transition complete."); + + Ok(()) +} + +/// Verifies the signature of a block. +/// +/// Spec v0.4.0 +pub fn verify_block_signature( + block: &BeaconBlock, + block_proposer: &Validator, + fork: &Fork, + spec: &ChainSpec, +) -> bool { + // Let proposal = `Proposal(block.slot, BEACON_CHAIN_SHARD_NUMBER, signed_root(block, + // "signature"), block.signature)`. + let proposal = Proposal { + slot: block.slot, + shard: spec.beacon_chain_shard_number, + block_root: Hash256::from(&block.signed_root()[..]), + signature: block.signature.clone(), + }; + let domain = spec.get_domain( + block.slot.epoch(spec.slots_per_epoch), + Domain::Proposal, + fork, + ); + // Verify that `bls_verify(pubkey=proposer.pubkey, message_hash=signed_root(proposal, + // "signature"), signature=proposal.signature, domain=get_domain(state.fork, + // get_current_epoch(state), DOMAIN_PROPOSAL))`. + proposal + .signature + .verify(&proposal.signed_root()[..], domain, &block_proposer.pubkey) +} + +/// Updates the present randao mix. +/// +/// Set `state.latest_randao_mixes[get_current_epoch(state) % LATEST_RANDAO_MIXES_LENGTH] = +/// xor(get_randao_mix(state, get_current_epoch(state)), hash(block.randao_reveal))`. +/// +/// Spec v0.4.0 +pub fn update_randao( + state: &mut BeaconState, + reveal: &Signature, + spec: &ChainSpec, +) -> Result<(), BeaconStateError> { + let hashed_reveal = { + let encoded_signature = ssz_encode(reveal); + Hash256::from(&hash(&encoded_signature[..])[..]) + }; + + let current_epoch = state.slot.epoch(spec.slots_per_epoch); + + let current_mix = state + .get_randao_mix(current_epoch, spec) + .ok_or_else(|| BeaconStateError::InsufficientRandaoMixes)?; + + let new_mix = *current_mix ^ hashed_reveal; + + let index = current_epoch.as_usize() % spec.latest_randao_mixes_length; + + if index < state.latest_randao_mixes.len() { + state.latest_randao_mixes[index] = new_mix; + Ok(()) + } else { + Err(BeaconStateError::InsufficientRandaoMixes) + } +} diff --git a/eth2/state_processing/src/block_processable/validate_attestation.rs b/eth2/state_processing/src/per_block_processing/validate_attestation.rs similarity index 83% rename from eth2/state_processing/src/block_processable/validate_attestation.rs rename to eth2/state_processing/src/per_block_processing/validate_attestation.rs index 1b422711c..8214e8d9a 100644 --- a/eth2/state_processing/src/block_processable/validate_attestation.rs +++ b/eth2/state_processing/src/per_block_processing/validate_attestation.rs @@ -3,27 +3,6 @@ use ssz::TreeHash; use types::beacon_state::helpers::*; use types::*; -/// Validate the attestations in some block, converting each into a `PendingAttestation` which is -/// then added to `state.latest_attestations`. -/// -/// Spec v0.4.0 -pub fn validate_attestations( - state: &BeaconState, - block: &BeaconBlock, - spec: &ChainSpec, -) -> Result<(), Error> { - ensure!( - block.body.attestations.len() as u64 <= spec.max_attestations, - MaxAttestationsExceeded - ); - - for attestation in &block.body.attestations { - validate_attestation(&state, attestation, spec)?; - } - - Ok(()) -} - /// Validate an attestation, checking the aggregate signature. /// /// Spec v0.4.0 @@ -48,7 +27,7 @@ pub fn validate_attestation_without_signature( /// Validate an attestation, optionally checking the aggregate signature. /// -/// Spec v0.2.0 +/// Spec v0.4.0 fn validate_attestation_signature_optional( state: &BeaconState, attestation: &Attestation, @@ -56,38 +35,41 @@ fn validate_attestation_signature_optional( verify_signature: bool, ) -> Result<(), Error> { // Verify that `attestation.data.slot >= GENESIS_SLOT`. - ensure!(attestation.data.slot >= spec.genesis_slot, PreGenesis); + verify!( + attestation.data.slot >= spec.genesis_slot, + Invalid::PreGenesis + ); // Verify that `attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot`. - ensure!( + verify!( attestation.data.slot + spec.min_attestation_inclusion_delay <= state.slot, - IncludedTooEarly + Invalid::IncludedTooEarly ); // Verify that `state.slot < attestation.data.slot + SLOTS_PER_EPOCH`. - ensure!( + verify!( state.slot < attestation.data.slot + spec.slots_per_epoch, - IncludedTooLate + Invalid::IncludedTooLate ); // Verify that `attestation.data.justified_epoch` is equal to `state.justified_epoch` if // `slot_to_epoch(attestation.data.slot + 1) >= get_current_epoch(state) else // state.previous_justified_epoch`. if (attestation.data.slot + 1).epoch(spec.slots_per_epoch) >= state.current_epoch(spec) { - ensure!( + verify!( attestation.data.justified_epoch == state.justified_epoch, - WrongJustifiedSlot + Invalid::WrongJustifiedSlot ); } else { - ensure!( + verify!( attestation.data.justified_epoch == state.previous_justified_epoch, - WrongJustifiedSlot + Invalid::WrongJustifiedSlot ); } // Verify that `attestation.data.justified_block_root` is equal to `get_block_root(state, // get_epoch_start_slot(attestation.data.justified_epoch))`. - ensure!( + verify!( attestation.data.justified_block_root == *state .get_block_root( @@ -98,7 +80,7 @@ fn validate_attestation_signature_optional( &spec ) .ok_or(BeaconStateError::InsufficientBlockRoots)?, - WrongJustifiedRoot + Invalid::WrongJustifiedRoot ); // Verify that either: @@ -112,59 +94,61 @@ fn validate_attestation_signature_optional( crosslink_data_root: attestation.data.crosslink_data_root, epoch: attestation.data.slot.epoch(spec.slots_per_epoch), }; - ensure!( + verify!( (attestation.data.latest_crosslink == state.latest_crosslinks[attestation.data.shard as usize]) | (state.latest_crosslinks[attestation.data.shard as usize] == potential_crosslink), - BadLatestCrosslinkRoot + Invalid::BadLatestCrosslinkRoot ); // Get the committee for this attestation let (committee, _shard) = state .get_crosslink_committees_at_slot(attestation.data.slot, spec)? .iter() - .find(|(committee, shard)| *shard == attestation.data.shard) + .find(|(_committee, shard)| *shard == attestation.data.shard) .ok_or_else(|| Error::Invalid(Invalid::NoCommitteeForShard))?; // Custody bitfield is all zeros (phase 0 requirement). - ensure!( + verify!( attestation.custody_bitfield.num_set_bits() == 0, - CustodyBitfieldHasSetBits + Invalid::CustodyBitfieldHasSetBits ); // Custody bitfield length is correct. - ensure!( + verify!( verify_bitfield_length(&attestation.aggregation_bitfield, committee.len()), - BadCustodyBitfieldLength + Invalid::BadCustodyBitfieldLength ); // Aggregation bitfield isn't empty. - ensure!( + verify!( attestation.aggregation_bitfield.num_set_bits() != 0, - AggregationBitfieldIsEmpty + Invalid::AggregationBitfieldIsEmpty ); // Aggregation bitfield length is correct. - ensure!( + verify!( verify_bitfield_length(&attestation.aggregation_bitfield, committee.len()), - BadAggregationBitfieldLength + Invalid::BadAggregationBitfieldLength ); if verify_signature { - ensure!( + let attestation_epoch = attestation.data.slot.epoch(spec.slots_per_epoch); + verify!( verify_attestation_signature( state, committee, + attestation_epoch, &attestation.custody_bitfield, &attestation.data, &attestation.aggregate_signature, spec ), - BadSignature + Invalid::BadSignature ); } // [TO BE REMOVED IN PHASE 1] Verify that `attestation.data.crosslink_data_root == ZERO_HASH`. - ensure!( + verify!( attestation.data.crosslink_data_root == spec.zero_hash, - ShardBlockRootNotZero + Invalid::ShardBlockRootNotZero ); Ok(()) @@ -177,9 +161,12 @@ fn validate_attestation_signature_optional( /// - `aggregate_signature` was not signed correctly. /// - `custody_bitfield` does not have a bit for each index of `committee`. /// - A `validator_index` in `committee` is not in `state.validator_registry`. +/// +/// Spec v0.4.0 fn verify_attestation_signature( state: &BeaconState, committee: &[usize], + attestation_epoch: Epoch, custody_bitfield: &Bitfield, attestation_data: &AttestationData, aggregate_signature: &AggregateSignature, @@ -234,5 +221,7 @@ fn verify_attestation_signature( keys.push(&aggregate_pubs[1]); } - aggregate_signature.verify_multiple(&messages[..], spec.domain_attestation, &keys[..]) + let domain = spec.get_domain(attestation_epoch, Domain::Attestation, &state.fork); + + aggregate_signature.verify_multiple(&messages[..], domain, &keys[..]) } diff --git a/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs b/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs new file mode 100644 index 000000000..b12ffa5ae --- /dev/null +++ b/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs @@ -0,0 +1,49 @@ +use super::verify_slashable_attestation::verify_slashable_attestation; +use crate::errors::{AttesterSlashingInvalid as Invalid, AttesterSlashingValidationError as Error}; +use types::*; + +/// Returns `Ok(())` if some `AttesterSlashing` is valid to be included in some `BeaconState`, +/// otherwise returns an `Err`. +/// +/// Returns the slashable indices from the `AttesterSlashing`. +/// +/// Spec v0.4.0 +pub fn verify_attester_slashing( + state: &BeaconState, + attester_slashing: &AttesterSlashing, + spec: &ChainSpec, +) -> Result, Error> { + let slashable_attestation_1 = &attester_slashing.slashable_attestation_1; + let slashable_attestation_2 = &attester_slashing.slashable_attestation_2; + + verify!( + slashable_attestation_1.data != slashable_attestation_2.data, + Invalid::AttestationDataIdentical + ); + verify!( + slashable_attestation_1.is_double_vote(slashable_attestation_2, spec) + | slashable_attestation_1.is_surround_vote(slashable_attestation_2, spec), + Invalid::NotSlashable + ); + + verify_slashable_attestation(state, &slashable_attestation_1, spec) + .map_err(|e| Error::Invalid(Invalid::SlashableAttestation1Invalid(e.into())))?; + verify_slashable_attestation(state, &slashable_attestation_2, spec) + .map_err(|e| Error::Invalid(Invalid::SlashableAttestation2Invalid(e.into())))?; + + let mut slashable_indices = vec![]; + for i in &slashable_attestation_1.validator_indices { + let validator = state + .validator_registry + .get(*i as usize) + .ok_or_else(|| Error::Invalid(Invalid::UnknownValidator))?; + + if slashable_attestation_1.validator_indices.contains(&i) & !validator.slashed { + slashable_indices.push(*i); + } + } + + verify!(!slashable_indices.is_empty(), Invalid::NoSlashableIndices); + + Ok(slashable_indices) +} diff --git a/eth2/state_processing/src/per_block_processing/verify_deposit.rs b/eth2/state_processing/src/per_block_processing/verify_deposit.rs new file mode 100644 index 000000000..20ed2f0b2 --- /dev/null +++ b/eth2/state_processing/src/per_block_processing/verify_deposit.rs @@ -0,0 +1,24 @@ +use crate::errors::{DepositInvalid as Invalid, DepositValidationError as Error}; +use ssz::TreeHash; +use types::beacon_state::helpers::verify_bitfield_length; +use types::*; + +/// Verify validity of ``slashable_attestation`` fields. +/// +/// Returns `Ok(())` if all fields are valid. +/// +/// Spec v0.4.0 +pub fn verify_deposit( + state: &BeaconState, + deposit: &Deposit, + spec: &ChainSpec, +) -> Result<(), Error> { + // TODO: verify serialized deposit data. + + // TODO: verify deposit index. + verify!(deposit.index == state.deposit_index, Invalid::BadIndex); + + // TODO: verify merkle branch. + + Ok(()) +} diff --git a/eth2/state_processing/src/per_block_processing/verify_exit.rs b/eth2/state_processing/src/per_block_processing/verify_exit.rs new file mode 100644 index 000000000..d4c2f7baa --- /dev/null +++ b/eth2/state_processing/src/per_block_processing/verify_exit.rs @@ -0,0 +1,41 @@ +use crate::errors::{ExitInvalid as Invalid, ExitValidationError as Error}; +use ssz::SignedRoot; +use types::*; + +/// Verify validity of ``slashable_attestation`` fields. +/// +/// Returns `Ok(())` if all fields are valid. +/// +/// Spec v0.4.0 +pub fn verify_exit( + state: &BeaconState, + exit: &VoluntaryExit, + spec: &ChainSpec, +) -> Result<(), Error> { + let validator = state + .validator_registry + .get(exit.validator_index as usize) + .ok_or(Error::Invalid(Invalid::ValidatorUnknown))?; + + verify!( + validator.exit_epoch + > state.get_delayed_activation_exit_epoch(state.current_epoch(spec), spec), + Invalid::AlreadyExited + ); + + verify!( + state.current_epoch(spec) >= exit.epoch, + Invalid::FutureEpoch + ); + + let message = exit.signed_root(); + let domain = spec.get_domain(exit.epoch, Domain::Exit, &state.fork); + + verify!( + exit.signature + .verify(&message[..], domain, &validator.pubkey), + Invalid::BadSignature + ); + + Ok(()) +} diff --git a/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs b/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs new file mode 100644 index 000000000..a7b344b2c --- /dev/null +++ b/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs @@ -0,0 +1,71 @@ +use crate::errors::{ProposerSlashingInvalid as Invalid, ProposerSlashingValidationError as Error}; +use ssz::SignedRoot; +use types::*; + +/// Returns `Ok(())` if some `ProposerSlashing` is valid to be included in some `BeaconState`, +/// otherwise returns an `Err`. +/// +/// Spec v0.4.0 +pub fn verify_proposer_slashing( + proposer_slashing: &ProposerSlashing, + state: &BeaconState, + spec: &ChainSpec, +) -> Result<(), Error> { + let proposer = state + .validator_registry + .get(proposer_slashing.proposer_index as usize) + .ok_or(Error::Invalid(Invalid::ProposerUnknown))?; + + verify!( + proposer_slashing.proposal_1.slot == proposer_slashing.proposal_2.slot, + Invalid::ProposalSlotMismatch + ); + + verify!( + proposer_slashing.proposal_1.shard == proposer_slashing.proposal_2.shard, + Invalid::ProposalShardMismatch + ); + + verify!( + proposer_slashing.proposal_1.block_root != proposer_slashing.proposal_2.block_root, + Invalid::ProposalBlockRootMismatch + ); + + verify!(!proposer.slashed, Invalid::ProposerAlreadySlashed); + + verify!( + verify_proposal_signature( + &proposer_slashing.proposal_1, + &proposer.pubkey, + &state.fork, + spec + ), + Invalid::BadProposal1Signature + ); + verify!( + verify_proposal_signature( + &proposer_slashing.proposal_2, + &proposer.pubkey, + &state.fork, + spec + ), + Invalid::BadProposal2Signature + ); + + Ok(()) +} + +fn verify_proposal_signature( + proposal: &Proposal, + pubkey: &PublicKey, + fork: &Fork, + spec: &ChainSpec, +) -> bool { + let message = proposal.signed_root(); + let domain = spec.get_domain( + proposal.slot.epoch(spec.slots_per_epoch), + Domain::Proposal, + fork, + ); + proposal.signature.verify(&message[..], domain, pubkey) +} diff --git a/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs b/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs new file mode 100644 index 000000000..cd0f9a201 --- /dev/null +++ b/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs @@ -0,0 +1,105 @@ +use crate::errors::{ + SlashableAttestationInvalid as Invalid, SlashableAttestationValidationError as Error, +}; +use ssz::TreeHash; +use types::beacon_state::helpers::verify_bitfield_length; +use types::*; + +/// Verify validity of ``slashable_attestation`` fields. +/// +/// Returns `Ok(())` if all fields are valid. +/// +/// Spec v0.4.0 +pub fn verify_slashable_attestation( + state: &BeaconState, + slashable_attestation: &SlashableAttestation, + spec: &ChainSpec, +) -> Result<(), Error> { + if slashable_attestation.custody_bitfield.num_set_bits() > 0 { + invalid!(Invalid::CustodyBitfieldHasSetBits); + } + + if slashable_attestation.validator_indices.is_empty() { + invalid!(Invalid::NoValidatorIndices); + } + + for i in 0..(slashable_attestation.validator_indices.len() - 1) { + if slashable_attestation.validator_indices[i] + >= slashable_attestation.validator_indices[i + 1] + { + invalid!(Invalid::BadValidatorIndicesOrdering); + } + } + + if !verify_bitfield_length( + &slashable_attestation.custody_bitfield, + slashable_attestation.validator_indices.len(), + ) { + invalid!(Invalid::BadCustodyBitfieldLength); + } + + if slashable_attestation.validator_indices.len() > spec.max_indices_per_slashable_vote as usize + { + invalid!(Invalid::MaxIndicesExceed); + } + + // TODO: this signature verification could likely be replaced with: + // + // super::validate_attestation::validate_attestation_signature(..) + + let mut aggregate_pubs = vec![AggregatePublicKey::new(); 2]; + let mut message_exists = vec![false; 2]; + + for (i, v) in slashable_attestation.validator_indices.iter().enumerate() { + let custody_bit = match slashable_attestation.custody_bitfield.get(i) { + Ok(bit) => bit, + Err(_) => unreachable!(), + }; + + message_exists[custody_bit as usize] = true; + + match state.validator_registry.get(*v as usize) { + Some(validator) => { + aggregate_pubs[custody_bit as usize].add(&validator.pubkey); + } + None => invalid!(Invalid::UnknownValidator), + }; + } + + let message_0 = AttestationDataAndCustodyBit { + data: slashable_attestation.data.clone(), + custody_bit: false, + } + .hash_tree_root(); + let message_1 = AttestationDataAndCustodyBit { + data: slashable_attestation.data.clone(), + custody_bit: true, + } + .hash_tree_root(); + + let mut messages = vec![]; + let mut keys = vec![]; + + if message_exists[0] { + messages.push(&message_0[..]); + keys.push(&aggregate_pubs[0]); + } + if message_exists[1] { + messages.push(&message_1[..]); + keys.push(&aggregate_pubs[1]); + } + + let domain = { + let epoch = slashable_attestation.data.slot.epoch(spec.slots_per_epoch); + spec.get_domain(epoch, Domain::Attestation, &state.fork) + }; + + verify!( + slashable_attestation + .aggregate_signature + .verify_multiple(&messages[..], domain, &keys[..]), + Invalid::BadSignature + ); + + Ok(()) +} diff --git a/eth2/state_processing/src/per_block_processing/verify_transfer.rs b/eth2/state_processing/src/per_block_processing/verify_transfer.rs new file mode 100644 index 000000000..0dee9a7d2 --- /dev/null +++ b/eth2/state_processing/src/per_block_processing/verify_transfer.rs @@ -0,0 +1,19 @@ +use crate::errors::{TransferInvalid as Invalid, TransferValidationError as Error}; +use ssz::TreeHash; +use types::beacon_state::helpers::verify_bitfield_length; +use types::*; + +/// Verify validity of ``slashable_attestation`` fields. +/// +/// Returns `Ok(())` if all fields are valid. +/// +/// Spec v0.4.0 +pub fn verify_transfer( + state: &BeaconState, + transfer: &Transfer, + spec: &ChainSpec, +) -> Result<(), Error> { + // TODO: verify transfer. + + Ok(()) +} diff --git a/eth2/types/src/attester_slashing/builder.rs b/eth2/types/src/attester_slashing/builder.rs index 6638eb2a5..bdb46b154 100644 --- a/eth2/types/src/attester_slashing/builder.rs +++ b/eth2/types/src/attester_slashing/builder.rs @@ -12,7 +12,7 @@ impl AttesterSlashingBuilder { /// - `validator_index: u64` /// - `message: &[u8]` /// - `epoch: Epoch` - /// - `domain: u64` + /// - `domain: Domain` /// /// Where domain is a domain "constant" (e.g., `spec.domain_attestation`). pub fn double_vote( @@ -21,7 +21,7 @@ impl AttesterSlashingBuilder { spec: &ChainSpec, ) -> AttesterSlashing where - F: Fn(u64, &[u8], Epoch, u64) -> Signature, + F: Fn(u64, &[u8], Epoch, Domain) -> Signature, { let double_voted_slot = Slot::new(0); let shard = 0; @@ -75,12 +75,7 @@ impl AttesterSlashingBuilder { custody_bit: attestation.custody_bitfield.get(i).unwrap(), }; let message = attestation_data_and_custody_bit.hash_tree_root(); - let signature = signer( - *validator_index, - &message[..], - epoch, - spec.domain_attestation, - ); + let signature = signer(*validator_index, &message[..], epoch, Domain::Attestation); attestation.aggregate_signature.add(&signature); } }; diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 2394c2c0d..76b97b21d 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -112,6 +112,7 @@ pub struct BeaconState { // Ethereum 1.0 chain data pub latest_eth1_data: Eth1Data, pub eth1_data_votes: Vec, + pub deposit_index: u64, // Caching (not in the spec) pub cache_index_offset: usize, @@ -187,6 +188,7 @@ impl BeaconState { */ latest_eth1_data, eth1_data_votes: vec![], + deposit_index: 0, /* * Caching (not in spec) @@ -224,6 +226,8 @@ impl BeaconState { } } + genesis_state.deposit_index = initial_validator_deposits.len() as u64; + let genesis_active_index_root = hash_tree_root(get_active_validator_indices( &genesis_state.validator_registry, spec.genesis_epoch, @@ -566,92 +570,6 @@ impl BeaconState { .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)) } - /// Verify validity of ``slashable_attestation`` fields. - /// - /// Spec v0.4.0 - pub fn verify_slashable_attestation( - &self, - slashable_attestation: &SlashableAttestation, - spec: &ChainSpec, - ) -> bool { - if slashable_attestation.custody_bitfield.num_set_bits() > 0 { - return false; - } - - if slashable_attestation.validator_indices.is_empty() { - return false; - } - - for i in 0..(slashable_attestation.validator_indices.len() - 1) { - if slashable_attestation.validator_indices[i] - >= slashable_attestation.validator_indices[i + 1] - { - return false; - } - } - - if !verify_bitfield_length( - &slashable_attestation.custody_bitfield, - slashable_attestation.validator_indices.len(), - ) { - return false; - } - - if slashable_attestation.validator_indices.len() - > spec.max_indices_per_slashable_vote as usize - { - return false; - } - - let mut aggregate_pubs = vec![AggregatePublicKey::new(); 2]; - let mut message_exists = vec![false; 2]; - - for (i, v) in slashable_attestation.validator_indices.iter().enumerate() { - let custody_bit = match slashable_attestation.custody_bitfield.get(i) { - Ok(bit) => bit, - Err(_) => unreachable!(), - }; - - message_exists[custody_bit as usize] = true; - - match self.validator_registry.get(*v as usize) { - Some(validator) => { - aggregate_pubs[custody_bit as usize].add(&validator.pubkey); - } - None => return false, - }; - } - - let message_0 = AttestationDataAndCustodyBit { - data: slashable_attestation.data.clone(), - custody_bit: false, - } - .hash_tree_root(); - let message_1 = AttestationDataAndCustodyBit { - data: slashable_attestation.data.clone(), - custody_bit: true, - } - .hash_tree_root(); - - let mut messages = vec![]; - let mut keys = vec![]; - - if message_exists[0] { - messages.push(&message_0[..]); - keys.push(&aggregate_pubs[0]); - } - if message_exists[1] { - messages.push(&message_1[..]); - keys.push(&aggregate_pubs[1]); - } - - slashable_attestation.aggregate_signature.verify_multiple( - &messages[..], - spec.domain_attestation, - &keys[..], - ) - } - /// Return the epoch at which an activation or exit triggered in ``epoch`` takes effect. /// /// Spec v0.4.0 @@ -1163,8 +1081,11 @@ impl BeaconState { proof_of_possession.verify( &proof_of_possession_data.hash_tree_root(), - self.fork - .get_domain(self.slot.epoch(spec.slots_per_epoch), spec.domain_deposit), + spec.get_domain( + self.slot.epoch(spec.slots_per_epoch), + Domain::Deposit, + &self.fork, + ), &pubkey, ) } @@ -1338,6 +1259,7 @@ impl Encodable for BeaconState { s.append(&self.batched_block_roots); s.append(&self.latest_eth1_data); s.append(&self.eth1_data_votes); + s.append(&self.deposit_index); } } @@ -1368,6 +1290,7 @@ impl Decodable for BeaconState { let (batched_block_roots, i) = <_>::ssz_decode(bytes, i)?; let (latest_eth1_data, i) = <_>::ssz_decode(bytes, i)?; let (eth1_data_votes, i) = <_>::ssz_decode(bytes, i)?; + let (deposit_index, i) = <_>::ssz_decode(bytes, i)?; Ok(( Self { @@ -1396,6 +1319,7 @@ impl Decodable for BeaconState { batched_block_roots, latest_eth1_data, eth1_data_votes, + deposit_index, cache_index_offset: 0, caches: vec![EpochCache::empty(); CACHED_EPOCHS], }, @@ -1440,6 +1364,7 @@ impl TreeHash for BeaconState { result.append(&mut self.batched_block_roots.hash_tree_root_internal()); result.append(&mut self.latest_eth1_data.hash_tree_root_internal()); result.append(&mut self.eth1_data_votes.hash_tree_root_internal()); + result.append(&mut self.deposit_index.hash_tree_root_internal()); hash(&result) } } @@ -1472,6 +1397,7 @@ impl TestRandom for BeaconState { batched_block_roots: <_>::random_for_test(rng), latest_eth1_data: <_>::random_for_test(rng), eth1_data_votes: <_>::random_for_test(rng), + deposit_index: <_>::random_for_test(rng), cache_index_offset: 0, caches: vec![EpochCache::empty(); CACHED_EPOCHS], } diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs index dea079755..789bb6c0c 100644 --- a/eth2/types/src/chain_spec.rs +++ b/eth2/types/src/chain_spec.rs @@ -1,8 +1,17 @@ -use crate::{Address, Epoch, Hash256, Slot}; +use crate::{Address, Epoch, Fork, Hash256, Slot}; use bls::Signature; const GWEI: u64 = 1_000_000_000; +pub enum Domain { + Deposit, + Attestation, + Proposal, + Exit, + Randao, + Transfer, +} + /// Holds all the "constants" for a BeaconChain. /// /// Spec v0.4.0 @@ -85,14 +94,20 @@ pub struct ChainSpec { /* * Signature domains + * + * Fields should be private to prevent accessing a domain that hasn't been modified to suit + * some `Fork`. + * + * Use `ChainSpec::get_domain(..)` to access these values. */ - pub domain_deposit: u64, - pub domain_attestation: u64, - pub domain_proposal: u64, - pub domain_exit: u64, - pub domain_randao: u64, - pub domain_transfer: u64, + domain_deposit: u64, + domain_attestation: u64, + domain_proposal: u64, + domain_exit: u64, + domain_randao: u64, + domain_transfer: u64, } + impl ChainSpec { /// Return the number of committees in one epoch. /// @@ -107,6 +122,23 @@ impl ChainSpec { ) * self.slots_per_epoch } + /// Get the domain number that represents the fork meta and signature domain. + /// + /// Spec v0.4.0 + pub fn get_domain(&self, epoch: Epoch, domain: Domain, fork: &Fork) -> u64 { + let domain_constant = match domain { + Domain::Deposit => self.domain_deposit, + Domain::Attestation => self.domain_attestation, + Domain::Proposal => self.domain_proposal, + Domain::Exit => self.domain_exit, + Domain::Randao => self.domain_randao, + Domain::Transfer => self.domain_transfer, + }; + + let fork_version = fork.get_fork_version(epoch); + fork_version * u64::pow(2, 32) + domain_constant + } + /// Returns a `ChainSpec` compatible with the Ethereum Foundation specification. /// /// Spec v0.4.0 diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs index 6c524a2d9..0acd6da90 100644 --- a/eth2/types/src/fork.rs +++ b/eth2/types/src/fork.rs @@ -24,14 +24,6 @@ impl Fork { } self.current_version } - - /// Get the domain number that represents the fork meta and signature domain. - /// - /// Spec v0.4.0 - pub fn get_domain(&self, epoch: Epoch, domain_type: u64) -> u64 { - let fork_version = self.get_fork_version(epoch); - fork_version * u64::pow(2, 32) + domain_type - } } #[cfg(test)] diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index f88cc6a17..e595684bc 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -43,7 +43,7 @@ pub use crate::beacon_block_body::BeaconBlockBody; pub use crate::beacon_state::{ BeaconState, Error as BeaconStateError, InclusionError, RelativeEpoch, }; -pub use crate::chain_spec::ChainSpec; +pub use crate::chain_spec::{ChainSpec, Domain}; pub use crate::crosslink::Crosslink; pub use crate::deposit::Deposit; pub use crate::deposit_data::DepositData; diff --git a/eth2/types/src/proposer_slashing/builder.rs b/eth2/types/src/proposer_slashing/builder.rs index 8dc1abfbe..1110bdf30 100644 --- a/eth2/types/src/proposer_slashing/builder.rs +++ b/eth2/types/src/proposer_slashing/builder.rs @@ -12,12 +12,12 @@ impl ProposerSlashingBuilder { /// - `validator_index: u64` /// - `message: &[u8]` /// - `epoch: Epoch` - /// - `domain: u64` + /// - `domain: Domain` /// /// Where domain is a domain "constant" (e.g., `spec.domain_attestation`). pub fn double_vote(proposer_index: u64, signer: F, spec: &ChainSpec) -> ProposerSlashing where - F: Fn(u64, &[u8], Epoch, u64) -> Signature, + F: Fn(u64, &[u8], Epoch, Domain) -> Signature, { let slot = Slot::new(0); let shard = 0; @@ -39,15 +39,13 @@ impl ProposerSlashingBuilder { proposal_1.signature = { let message = proposal_1.signed_root(); let epoch = slot.epoch(spec.slots_per_epoch); - let domain = spec.domain_proposal; - signer(proposer_index, &message[..], epoch, domain) + signer(proposer_index, &message[..], epoch, Domain::Proposal) }; proposal_2.signature = { let message = proposal_2.signed_root(); let epoch = slot.epoch(spec.slots_per_epoch); - let domain = spec.domain_proposal; - signer(proposer_index, &message[..], epoch, domain) + signer(proposer_index, &message[..], epoch, Domain::Proposal) }; ProposerSlashing { From 599948b26b5a430fd971c458dd10af99da7066b7 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 6 Mar 2019 16:24:56 +1100 Subject: [PATCH 115/132] Add comments to block_processing code --- eth2/state_processing/src/errors.rs | 148 +++++++++++++++--- .../src/per_block_processing.rs | 20 +++ .../validate_attestation.rs | 74 ++++++--- .../verify_attester_slashing.rs | 8 +- .../per_block_processing/verify_deposit.rs | 16 +- .../src/per_block_processing/verify_exit.rs | 11 +- .../verify_proposer_slashing.rs | 28 +++- .../verify_slashable_attestation.rs | 19 ++- .../per_block_processing/verify_transfer.rs | 15 +- 9 files changed, 258 insertions(+), 81 deletions(-) diff --git a/eth2/state_processing/src/errors.rs b/eth2/state_processing/src/errors.rs index 89cd82b24..6c9719fc2 100644 --- a/eth2/state_processing/src/errors.rs +++ b/eth2/state_processing/src/errors.rs @@ -1,4 +1,4 @@ -use types::BeaconStateError; +use types::*; macro_rules! impl_from_beacon_state_error { ($type: ident) => { @@ -39,23 +39,31 @@ macro_rules! impl_into_with_index_without_beacon_error { }; } +/// A conversion that consumes `self` and adds an `index` variable to resulting struct. +/// +/// Used here to allow converting an error into an upstream error that points to the object that +/// caused the error. For example, pointing to the index of an attestation that caused the +/// `AttestationInvalid` error. pub trait IntoWithIndex: Sized { - fn into_with_index(self, i: usize) -> T; + fn into_with_index(self, index: usize) -> T; } /* * Block Validation */ +/// The object is invalid or validation failed. #[derive(Debug, PartialEq)] pub enum BlockProcessingError { - /// The `BeaconBlock` is invalid. + /// Validation completed successfully and the object is invalid. Invalid(BlockInvalid), + /// Encountered a `BeaconStateError` whilst attempting to determine validity. BeaconStateError(BeaconStateError), } impl_from_beacon_state_error!(BlockProcessingError); +/// Describes why an object is invalid. #[derive(Debug, PartialEq)] pub enum BlockInvalid { StateSlotMismatch, @@ -87,28 +95,61 @@ impl Into for BlockInvalid { * Attestation Validation */ +/// The object is invalid or validation failed. #[derive(Debug, PartialEq)] pub enum AttestationValidationError { - /// The `Attestation` is invalid. + /// Validation completed successfully and the object is invalid. Invalid(AttestationInvalid), /// Encountered a `BeaconStateError` whilst attempting to determine validity. BeaconStateError(BeaconStateError), } +/// Describes why an object is invalid. #[derive(Debug, PartialEq)] pub enum AttestationInvalid { - PreGenesis, - IncludedTooEarly, - IncludedTooLate, - WrongJustifiedSlot, - WrongJustifiedRoot, + /// Attestation references a pre-genesis slot. + /// + /// (genesis_slot, attestation_slot) + PreGenesis(Slot, Slot), + /// Attestation included before the inclusion delay. + /// + /// (state_slot, inclusion_delay, attestation_slot) + IncludedTooEarly(Slot, u64, Slot), + /// Attestation slot is too far in the past to be included in a block. + /// + /// (state_slot, attestation_slot) + IncludedTooLate(Slot, Slot), + /// Attestation justified epoch does not match the states current or previous justified epoch. + /// + /// (attestation_justified_epoch, state_epoch, used_previous_epoch) + WrongJustifiedEpoch(Epoch, Epoch, bool), + /// Attestation justified epoch root does not match root known to the state. + /// + /// (state_justified_root, attestation_justified_root) + WrongJustifiedRoot(Hash256, Hash256), + /// Attestation crosslink root does not match the state crosslink root for the attestations + /// slot. BadLatestCrosslinkRoot, + /// The custody bitfield has some bits set `true`. This is not allowed in phase 0. CustodyBitfieldHasSetBits, + /// There are no set bits on the attestation -- an attestation must be signed by at least one + /// validator. AggregationBitfieldIsEmpty, - BadAggregationBitfieldLength, - BadCustodyBitfieldLength, - NoCommitteeForShard, + /// The custody bitfield length is not the smallest possible size to represent the committee. + /// + /// (committee_len, bitfield_len) + BadCustodyBitfieldLength(usize, usize), + /// The aggregation bitfield length is not the smallest possible size to represent the committee. + /// + /// (committee_len, bitfield_len) + BadAggregationBitfieldLength(usize, usize), + /// There was no known committee for the given shard in the given slot. + /// + /// (attestation_data_shard, attestation_data_slot) + NoCommitteeForShard(u64, Slot), + /// The attestation signature verification failed. BadSignature, + /// The shard block root was not set to zero. This is a phase 0 requirement. ShardBlockRootNotZero, } @@ -119,21 +160,29 @@ impl_into_with_index_with_beacon_error!(AttestationValidationError, AttestationI * `AttesterSlashing` Validation */ +/// The object is invalid or validation failed. #[derive(Debug, PartialEq)] pub enum AttesterSlashingValidationError { - /// The `SlashableAttestation` is invalid. + /// Validation completed successfully and the object is invalid. Invalid(AttesterSlashingInvalid), /// Encountered a `BeaconStateError` whilst attempting to determine validity. BeaconStateError(BeaconStateError), } +/// Describes why an object is invalid. #[derive(Debug, PartialEq)] pub enum AttesterSlashingInvalid { + /// The attestation data is identical, an attestation cannot conflict with itself. AttestationDataIdentical, + /// The attestations were not in conflict. NotSlashable, + /// The first `SlashableAttestation` was invalid. SlashableAttestation1Invalid(SlashableAttestationInvalid), + /// The second `SlashableAttestation` was invalid. SlashableAttestation2Invalid(SlashableAttestationInvalid), - UnknownValidator, + /// The validator index is unknown. One cannot slash one who does not exist. + UnknownValidator(u64), + /// There were no indices able to be slashed. NoSlashableIndices, } @@ -144,19 +193,35 @@ impl_into_with_index_with_beacon_error!(AttesterSlashingValidationError, Atteste * `SlashableAttestation` Validation */ +/// The object is invalid or validation failed. #[derive(Debug, PartialEq)] pub enum SlashableAttestationValidationError { + /// Validation completed successfully and the object is invalid. Invalid(SlashableAttestationInvalid), } +/// Describes why an object is invalid. #[derive(Debug, PartialEq)] pub enum SlashableAttestationInvalid { + /// The custody bitfield has some bits set `true`. This is not allowed in phase 0. CustodyBitfieldHasSetBits, + /// No validator indices were specified. NoValidatorIndices, - BadValidatorIndicesOrdering, - BadCustodyBitfieldLength, - MaxIndicesExceed, - UnknownValidator, + /// The validator indices were not in increasing order. + /// + /// The error occured between the given `index` and `index + 1` + BadValidatorIndicesOrdering(usize), + /// The custody bitfield length is not the smallest possible size to represent the validators. + /// + /// (validators_len, bitfield_len) + BadCustodyBitfieldLength(usize, usize), + /// The number of slashable indices exceed the global maximum. + /// + /// (max_indices, indices_given) + MaxIndicesExceed(usize, usize), + /// The validator index is unknown. One cannot slash one who does not exist. + UnknownValidator(u64), + /// The slashable attestation aggregate signature was not valid. BadSignature, } @@ -172,19 +237,35 @@ impl Into for SlashableAttestationValidationError { * `ProposerSlashing` Validation */ +/// The object is invalid or validation failed. #[derive(Debug, PartialEq)] pub enum ProposerSlashingValidationError { + /// Validation completed successfully and the object is invalid. Invalid(ProposerSlashingInvalid), } +/// Describes why an object is invalid. #[derive(Debug, PartialEq)] pub enum ProposerSlashingInvalid { - ProposerUnknown, - ProposalSlotMismatch, - ProposalShardMismatch, - ProposalBlockRootMismatch, + /// The proposer index is not a known validator. + ProposerUnknown(u64), + /// The two proposal have different slots. + /// + /// (proposal_1_slot, proposal_2_slot) + ProposalSlotMismatch(Slot, Slot), + /// The two proposal have different shards. + /// + /// (proposal_1_shard, proposal_2_shard) + ProposalShardMismatch(u64, u64), + /// The two proposal have different block roots. + /// + /// (proposal_1_root, proposal_2_root) + ProposalBlockRootMismatch(Hash256, Hash256), + /// The specified proposer has already been slashed. ProposerAlreadySlashed, + /// The first proposal signature was invalid. BadProposal1Signature, + /// The second proposal signature was invalid. BadProposal2Signature, } @@ -197,14 +278,20 @@ impl_into_with_index_without_beacon_error!( * `Deposit` Validation */ +/// The object is invalid or validation failed. #[derive(Debug, PartialEq)] pub enum DepositValidationError { + /// Validation completed successfully and the object is invalid. Invalid(DepositInvalid), } +/// Describes why an object is invalid. #[derive(Debug, PartialEq)] pub enum DepositInvalid { - BadIndex, + /// The deposit index does not match the state index. + /// + /// (state_index, deposit_index) + BadIndex(u64, u64), } impl_into_with_index_without_beacon_error!(DepositValidationError, DepositInvalid); @@ -213,16 +300,24 @@ impl_into_with_index_without_beacon_error!(DepositValidationError, DepositInvali * `Exit` Validation */ +/// The object is invalid or validation failed. #[derive(Debug, PartialEq)] pub enum ExitValidationError { + /// Validation completed successfully and the object is invalid. Invalid(ExitInvalid), } +/// Describes why an object is invalid. #[derive(Debug, PartialEq)] pub enum ExitInvalid { - ValidatorUnknown, + /// The specified validator is not in the state's validator registry. + ValidatorUnknown(u64), AlreadyExited, - FutureEpoch, + /// The exit is for a future epoch. + /// + /// (state_epoch, exit_epoch) + FutureEpoch(Epoch, Epoch), + /// The exit signature was not signed by the validator. BadSignature, } @@ -232,11 +327,14 @@ impl_into_with_index_without_beacon_error!(ExitValidationError, ExitInvalid); * `Transfer` Validation */ +/// The object is invalid or validation failed. #[derive(Debug, PartialEq)] pub enum TransferValidationError { + /// Validation completed successfully and the object is invalid. Invalid(TransferInvalid), } +/// Describes why an object is invalid. #[derive(Debug, PartialEq)] pub enum TransferInvalid {} diff --git a/eth2/state_processing/src/per_block_processing.rs b/eth2/state_processing/src/per_block_processing.rs index 06fe3b556..b9006d6f9 100644 --- a/eth2/state_processing/src/per_block_processing.rs +++ b/eth2/state_processing/src/per_block_processing.rs @@ -19,6 +19,12 @@ mod verify_proposer_slashing; mod verify_slashable_attestation; mod verify_transfer; +/// Updates the state for a new block, whilst validating that the block is valid. +/// +/// Returns `Ok(())` if the block is valid and the state was successfully updated. Otherwise +/// returns an error describing why the block was invalid or how the function failed to execute. +/// +/// Spec v0.4.0 pub fn per_block_processing( state: &mut BeaconState, block: &BeaconBlock, @@ -27,6 +33,13 @@ pub fn per_block_processing( per_block_processing_signature_optional(state, block, true, spec) } +/// Updates the state for a new block, whilst validating that the block is valid, without actually +/// checking the block proposer signature. +/// +/// Returns `Ok(())` if the block is valid and the state was successfully updated. Otherwise +/// returns an error describing why the block was invalid or how the function failed to execute. +/// +/// Spec v0.4.0 pub fn per_block_processing_without_verifying_block_signature( state: &mut BeaconState, block: &BeaconBlock, @@ -35,6 +48,13 @@ pub fn per_block_processing_without_verifying_block_signature( per_block_processing_signature_optional(state, block, false, spec) } +/// Updates the state for a new block, whilst validating that the block is valid, optionally +/// checking the block proposer signature. +/// +/// Returns `Ok(())` if the block is valid and the state was successfully updated. Otherwise +/// returns an error describing why the block was invalid or how the function failed to execute. +/// +/// Spec v0.4.0 fn per_block_processing_signature_optional( mut state: &mut BeaconState, block: &BeaconBlock, diff --git a/eth2/state_processing/src/per_block_processing/validate_attestation.rs b/eth2/state_processing/src/per_block_processing/validate_attestation.rs index 8214e8d9a..8749bc5ea 100644 --- a/eth2/state_processing/src/per_block_processing/validate_attestation.rs +++ b/eth2/state_processing/src/per_block_processing/validate_attestation.rs @@ -3,7 +3,10 @@ use ssz::TreeHash; use types::beacon_state::helpers::*; use types::*; -/// Validate an attestation, checking the aggregate signature. +/// Indicates if an `Attestation` is valid to be included in a block in the current epoch of the +/// given state. +/// +/// Returns `Ok(())` if the `Attestation` is valid, otherwise indicates the reason for invalidity. /// /// Spec v0.4.0 pub fn validate_attestation( @@ -14,7 +17,10 @@ pub fn validate_attestation( validate_attestation_signature_optional(state, attestation, spec, true) } -/// Validate an attestation, without checking the aggregate signature. +/// Indicates if an `Attestation` is valid to be included in a block in the current epoch of the +/// given state, without validating the aggregate signature. +/// +/// Returns `Ok(())` if the `Attestation` is valid, otherwise indicates the reason for invalidity. /// /// Spec v0.4.0 pub fn validate_attestation_without_signature( @@ -25,7 +31,9 @@ pub fn validate_attestation_without_signature( validate_attestation_signature_optional(state, attestation, spec, false) } -/// Validate an attestation, optionally checking the aggregate signature. +/// Indicates if an `Attestation` is valid to be included in a block in the current epoch of the +/// given state, optionally validating the aggregate signature. +/// /// /// Spec v0.4.0 fn validate_attestation_signature_optional( @@ -37,19 +45,23 @@ fn validate_attestation_signature_optional( // Verify that `attestation.data.slot >= GENESIS_SLOT`. verify!( attestation.data.slot >= spec.genesis_slot, - Invalid::PreGenesis + Invalid::PreGenesis(spec.genesis_slot, attestation.data.slot) ); // Verify that `attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot`. verify!( attestation.data.slot + spec.min_attestation_inclusion_delay <= state.slot, - Invalid::IncludedTooEarly + Invalid::IncludedTooEarly( + state.slot, + spec.min_attestation_inclusion_delay, + attestation.data.slot + ) ); // Verify that `state.slot < attestation.data.slot + SLOTS_PER_EPOCH`. verify!( state.slot < attestation.data.slot + spec.slots_per_epoch, - Invalid::IncludedTooLate + Invalid::IncludedTooLate(state.slot, attestation.data.slot) ); // Verify that `attestation.data.justified_epoch` is equal to `state.justified_epoch` if @@ -58,29 +70,37 @@ fn validate_attestation_signature_optional( if (attestation.data.slot + 1).epoch(spec.slots_per_epoch) >= state.current_epoch(spec) { verify!( attestation.data.justified_epoch == state.justified_epoch, - Invalid::WrongJustifiedSlot + Invalid::WrongJustifiedEpoch( + attestation.data.justified_epoch, + state.justified_epoch, + false + ) ); } else { verify!( attestation.data.justified_epoch == state.previous_justified_epoch, - Invalid::WrongJustifiedSlot + Invalid::WrongJustifiedEpoch( + attestation.data.justified_epoch, + state.previous_justified_epoch, + true + ) ); } // Verify that `attestation.data.justified_block_root` is equal to `get_block_root(state, // get_epoch_start_slot(attestation.data.justified_epoch))`. + let justified_block_root = *state + .get_block_root( + attestation + .data + .justified_epoch + .start_slot(spec.slots_per_epoch), + &spec, + ) + .ok_or(BeaconStateError::InsufficientBlockRoots)?; verify!( - attestation.data.justified_block_root - == *state - .get_block_root( - attestation - .data - .justified_epoch - .start_slot(spec.slots_per_epoch), - &spec - ) - .ok_or(BeaconStateError::InsufficientBlockRoots)?, - Invalid::WrongJustifiedRoot + attestation.data.justified_block_root == justified_block_root, + Invalid::WrongJustifiedRoot(justified_block_root, attestation.data.justified_block_root) ); // Verify that either: @@ -106,7 +126,12 @@ fn validate_attestation_signature_optional( .get_crosslink_committees_at_slot(attestation.data.slot, spec)? .iter() .find(|(_committee, shard)| *shard == attestation.data.shard) - .ok_or_else(|| Error::Invalid(Invalid::NoCommitteeForShard))?; + .ok_or_else(|| { + Error::Invalid(Invalid::NoCommitteeForShard( + attestation.data.shard, + attestation.data.slot, + )) + })?; // Custody bitfield is all zeros (phase 0 requirement). verify!( @@ -115,8 +140,8 @@ fn validate_attestation_signature_optional( ); // Custody bitfield length is correct. verify!( - verify_bitfield_length(&attestation.aggregation_bitfield, committee.len()), - Invalid::BadCustodyBitfieldLength + verify_bitfield_length(&attestation.custody_bitfield, committee.len()), + Invalid::BadCustodyBitfieldLength(committee.len(), attestation.custody_bitfield.len()) ); // Aggregation bitfield isn't empty. verify!( @@ -126,7 +151,10 @@ fn validate_attestation_signature_optional( // Aggregation bitfield length is correct. verify!( verify_bitfield_length(&attestation.aggregation_bitfield, committee.len()), - Invalid::BadAggregationBitfieldLength + Invalid::BadAggregationBitfieldLength( + committee.len(), + attestation.aggregation_bitfield.len() + ) ); if verify_signature { diff --git a/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs b/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs index b12ffa5ae..aef09943a 100644 --- a/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs +++ b/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs @@ -2,10 +2,10 @@ use super::verify_slashable_attestation::verify_slashable_attestation; use crate::errors::{AttesterSlashingInvalid as Invalid, AttesterSlashingValidationError as Error}; use types::*; -/// Returns `Ok(())` if some `AttesterSlashing` is valid to be included in some `BeaconState`, -/// otherwise returns an `Err`. +/// Indicates if an `AttesterSlashing` is valid to be included in a block in the current epoch of the given +/// state. /// -/// Returns the slashable indices from the `AttesterSlashing`. +/// Returns `Ok(())` if the `AttesterSlashing` is valid, otherwise indicates the reason for invalidity. /// /// Spec v0.4.0 pub fn verify_attester_slashing( @@ -36,7 +36,7 @@ pub fn verify_attester_slashing( let validator = state .validator_registry .get(*i as usize) - .ok_or_else(|| Error::Invalid(Invalid::UnknownValidator))?; + .ok_or_else(|| Error::Invalid(Invalid::UnknownValidator(*i)))?; if slashable_attestation_1.validator_indices.contains(&i) & !validator.slashed { slashable_indices.push(*i); diff --git a/eth2/state_processing/src/per_block_processing/verify_deposit.rs b/eth2/state_processing/src/per_block_processing/verify_deposit.rs index 20ed2f0b2..59a2a66c4 100644 --- a/eth2/state_processing/src/per_block_processing/verify_deposit.rs +++ b/eth2/state_processing/src/per_block_processing/verify_deposit.rs @@ -1,22 +1,26 @@ use crate::errors::{DepositInvalid as Invalid, DepositValidationError as Error}; -use ssz::TreeHash; -use types::beacon_state::helpers::verify_bitfield_length; use types::*; -/// Verify validity of ``slashable_attestation`` fields. +/// Indicates if a `Deposit` is valid to be included in a block in the current epoch of the given +/// state. /// -/// Returns `Ok(())` if all fields are valid. +/// Returns `Ok(())` if the `Deposit` is valid, otherwise indicates the reason for invalidity. +/// +/// Note: this function is incomplete. /// /// Spec v0.4.0 pub fn verify_deposit( state: &BeaconState, deposit: &Deposit, - spec: &ChainSpec, + _spec: &ChainSpec, ) -> Result<(), Error> { // TODO: verify serialized deposit data. // TODO: verify deposit index. - verify!(deposit.index == state.deposit_index, Invalid::BadIndex); + verify!( + deposit.index == state.deposit_index, + Invalid::BadIndex(state.deposit_index, deposit.index) + ); // TODO: verify merkle branch. diff --git a/eth2/state_processing/src/per_block_processing/verify_exit.rs b/eth2/state_processing/src/per_block_processing/verify_exit.rs index d4c2f7baa..34c55c55e 100644 --- a/eth2/state_processing/src/per_block_processing/verify_exit.rs +++ b/eth2/state_processing/src/per_block_processing/verify_exit.rs @@ -2,9 +2,10 @@ use crate::errors::{ExitInvalid as Invalid, ExitValidationError as Error}; use ssz::SignedRoot; use types::*; -/// Verify validity of ``slashable_attestation`` fields. +/// Indicates if an `Exit` is valid to be included in a block in the current epoch of the given +/// state. /// -/// Returns `Ok(())` if all fields are valid. +/// Returns `Ok(())` if the `Exit` is valid, otherwise indicates the reason for invalidity. /// /// Spec v0.4.0 pub fn verify_exit( @@ -15,7 +16,9 @@ pub fn verify_exit( let validator = state .validator_registry .get(exit.validator_index as usize) - .ok_or(Error::Invalid(Invalid::ValidatorUnknown))?; + .ok_or(Error::Invalid(Invalid::ValidatorUnknown( + exit.validator_index, + )))?; verify!( validator.exit_epoch @@ -25,7 +28,7 @@ pub fn verify_exit( verify!( state.current_epoch(spec) >= exit.epoch, - Invalid::FutureEpoch + Invalid::FutureEpoch(state.current_epoch(spec), exit.epoch) ); let message = exit.signed_root(); diff --git a/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs b/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs index a7b344b2c..c4e51b1a8 100644 --- a/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs +++ b/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs @@ -2,8 +2,10 @@ use crate::errors::{ProposerSlashingInvalid as Invalid, ProposerSlashingValidati use ssz::SignedRoot; use types::*; -/// Returns `Ok(())` if some `ProposerSlashing` is valid to be included in some `BeaconState`, -/// otherwise returns an `Err`. +/// Indicates if a `ProposerSlashing` is valid to be included in a block in the current epoch of the given +/// state. +/// +/// Returns `Ok(())` if the `ProposerSlashing` is valid, otherwise indicates the reason for invalidity. /// /// Spec v0.4.0 pub fn verify_proposer_slashing( @@ -14,21 +16,32 @@ pub fn verify_proposer_slashing( let proposer = state .validator_registry .get(proposer_slashing.proposer_index as usize) - .ok_or(Error::Invalid(Invalid::ProposerUnknown))?; + .ok_or(Error::Invalid(Invalid::ProposerUnknown( + proposer_slashing.proposer_index, + )))?; verify!( proposer_slashing.proposal_1.slot == proposer_slashing.proposal_2.slot, - Invalid::ProposalSlotMismatch + Invalid::ProposalSlotMismatch( + proposer_slashing.proposal_1.slot, + proposer_slashing.proposal_2.slot + ) ); verify!( proposer_slashing.proposal_1.shard == proposer_slashing.proposal_2.shard, - Invalid::ProposalShardMismatch + Invalid::ProposalShardMismatch( + proposer_slashing.proposal_1.shard, + proposer_slashing.proposal_2.shard + ) ); verify!( proposer_slashing.proposal_1.block_root != proposer_slashing.proposal_2.block_root, - Invalid::ProposalBlockRootMismatch + Invalid::ProposalBlockRootMismatch( + proposer_slashing.proposal_1.block_root, + proposer_slashing.proposal_2.block_root + ) ); verify!(!proposer.slashed, Invalid::ProposerAlreadySlashed); @@ -55,6 +68,9 @@ pub fn verify_proposer_slashing( Ok(()) } +/// Verifies the signature of a proposal. +/// +/// Returns `true` if the signature is valid. fn verify_proposal_signature( proposal: &Proposal, pubkey: &PublicKey, diff --git a/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs b/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs index cd0f9a201..0b948ad89 100644 --- a/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs +++ b/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs @@ -5,9 +5,10 @@ use ssz::TreeHash; use types::beacon_state::helpers::verify_bitfield_length; use types::*; -/// Verify validity of ``slashable_attestation`` fields. +/// Indicates if a `SlashableAttestation` is valid to be included in a block in the current epoch of the given +/// state. /// -/// Returns `Ok(())` if all fields are valid. +/// Returns `Ok(())` if the `SlashableAttestation` is valid, otherwise indicates the reason for invalidity. /// /// Spec v0.4.0 pub fn verify_slashable_attestation( @@ -27,7 +28,7 @@ pub fn verify_slashable_attestation( if slashable_attestation.validator_indices[i] >= slashable_attestation.validator_indices[i + 1] { - invalid!(Invalid::BadValidatorIndicesOrdering); + invalid!(Invalid::BadValidatorIndicesOrdering(i)); } } @@ -35,12 +36,18 @@ pub fn verify_slashable_attestation( &slashable_attestation.custody_bitfield, slashable_attestation.validator_indices.len(), ) { - invalid!(Invalid::BadCustodyBitfieldLength); + invalid!(Invalid::BadCustodyBitfieldLength( + slashable_attestation.validator_indices.len(), + slashable_attestation.custody_bitfield.len() + )); } if slashable_attestation.validator_indices.len() > spec.max_indices_per_slashable_vote as usize { - invalid!(Invalid::MaxIndicesExceed); + invalid!(Invalid::MaxIndicesExceed( + spec.max_indices_per_slashable_vote as usize, + slashable_attestation.validator_indices.len() + )); } // TODO: this signature verification could likely be replaced with: @@ -62,7 +69,7 @@ pub fn verify_slashable_attestation( Some(validator) => { aggregate_pubs[custody_bit as usize].add(&validator.pubkey); } - None => invalid!(Invalid::UnknownValidator), + None => invalid!(Invalid::UnknownValidator(*v)), }; } diff --git a/eth2/state_processing/src/per_block_processing/verify_transfer.rs b/eth2/state_processing/src/per_block_processing/verify_transfer.rs index 0dee9a7d2..220f5e496 100644 --- a/eth2/state_processing/src/per_block_processing/verify_transfer.rs +++ b/eth2/state_processing/src/per_block_processing/verify_transfer.rs @@ -1,17 +1,18 @@ use crate::errors::{TransferInvalid as Invalid, TransferValidationError as Error}; -use ssz::TreeHash; -use types::beacon_state::helpers::verify_bitfield_length; use types::*; -/// Verify validity of ``slashable_attestation`` fields. +/// Indicates if a `Transfer` is valid to be included in a block in the current epoch of the given +/// state. /// -/// Returns `Ok(())` if all fields are valid. +/// Returns `Ok(())` if the `Transfer` is valid, otherwise indicates the reason for invalidity. +/// +/// Note: this function is incomplete. /// /// Spec v0.4.0 pub fn verify_transfer( - state: &BeaconState, - transfer: &Transfer, - spec: &ChainSpec, + _state: &BeaconState, + _transfer: &Transfer, + _spec: &ChainSpec, ) -> Result<(), Error> { // TODO: verify transfer. From 7bb5e1c1512729100c6d773456d65258ccbcedcc Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 6 Mar 2019 16:59:36 +1100 Subject: [PATCH 116/132] Implement library for verifying Merkle proofs. --- Cargo.toml | 1 + eth2/utils/merkle_proof/Cargo.toml | 9 ++ eth2/utils/merkle_proof/src/lib.rs | 148 +++++++++++++++++++++++++++++ 3 files changed, 158 insertions(+) create mode 100644 eth2/utils/merkle_proof/Cargo.toml create mode 100644 eth2/utils/merkle_proof/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 42d69489b..c5aae7f43 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "eth2/utils/boolean-bitfield", "eth2/utils/hashing", "eth2/utils/honey-badger-split", + "eth2/utils/merkle_proof", "eth2/utils/int_to_bytes", "eth2/utils/slot_clock", "eth2/utils/ssz", diff --git a/eth2/utils/merkle_proof/Cargo.toml b/eth2/utils/merkle_proof/Cargo.toml new file mode 100644 index 000000000..b7cd81216 --- /dev/null +++ b/eth2/utils/merkle_proof/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "merkle_proof" +version = "0.1.0" +authors = ["Michael Sproul "] +edition = "2018" + +[dependencies] +ethereum-types = "0.5" +hashing = { path = "../hashing" } diff --git a/eth2/utils/merkle_proof/src/lib.rs b/eth2/utils/merkle_proof/src/lib.rs new file mode 100644 index 000000000..4d0abcea3 --- /dev/null +++ b/eth2/utils/merkle_proof/src/lib.rs @@ -0,0 +1,148 @@ +use ethereum_types::H256; +use hashing::hash; + +/// Verify a proof that `leaf` exists at `index` in a Merkle tree rooted at `root`. +/// +/// The `branch` argument is the main component of the proof: it should be a list of internal +/// node hashes such that the root can be reconstructed (in bottom-up order). +pub fn verify_merkle_proof( + leaf: H256, + branch: &[H256], + depth: usize, + index: usize, + root: H256, +) -> bool { + if branch.len() == depth { + merkle_root_from_branch(leaf, branch, depth, index) == root + } else { + false + } +} + +/// Compute a root hash from a leaf and a Merkle proof. +fn merkle_root_from_branch(leaf: H256, branch: &[H256], depth: usize, index: usize) -> H256 { + assert_eq!(branch.len(), depth, "proof length should equal depth"); + + let mut merkle_root = leaf.as_bytes().to_vec(); + + for i in 0..depth { + let ith_bit = (index >> i) & 0x01; + if ith_bit == 1 { + let input = concat(branch[i].as_bytes().to_vec(), merkle_root); + merkle_root = hash(&input); + } else { + let mut input = merkle_root; + input.extend_from_slice(branch[i].as_bytes()); + merkle_root = hash(&input); + } + } + + H256::from_slice(&merkle_root) +} + +/// Concatenate two vectors. +fn concat(mut vec1: Vec, mut vec2: Vec) -> Vec { + vec1.append(&mut vec2); + vec1 +} + +#[cfg(test)] +mod tests { + use super::*; + + fn hash_concat(h1: H256, h2: H256) -> H256 { + H256::from_slice(&hash(&concat( + h1.as_bytes().to_vec(), + h2.as_bytes().to_vec(), + ))) + } + + #[test] + fn verify_small_example() { + // Construct a small merkle tree manually + let leaf_b00 = H256::from([0xAA; 32]); + let leaf_b01 = H256::from([0xBB; 32]); + let leaf_b10 = H256::from([0xCC; 32]); + let leaf_b11 = H256::from([0xDD; 32]); + + let node_b0x = hash_concat(leaf_b00, leaf_b01); + let node_b1x = hash_concat(leaf_b10, leaf_b11); + + let root = hash_concat(node_b0x, node_b1x); + + // Run some proofs + assert!(verify_merkle_proof( + leaf_b00, + &[leaf_b01, node_b1x], + 2, + 0b00, + root + )); + assert!(verify_merkle_proof( + leaf_b01, + &[leaf_b00, node_b1x], + 2, + 0b01, + root + )); + assert!(verify_merkle_proof( + leaf_b10, + &[leaf_b11, node_b0x], + 2, + 0b10, + root + )); + assert!(verify_merkle_proof( + leaf_b11, + &[leaf_b10, node_b0x], + 2, + 0b11, + root + )); + assert!(verify_merkle_proof( + leaf_b11, + &[leaf_b10], + 1, + 0b11, + node_b1x + )); + + // Ensure that incorrect proofs fail + // Zero-length proof + assert!(!verify_merkle_proof(leaf_b01, &[], 2, 0b01, root)); + // Proof in reverse order + assert!(!verify_merkle_proof( + leaf_b01, + &[node_b1x, leaf_b00], + 2, + 0b01, + root + )); + // Proof too short + assert!(!verify_merkle_proof(leaf_b01, &[leaf_b00], 2, 0b01, root)); + // Wrong index + assert!(!verify_merkle_proof( + leaf_b01, + &[leaf_b00, node_b1x], + 2, + 0b10, + root + )); + // Wrong root + assert!(!verify_merkle_proof( + leaf_b01, + &[leaf_b00, node_b1x], + 2, + 0b01, + node_b1x + )); + } + + #[test] + fn verify_zero_depth() { + let leaf = H256::from([0xD6; 32]); + let junk = H256::from([0xD7; 32]); + assert!(verify_merkle_proof(leaf, &[], 0, 0, leaf)); + assert!(!verify_merkle_proof(leaf, &[], 0, 7, junk)); + } +} From 521d48d37c3bf1e1c5034876cb409202ea83a92b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 6 Mar 2019 17:03:18 +1100 Subject: [PATCH 117/132] Break per-block-processing into smaller fns Will enable benchmarking of each individual fn --- .../src/per_block_processing.rs | 418 +++++++++++------- 1 file changed, 257 insertions(+), 161 deletions(-) diff --git a/eth2/state_processing/src/per_block_processing.rs b/eth2/state_processing/src/per_block_processing.rs index b9006d6f9..573513819 100644 --- a/eth2/state_processing/src/per_block_processing.rs +++ b/eth2/state_processing/src/per_block_processing.rs @@ -64,161 +64,22 @@ fn per_block_processing_signature_optional( // Verify that `block.slot == state.slot`. verify!(block.slot == state.slot, Invalid::StateSlotMismatch); - // Get the epoch for future ergonomics. - let epoch = block.slot.epoch(spec.slots_per_epoch); - // Ensure the current epoch cache is built. state.build_epoch_cache(RelativeEpoch::Current, spec)?; - // Let `proposer = state.validator_registry[get_beacon_proposer_index(state, state.slot)]`. - let block_proposer = - &state.validator_registry[state.get_beacon_proposer_index(block.slot, spec)?]; - - // Block signature if should_verify_block_signature { - verify!( - verify_block_signature(&block, &block_proposer, &state.fork, spec,), - Invalid::BadSignature - ); + verify_block_signature(&state, &block, &spec)?; } + process_randao(&mut state, &block, &spec)?; + process_eth1_data(&mut state, &block.eth1_data)?; + process_proposer_slashings(&mut state, &block.body.proposer_slashings[..], spec)?; + process_attester_slashings(&mut state, &block.body.attester_slashings[..], spec)?; + process_attestations(&mut state, &block.body.attestations[..], spec)?; + process_deposits(&mut state, &block.body.deposits[..], spec)?; + process_exits(&mut state, &block.body.voluntary_exits[..], spec)?; + process_transfers(&mut state, &block.body.transfers[..], spec)?; - // Randao - - // Verify that `bls_verify(pubkey=proposer.pubkey, - // message_hash=hash_tree_root(get_current_epoch(state)), signature=block.randao_reveal, - // domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_RANDAO))`. - verify!( - block.randao_reveal.verify( - &state.current_epoch(spec).hash_tree_root()[..], - spec.get_domain(epoch, Domain::Randao, &state.fork), - &block_proposer.pubkey - ), - Invalid::BadRandaoSignature - ); - - // Update the state's RANDAO mix with the one revealed in the block. - update_randao(&mut state, &block.randao_reveal, spec)?; - - // Eth1 Data - - // Either increment the eth1_data vote count, or add a new eth1_data. - let matching_eth1_vote_index = state - .eth1_data_votes - .iter() - .position(|vote| vote.eth1_data == block.eth1_data); - if let Some(index) = matching_eth1_vote_index { - state.eth1_data_votes[index].vote_count += 1; - } else { - state.eth1_data_votes.push(Eth1DataVote { - eth1_data: block.eth1_data.clone(), - vote_count: 1, - }); - } - - //Proposer slashings - - verify!( - block.body.proposer_slashings.len() as u64 <= spec.max_proposer_slashings, - Invalid::MaxProposerSlashingsExceeded - ); - for (i, proposer_slashing) in block.body.proposer_slashings.iter().enumerate() { - verify_proposer_slashing(proposer_slashing, &state, spec) - .map_err(|e| e.into_with_index(i))?; - state.slash_validator(proposer_slashing.proposer_index as usize, spec)?; - } - - // Attester Slashings - - verify!( - block.body.attester_slashings.len() as u64 <= spec.max_attester_slashings, - Invalid::MaxAttesterSlashingsExceed - ); - for (i, attester_slashing) in block.body.attester_slashings.iter().enumerate() { - let slashable_indices = verify_attester_slashing(&state, &attester_slashing, spec) - .map_err(|e| e.into_with_index(i))?; - for i in slashable_indices { - state.slash_validator(i as usize, spec)?; - } - } - - // Attestations - - verify!( - block.body.attestations.len() as u64 <= spec.max_attestations, - Invalid::MaxAttestationsExceeded - ); - for (i, attestation) in block.body.attestations.iter().enumerate() { - // Build the previous epoch cache only if required by an attestation. - if attestation.data.slot.epoch(spec.slots_per_epoch) == state.previous_epoch(spec) { - state.build_epoch_cache(RelativeEpoch::Previous, spec)?; - } - - validate_attestation(&mut state, attestation, spec).map_err(|e| e.into_with_index(i))?; - - let pending_attestation = PendingAttestation { - data: attestation.data.clone(), - aggregation_bitfield: attestation.aggregation_bitfield.clone(), - custody_bitfield: attestation.custody_bitfield.clone(), - inclusion_slot: state.slot, - }; - state.latest_attestations.push(pending_attestation); - } - - // Deposits - - verify!( - block.body.deposits.len() as u64 <= spec.max_deposits, - Invalid::MaxDepositsExceeded - ); - for (i, deposit) in block.body.deposits.iter().enumerate() { - verify_deposit(&mut state, deposit, spec).map_err(|e| e.into_with_index(i))?; - - state - .process_deposit( - deposit.deposit_data.deposit_input.pubkey.clone(), - deposit.deposit_data.amount, - deposit - .deposit_data - .deposit_input - .proof_of_possession - .clone(), - deposit.deposit_data.deposit_input.withdrawal_credentials, - None, - spec, - ) - .map_err(|_| Error::Invalid(Invalid::DepositProcessingFailed(i)))?; - - state.deposit_index += 1; - } - - // Exits - - verify!( - block.body.voluntary_exits.len() as u64 <= spec.max_voluntary_exits, - Invalid::MaxExitsExceeded - ); - for (i, exit) in block.body.voluntary_exits.iter().enumerate() { - verify_exit(&state, exit, spec).map_err(|e| e.into_with_index(i))?; - - state.initiate_validator_exit(exit.validator_index as usize); - } - - // Transfers - verify!( - block.body.transfers.len() as u64 <= spec.max_transfers, - Invalid::MaxTransfersExceed - ); - for (i, transfer) in block.body.transfers.iter().enumerate() { - verify_transfer(&state, transfer, spec).map_err(|e| e.into_with_index(i))?; - - let block_proposer = state.get_beacon_proposer_index(state.slot, spec)?; - - state.validator_balances[transfer.from as usize] -= transfer.amount + transfer.fee; - state.validator_balances[transfer.to as usize] += transfer.amount + transfer.fee; - state.validator_balances[block_proposer as usize] += transfer.fee; - } - - debug!("State transition complete."); + debug!("per_block_processing complete."); Ok(()) } @@ -227,13 +88,13 @@ fn per_block_processing_signature_optional( /// /// Spec v0.4.0 pub fn verify_block_signature( + state: &BeaconState, block: &BeaconBlock, - block_proposer: &Validator, - fork: &Fork, spec: &ChainSpec, -) -> bool { - // Let proposal = `Proposal(block.slot, BEACON_CHAIN_SHARD_NUMBER, signed_root(block, - // "signature"), block.signature)`. +) -> Result<(), Error> { + let block_proposer = + &state.validator_registry[state.get_beacon_proposer_index(block.slot, spec)?]; + let proposal = Proposal { slot: block.slot, shard: spec.beacon_chain_shard_number, @@ -243,14 +104,73 @@ pub fn verify_block_signature( let domain = spec.get_domain( block.slot.epoch(spec.slots_per_epoch), Domain::Proposal, - fork, + &state.fork, ); - // Verify that `bls_verify(pubkey=proposer.pubkey, message_hash=signed_root(proposal, - // "signature"), signature=proposal.signature, domain=get_domain(state.fork, - // get_current_epoch(state), DOMAIN_PROPOSAL))`. - proposal - .signature - .verify(&proposal.signed_root()[..], domain, &block_proposer.pubkey) + + verify!( + proposal + .signature + .verify(&proposal.signed_root()[..], domain, &block_proposer.pubkey), + Invalid::BadSignature + ); + + Ok(()) +} + +/// Verifies the `randao_reveal` against the block's proposer pubkey and updates +/// `state.latest_randao_mixes`. +/// +/// Spec v0.4.0 +pub fn process_randao( + state: &mut BeaconState, + block: &BeaconBlock, + spec: &ChainSpec, +) -> Result<(), Error> { + // Let `proposer = state.validator_registry[get_beacon_proposer_index(state, state.slot)]`. + let block_proposer = + &state.validator_registry[state.get_beacon_proposer_index(block.slot, spec)?]; + + // Verify that `bls_verify(pubkey=proposer.pubkey, + // message_hash=hash_tree_root(get_current_epoch(state)), signature=block.randao_reveal, + // domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_RANDAO))`. + verify!( + block.randao_reveal.verify( + &state.current_epoch(spec).hash_tree_root()[..], + spec.get_domain( + block.slot.epoch(spec.slots_per_epoch), + Domain::Randao, + &state.fork + ), + &block_proposer.pubkey + ), + Invalid::BadRandaoSignature + ); + + // Update the state's RANDAO mix with the one revealed in the block. + update_randao(state, &block.randao_reveal, spec)?; + + Ok(()) +} + +/// Update the `state.eth1_data_votes` based upon the `eth1_data` provided. +/// +/// Spec v0.4.0 +pub fn process_eth1_data(state: &mut BeaconState, eth1_data: &Eth1Data) -> Result<(), Error> { + // Either increment the eth1_data vote count, or add a new eth1_data. + let matching_eth1_vote_index = state + .eth1_data_votes + .iter() + .position(|vote| vote.eth1_data == *eth1_data); + if let Some(index) = matching_eth1_vote_index { + state.eth1_data_votes[index].vote_count += 1; + } else { + state.eth1_data_votes.push(Eth1DataVote { + eth1_data: eth1_data.clone(), + vote_count: 1, + }); + } + + Ok(()) } /// Updates the present randao mix. @@ -286,3 +206,179 @@ pub fn update_randao( Err(BeaconStateError::InsufficientRandaoMixes) } } + +/// Validates each `ProposerSlashing` and updates the state, short-circuiting on an invalid object. +/// +/// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns +/// an `Err` describing the invalid object or cause of failure. +/// +/// Spec v0.4.0 +pub fn process_proposer_slashings( + state: &mut BeaconState, + proposer_slashings: &[ProposerSlashing], + spec: &ChainSpec, +) -> Result<(), Error> { + verify!( + proposer_slashings.len() as u64 <= spec.max_proposer_slashings, + Invalid::MaxProposerSlashingsExceeded + ); + for (i, proposer_slashing) in proposer_slashings.iter().enumerate() { + verify_proposer_slashing(proposer_slashing, &state, spec) + .map_err(|e| e.into_with_index(i))?; + state.slash_validator(proposer_slashing.proposer_index as usize, spec)?; + } + + Ok(()) +} + +/// Validates each `AttesterSlsashing` and updates the state, short-circuiting on an invalid object. +/// +/// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns +/// an `Err` describing the invalid object or cause of failure. +/// +/// Spec v0.4.0 +pub fn process_attester_slashings( + state: &mut BeaconState, + attester_slashings: &[AttesterSlashing], + spec: &ChainSpec, +) -> Result<(), Error> { + verify!( + attester_slashings.len() as u64 <= spec.max_attester_slashings, + Invalid::MaxAttesterSlashingsExceed + ); + for (i, attester_slashing) in attester_slashings.iter().enumerate() { + let slashable_indices = verify_attester_slashing(&state, &attester_slashing, spec) + .map_err(|e| e.into_with_index(i))?; + for i in slashable_indices { + state.slash_validator(i as usize, spec)?; + } + } + + Ok(()) +} + +/// Validates each `Attestation` and updates the state, short-circuiting on an invalid object. +/// +/// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns +/// an `Err` describing the invalid object or cause of failure. +/// +/// Spec v0.4.0 +pub fn process_attestations( + state: &mut BeaconState, + attestations: &[Attestation], + spec: &ChainSpec, +) -> Result<(), Error> { + verify!( + attestations.len() as u64 <= spec.max_attestations, + Invalid::MaxAttestationsExceeded + ); + for (i, attestation) in attestations.iter().enumerate() { + // Build the previous epoch cache only if required by an attestation. + if attestation.data.slot.epoch(spec.slots_per_epoch) == state.previous_epoch(spec) { + state.build_epoch_cache(RelativeEpoch::Previous, spec)?; + } + + validate_attestation(state, attestation, spec).map_err(|e| e.into_with_index(i))?; + + let pending_attestation = PendingAttestation { + data: attestation.data.clone(), + aggregation_bitfield: attestation.aggregation_bitfield.clone(), + custody_bitfield: attestation.custody_bitfield.clone(), + inclusion_slot: state.slot, + }; + state.latest_attestations.push(pending_attestation); + } + + Ok(()) +} + +/// Validates each `Deposit` and updates the state, short-circuiting on an invalid object. +/// +/// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns +/// an `Err` describing the invalid object or cause of failure. +/// +/// Spec v0.4.0 +pub fn process_deposits( + state: &mut BeaconState, + deposits: &[Deposit], + spec: &ChainSpec, +) -> Result<(), Error> { + verify!( + deposits.len() as u64 <= spec.max_deposits, + Invalid::MaxDepositsExceeded + ); + for (i, deposit) in deposits.iter().enumerate() { + verify_deposit(state, deposit, spec).map_err(|e| e.into_with_index(i))?; + + state + .process_deposit( + deposit.deposit_data.deposit_input.pubkey.clone(), + deposit.deposit_data.amount, + deposit + .deposit_data + .deposit_input + .proof_of_possession + .clone(), + deposit.deposit_data.deposit_input.withdrawal_credentials, + None, + spec, + ) + .map_err(|_| Error::Invalid(Invalid::DepositProcessingFailed(i)))?; + + state.deposit_index += 1; + } + + Ok(()) +} + +/// Validates each `Exit` and updates the state, short-circuiting on an invalid object. +/// +/// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns +/// an `Err` describing the invalid object or cause of failure. +/// +/// Spec v0.4.0 +pub fn process_exits( + state: &mut BeaconState, + voluntary_exits: &[VoluntaryExit], + spec: &ChainSpec, +) -> Result<(), Error> { + verify!( + voluntary_exits.len() as u64 <= spec.max_voluntary_exits, + Invalid::MaxExitsExceeded + ); + for (i, exit) in voluntary_exits.iter().enumerate() { + verify_exit(&state, exit, spec).map_err(|e| e.into_with_index(i))?; + + state.initiate_validator_exit(exit.validator_index as usize); + } + + Ok(()) +} + +/// Validates each `Transfer` and updates the state, short-circuiting on an invalid object. +/// +/// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns +/// an `Err` describing the invalid object or cause of failure. +/// +/// Spec v0.4.0 +pub fn process_transfers( + state: &mut BeaconState, + transfers: &[Transfer], + spec: &ChainSpec, +) -> Result<(), Error> { + verify!( + transfers.len() as u64 <= spec.max_transfers, + Invalid::MaxTransfersExceed + ); + for (i, transfer) in transfers.iter().enumerate() { + verify_transfer(&state, transfer, spec).map_err(|e| e.into_with_index(i))?; + + let block_proposer = state.get_beacon_proposer_index(state.slot, spec)?; + + state.validator_balances[transfer.from as usize] -= transfer.amount + transfer.fee; + state.validator_balances[transfer.to as usize] += transfer.amount + transfer.fee; + state.validator_balances[block_proposer as usize] += transfer.fee; + } + + Ok(()) +} From 17210faf3a3c9dd6f1266e19941888ecbdf71e6a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 6 Mar 2019 17:14:54 +1100 Subject: [PATCH 118/132] Start reorg of per_epoch_processing --- eth2/state_processing/src/lib.rs | 6 +- .../src/per_block_processing.rs | 3 +- .../src/{ => per_block_processing}/errors.rs | 0 .../validate_attestation.rs | 2 +- .../verify_attester_slashing.rs | 2 +- .../per_block_processing/verify_deposit.rs | 2 +- .../src/per_block_processing/verify_exit.rs | 2 +- .../verify_proposer_slashing.rs | 2 +- .../verify_slashable_attestation.rs | 2 +- .../per_block_processing/verify_transfer.rs | 2 +- ...processable.rs => per_epoch_processing.rs} | 121 +----------------- .../src/per_epoch_processing/errors.rs | 36 ++++++ .../tests.rs | 0 .../src/per_epoch_processing/winning_root.rs | 85 ++++++++++++ 14 files changed, 136 insertions(+), 129 deletions(-) rename eth2/state_processing/src/{ => per_block_processing}/errors.rs (100%) rename eth2/state_processing/src/{epoch_processable.rs => per_epoch_processing.rs} (88%) create mode 100644 eth2/state_processing/src/per_epoch_processing/errors.rs rename eth2/state_processing/src/{epoch_processable => per_epoch_processing}/tests.rs (100%) create mode 100644 eth2/state_processing/src/per_epoch_processing/winning_root.rs diff --git a/eth2/state_processing/src/lib.rs b/eth2/state_processing/src/lib.rs index ec883ddca..a5cc4f26e 100644 --- a/eth2/state_processing/src/lib.rs +++ b/eth2/state_processing/src/lib.rs @@ -1,12 +1,12 @@ #[macro_use] mod macros; + pub mod per_block_processing; -// mod epoch_processable; -pub mod errors; +pub mod per_epoch_processing; // mod slot_processable; -pub use errors::{BlockInvalid, BlockProcessingError}; pub use per_block_processing::{ + errors::{BlockInvalid, BlockProcessingError}, per_block_processing, per_block_processing_without_verifying_block_signature, }; // pub use epoch_processable::{EpochProcessable, Error as EpochProcessingError}; diff --git a/eth2/state_processing/src/per_block_processing.rs b/eth2/state_processing/src/per_block_processing.rs index 573513819..fc3a8204e 100644 --- a/eth2/state_processing/src/per_block_processing.rs +++ b/eth2/state_processing/src/per_block_processing.rs @@ -1,5 +1,5 @@ use self::verify_proposer_slashing::verify_proposer_slashing; -use crate::errors::{BlockInvalid as Invalid, BlockProcessingError as Error, IntoWithIndex}; +use errors::{BlockInvalid as Invalid, BlockProcessingError as Error, IntoWithIndex}; use hashing::hash; use log::debug; use ssz::{ssz_encode, SignedRoot, TreeHash}; @@ -11,6 +11,7 @@ pub use verify_deposit::verify_deposit; pub use verify_exit::verify_exit; pub use verify_transfer::verify_transfer; +pub mod errors; mod validate_attestation; mod verify_attester_slashing; mod verify_deposit; diff --git a/eth2/state_processing/src/errors.rs b/eth2/state_processing/src/per_block_processing/errors.rs similarity index 100% rename from eth2/state_processing/src/errors.rs rename to eth2/state_processing/src/per_block_processing/errors.rs diff --git a/eth2/state_processing/src/per_block_processing/validate_attestation.rs b/eth2/state_processing/src/per_block_processing/validate_attestation.rs index 8749bc5ea..54bd2d332 100644 --- a/eth2/state_processing/src/per_block_processing/validate_attestation.rs +++ b/eth2/state_processing/src/per_block_processing/validate_attestation.rs @@ -1,4 +1,4 @@ -use crate::errors::{AttestationInvalid as Invalid, AttestationValidationError as Error}; +use super::errors::{AttestationInvalid as Invalid, AttestationValidationError as Error}; use ssz::TreeHash; use types::beacon_state::helpers::*; use types::*; diff --git a/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs b/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs index aef09943a..71ac97469 100644 --- a/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs +++ b/eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs @@ -1,5 +1,5 @@ +use super::errors::{AttesterSlashingInvalid as Invalid, AttesterSlashingValidationError as Error}; use super::verify_slashable_attestation::verify_slashable_attestation; -use crate::errors::{AttesterSlashingInvalid as Invalid, AttesterSlashingValidationError as Error}; use types::*; /// Indicates if an `AttesterSlashing` is valid to be included in a block in the current epoch of the given diff --git a/eth2/state_processing/src/per_block_processing/verify_deposit.rs b/eth2/state_processing/src/per_block_processing/verify_deposit.rs index 59a2a66c4..a1731f1a1 100644 --- a/eth2/state_processing/src/per_block_processing/verify_deposit.rs +++ b/eth2/state_processing/src/per_block_processing/verify_deposit.rs @@ -1,4 +1,4 @@ -use crate::errors::{DepositInvalid as Invalid, DepositValidationError as Error}; +use super::errors::{DepositInvalid as Invalid, DepositValidationError as Error}; use types::*; /// Indicates if a `Deposit` is valid to be included in a block in the current epoch of the given diff --git a/eth2/state_processing/src/per_block_processing/verify_exit.rs b/eth2/state_processing/src/per_block_processing/verify_exit.rs index 34c55c55e..408b77077 100644 --- a/eth2/state_processing/src/per_block_processing/verify_exit.rs +++ b/eth2/state_processing/src/per_block_processing/verify_exit.rs @@ -1,4 +1,4 @@ -use crate::errors::{ExitInvalid as Invalid, ExitValidationError as Error}; +use super::errors::{ExitInvalid as Invalid, ExitValidationError as Error}; use ssz::SignedRoot; use types::*; diff --git a/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs b/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs index c4e51b1a8..0350255ec 100644 --- a/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs +++ b/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs @@ -1,4 +1,4 @@ -use crate::errors::{ProposerSlashingInvalid as Invalid, ProposerSlashingValidationError as Error}; +use super::errors::{ProposerSlashingInvalid as Invalid, ProposerSlashingValidationError as Error}; use ssz::SignedRoot; use types::*; diff --git a/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs b/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs index 0b948ad89..f0d371043 100644 --- a/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs +++ b/eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs @@ -1,4 +1,4 @@ -use crate::errors::{ +use super::errors::{ SlashableAttestationInvalid as Invalid, SlashableAttestationValidationError as Error, }; use ssz::TreeHash; diff --git a/eth2/state_processing/src/per_block_processing/verify_transfer.rs b/eth2/state_processing/src/per_block_processing/verify_transfer.rs index 220f5e496..cd7bcb42c 100644 --- a/eth2/state_processing/src/per_block_processing/verify_transfer.rs +++ b/eth2/state_processing/src/per_block_processing/verify_transfer.rs @@ -1,4 +1,4 @@ -use crate::errors::{TransferInvalid as Invalid, TransferValidationError as Error}; +use super::errors::{TransferInvalid as Invalid, TransferValidationError as Error}; use types::*; /// Indicates if a `Transfer` is valid to be included in a block in the current epoch of the given diff --git a/eth2/state_processing/src/epoch_processable.rs b/eth2/state_processing/src/per_epoch_processing.rs similarity index 88% rename from eth2/state_processing/src/epoch_processable.rs rename to eth2/state_processing/src/per_epoch_processing.rs index 0d6ca8038..4c66d8ddc 100644 --- a/eth2/state_processing/src/epoch_processable.rs +++ b/eth2/state_processing/src/per_epoch_processing.rs @@ -1,3 +1,4 @@ +use errors::EpochProcessingError as Error; use integer_sqrt::IntegerSquareRoot; use log::{debug, trace}; use rayon::prelude::*; @@ -9,7 +10,9 @@ use types::{ Crosslink, Epoch, Hash256, InclusionError, PendingAttestation, RelativeEpoch, }; +mod errors; mod tests; +mod winning_root; macro_rules! safe_add_assign { ($a: expr, $b: expr) => { @@ -22,31 +25,6 @@ macro_rules! safe_sub_assign { }; } -#[derive(Debug, PartialEq)] -pub enum Error { - UnableToDetermineProducer, - NoBlockRoots, - BaseRewardQuotientIsZero, - NoRandaoSeed, - BeaconStateError(BeaconStateError), - InclusionError(InclusionError), - WinningRootError(WinningRootError), -} - -#[derive(Debug, PartialEq)] -pub enum WinningRootError { - NoWinningRoot, - BeaconStateError(BeaconStateError), -} - -#[derive(Clone)] -pub struct WinningRoot { - pub shard_block_root: Hash256, - pub attesting_validator_indices: Vec, - pub total_balance: u64, - pub total_attesting_balance: u64, -} - pub trait EpochProcessable { fn per_epoch_processing(&mut self, spec: &ChainSpec) -> Result<(), Error>; } @@ -628,96 +606,3 @@ impl EpochProcessable for BeaconState { fn hash_tree_root(input: Vec) -> Hash256 { Hash256::from(&input.hash_tree_root()[..]) } - -fn winning_root( - state: &BeaconState, - shard: u64, - current_epoch_attestations: &[&PendingAttestation], - previous_epoch_attestations: &[&PendingAttestation], - spec: &ChainSpec, -) -> Result { - let mut attestations = current_epoch_attestations.to_vec(); - attestations.append(&mut previous_epoch_attestations.to_vec()); - - let mut candidates: HashMap = HashMap::new(); - - let mut highest_seen_balance = 0; - - for a in &attestations { - if a.data.shard != shard { - continue; - } - - let shard_block_root = &a.data.shard_block_root; - - if candidates.contains_key(shard_block_root) { - continue; - } - - let attesting_validator_indices = attestations - .iter() - .try_fold::<_, _, Result<_, BeaconStateError>>(vec![], |mut acc, a| { - if (a.data.shard == shard) && (a.data.shard_block_root == *shard_block_root) { - acc.append(&mut state.get_attestation_participants( - &a.data, - &a.aggregation_bitfield, - spec, - )?); - } - Ok(acc) - })?; - - let total_balance: u64 = attesting_validator_indices - .iter() - .fold(0, |acc, i| acc + state.get_effective_balance(*i, spec)); - - let total_attesting_balance: u64 = attesting_validator_indices - .iter() - .fold(0, |acc, i| acc + state.get_effective_balance(*i, spec)); - - if total_attesting_balance > highest_seen_balance { - highest_seen_balance = total_attesting_balance; - } - - let candidate_root = WinningRoot { - shard_block_root: *shard_block_root, - attesting_validator_indices, - total_attesting_balance, - total_balance, - }; - - candidates.insert(*shard_block_root, candidate_root); - } - - Ok(candidates - .iter() - .filter_map(|(_hash, candidate)| { - if candidate.total_attesting_balance == highest_seen_balance { - Some(candidate) - } else { - None - } - }) - .min_by_key(|candidate| candidate.shard_block_root) - .ok_or_else(|| WinningRootError::NoWinningRoot)? - // TODO: avoid clone. - .clone()) -} - -impl From for Error { - fn from(e: InclusionError) -> Error { - Error::InclusionError(e) - } -} - -impl From for Error { - fn from(e: BeaconStateError) -> Error { - Error::BeaconStateError(e) - } -} - -impl From for WinningRootError { - fn from(e: BeaconStateError) -> WinningRootError { - WinningRootError::BeaconStateError(e) - } -} diff --git a/eth2/state_processing/src/per_epoch_processing/errors.rs b/eth2/state_processing/src/per_epoch_processing/errors.rs new file mode 100644 index 000000000..0f7063e3b --- /dev/null +++ b/eth2/state_processing/src/per_epoch_processing/errors.rs @@ -0,0 +1,36 @@ +use types::*; + +#[derive(Debug, PartialEq)] +pub enum WinningRootError { + NoWinningRoot, + BeaconStateError(BeaconStateError), +} + +#[derive(Debug, PartialEq)] +pub enum EpochProcessingError { + UnableToDetermineProducer, + NoBlockRoots, + BaseRewardQuotientIsZero, + NoRandaoSeed, + BeaconStateError(BeaconStateError), + InclusionError(InclusionError), + WinningRootError(WinningRootError), +} + +impl From for EpochProcessingError { + fn from(e: InclusionError) -> EpochProcessingError { + EpochProcessingError::InclusionError(e) + } +} + +impl From for EpochProcessingError { + fn from(e: BeaconStateError) -> EpochProcessingError { + EpochProcessingError::BeaconStateError(e) + } +} + +impl From for WinningRootError { + fn from(e: BeaconStateError) -> WinningRootError { + WinningRootError::BeaconStateError(e) + } +} diff --git a/eth2/state_processing/src/epoch_processable/tests.rs b/eth2/state_processing/src/per_epoch_processing/tests.rs similarity index 100% rename from eth2/state_processing/src/epoch_processable/tests.rs rename to eth2/state_processing/src/per_epoch_processing/tests.rs diff --git a/eth2/state_processing/src/per_epoch_processing/winning_root.rs b/eth2/state_processing/src/per_epoch_processing/winning_root.rs new file mode 100644 index 000000000..5b1e5925f --- /dev/null +++ b/eth2/state_processing/src/per_epoch_processing/winning_root.rs @@ -0,0 +1,85 @@ +use super::WinningRootError; +use types::*; + +#[derive(Clone)] +pub struct WinningRoot { + pub shard_block_root: Hash256, + pub attesting_validator_indices: Vec, + pub total_balance: u64, + pub total_attesting_balance: u64, +} + +fn winning_root( + state: &BeaconState, + shard: u64, + current_epoch_attestations: &[&PendingAttestation], + previous_epoch_attestations: &[&PendingAttestation], + spec: &ChainSpec, +) -> Result { + let mut attestations = current_epoch_attestations.to_vec(); + attestations.append(&mut previous_epoch_attestations.to_vec()); + + let mut candidates: HashMap = HashMap::new(); + + let mut highest_seen_balance = 0; + + for a in &attestations { + if a.data.shard != shard { + continue; + } + + let shard_block_root = &a.data.shard_block_root; + + if candidates.contains_key(shard_block_root) { + continue; + } + + let attesting_validator_indices = attestations + .iter() + .try_fold::<_, _, Result<_, BeaconStateError>>(vec![], |mut acc, a| { + if (a.data.shard == shard) && (a.data.shard_block_root == *shard_block_root) { + acc.append(&mut state.get_attestation_participants( + &a.data, + &a.aggregation_bitfield, + spec, + )?); + } + Ok(acc) + })?; + + let total_balance: u64 = attesting_validator_indices + .iter() + .fold(0, |acc, i| acc + state.get_effective_balance(*i, spec)); + + let total_attesting_balance: u64 = attesting_validator_indices + .iter() + .fold(0, |acc, i| acc + state.get_effective_balance(*i, spec)); + + if total_attesting_balance > highest_seen_balance { + highest_seen_balance = total_attesting_balance; + } + + let candidate_root = WinningRoot { + shard_block_root: *shard_block_root, + attesting_validator_indices, + total_attesting_balance, + total_balance, + }; + + candidates.insert(*shard_block_root, candidate_root); + } + + Ok(candidates + .iter() + .filter_map(|(_hash, candidate)| { + if candidate.total_attesting_balance == highest_seen_balance { + Some(candidate) + } else { + None + } + }) + .min_by_key(|candidate| candidate.shard_block_root) + .ok_or_else(|| WinningRootError::NoWinningRoot)? + // TODO: avoid clone. + .clone()) +} From 8a25fd48cfe7740a77c7837b59407035bf909da2 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 6 Mar 2019 18:57:41 +1100 Subject: [PATCH 119/132] Fix per_epoch_processing so it compiles again --- eth2/state_processing/src/lib.rs | 1 + .../src/per_epoch_processing.rs | 1063 ++++++++--------- .../per_epoch_processing/grouped_attesters.rs | 98 ++ .../src/per_epoch_processing/tests.rs | 4 +- .../src/per_epoch_processing/winning_root.rs | 19 +- 5 files changed, 592 insertions(+), 593 deletions(-) create mode 100644 eth2/state_processing/src/per_epoch_processing/grouped_attesters.rs diff --git a/eth2/state_processing/src/lib.rs b/eth2/state_processing/src/lib.rs index a5cc4f26e..44c0e28a0 100644 --- a/eth2/state_processing/src/lib.rs +++ b/eth2/state_processing/src/lib.rs @@ -9,5 +9,6 @@ pub use per_block_processing::{ errors::{BlockInvalid, BlockProcessingError}, per_block_processing, per_block_processing_without_verifying_block_signature, }; +pub use per_epoch_processing::{errors::EpochProcessingError, per_epoch_processing}; // pub use epoch_processable::{EpochProcessable, Error as EpochProcessingError}; // pub use slot_processable::{Error as SlotProcessingError, SlotProcessable}; diff --git a/eth2/state_processing/src/per_epoch_processing.rs b/eth2/state_processing/src/per_epoch_processing.rs index 4c66d8ddc..a54518364 100644 --- a/eth2/state_processing/src/per_epoch_processing.rs +++ b/eth2/state_processing/src/per_epoch_processing.rs @@ -1,16 +1,16 @@ -use errors::EpochProcessingError as Error; +use errors::{EpochProcessingError as Error, WinningRootError}; +use grouped_attesters::GroupedAttesters; use integer_sqrt::IntegerSquareRoot; use log::{debug, trace}; use rayon::prelude::*; use ssz::TreeHash; use std::collections::{HashMap, HashSet}; use std::iter::FromIterator; -use types::{ - validator_registry::get_active_validator_indices, BeaconState, BeaconStateError, ChainSpec, - Crosslink, Epoch, Hash256, InclusionError, PendingAttestation, RelativeEpoch, -}; +use types::{validator_registry::get_active_validator_indices, *}; +use winning_root::{winning_root, WinningRoot}; -mod errors; +pub mod errors; +mod grouped_attesters; mod tests; mod winning_root; @@ -25,584 +25,483 @@ macro_rules! safe_sub_assign { }; } -pub trait EpochProcessable { - fn per_epoch_processing(&mut self, spec: &ChainSpec) -> Result<(), Error>; -} - -impl EpochProcessable for BeaconState { - // Cyclomatic complexity is ignored. It would be ideal to split this function apart, however it - // remains monolithic to allow for easier spec updates. Once the spec is more stable we can - // optimise. - #[allow(clippy::cyclomatic_complexity)] - fn per_epoch_processing(&mut self, spec: &ChainSpec) -> Result<(), Error> { - let current_epoch = self.current_epoch(spec); - let previous_epoch = self.previous_epoch(spec); - let next_epoch = self.next_epoch(spec); - - debug!( - "Starting per-epoch processing on epoch {}...", - self.current_epoch(spec) - ); - - // Ensure all of the caches are built. - self.build_epoch_cache(RelativeEpoch::Previous, spec)?; - self.build_epoch_cache(RelativeEpoch::Current, spec)?; - self.build_epoch_cache(RelativeEpoch::Next, spec)?; - - /* - * Validators attesting during the current epoch. - */ - let active_validator_indices = get_active_validator_indices( - &self.validator_registry, - self.slot.epoch(spec.slots_per_epoch), - ); - let current_total_balance = self.get_total_balance(&active_validator_indices[..], spec); - - trace!( - "{} validators with a total balance of {} wei.", - active_validator_indices.len(), - current_total_balance - ); - - let current_epoch_attestations: Vec<&PendingAttestation> = self - .latest_attestations - .par_iter() - .filter(|a| { - (a.data.slot / spec.slots_per_epoch).epoch(spec.slots_per_epoch) - == self.current_epoch(spec) - }) - .collect(); - - trace!( - "Current epoch attestations: {}", - current_epoch_attestations.len() - ); - - let current_epoch_boundary_attestations: Vec<&PendingAttestation> = - current_epoch_attestations - .par_iter() - .filter( - |a| match self.get_block_root(self.current_epoch_start_slot(spec), spec) { - Some(block_root) => { - (a.data.epoch_boundary_root == *block_root) - && (a.data.justified_epoch == self.justified_epoch) - } - None => unreachable!(), - }, - ) - .cloned() - .collect(); - - let current_epoch_boundary_attester_indices = self - .get_attestation_participants_union(¤t_epoch_boundary_attestations[..], spec)?; - let current_epoch_boundary_attesting_balance = - self.get_total_balance(¤t_epoch_boundary_attester_indices[..], spec); - - trace!( - "Current epoch boundary attesters: {}", - current_epoch_boundary_attester_indices.len() - ); - - /* - * Validators attesting during the previous epoch - */ - - /* - * Validators that made an attestation during the previous epoch - */ - let previous_epoch_attestations: Vec<&PendingAttestation> = self - .latest_attestations - .par_iter() - .filter(|a| { - //TODO: ensure these saturating subs are correct. - (a.data.slot / spec.slots_per_epoch).epoch(spec.slots_per_epoch) - == self.previous_epoch(spec) - }) - .collect(); - - debug!( - "previous epoch attestations: {}", - previous_epoch_attestations.len() - ); - - let previous_epoch_attester_indices = - self.get_attestation_participants_union(&previous_epoch_attestations[..], spec)?; - let previous_total_balance = self.get_total_balance( - &get_active_validator_indices(&self.validator_registry, previous_epoch), - spec, - ); - - /* - * Validators targetting the previous justified slot - */ - let previous_epoch_justified_attestations: Vec<&PendingAttestation> = { - let mut a: Vec<&PendingAttestation> = current_epoch_attestations - .iter() - .filter(|a| a.data.justified_epoch == self.previous_justified_epoch) - .cloned() - .collect(); - let mut b: Vec<&PendingAttestation> = previous_epoch_attestations - .iter() - .filter(|a| a.data.justified_epoch == self.previous_justified_epoch) - .cloned() - .collect(); - a.append(&mut b); - a - }; - - let previous_epoch_justified_attester_indices = self - .get_attestation_participants_union(&previous_epoch_justified_attestations[..], spec)?; - let previous_epoch_justified_attesting_balance = - self.get_total_balance(&previous_epoch_justified_attester_indices[..], spec); - - /* - * Validators justifying the epoch boundary block at the start of the previous epoch - */ - let previous_epoch_boundary_attestations: Vec<&PendingAttestation> = - previous_epoch_justified_attestations - .iter() - .filter( - |a| match self.get_block_root(self.previous_epoch_start_slot(spec), spec) { - Some(block_root) => a.data.epoch_boundary_root == *block_root, - None => unreachable!(), - }, - ) - .cloned() - .collect(); - - let previous_epoch_boundary_attester_indices = self - .get_attestation_participants_union(&previous_epoch_boundary_attestations[..], spec)?; - let previous_epoch_boundary_attesting_balance = - self.get_total_balance(&previous_epoch_boundary_attester_indices[..], spec); - - /* - * Validators attesting to the expected beacon chain head during the previous epoch. - */ - let previous_epoch_head_attestations: Vec<&PendingAttestation> = - previous_epoch_attestations - .iter() - .filter(|a| match self.get_block_root(a.data.slot, spec) { - Some(block_root) => a.data.beacon_block_root == *block_root, - None => unreachable!(), - }) - .cloned() - .collect(); - - let previous_epoch_head_attester_indices = - self.get_attestation_participants_union(&previous_epoch_head_attestations[..], spec)?; - let previous_epoch_head_attesting_balance = - self.get_total_balance(&previous_epoch_head_attester_indices[..], spec); - - debug!( - "previous_epoch_head_attester_balance of {} wei.", - previous_epoch_head_attesting_balance - ); - - /* - * Eth1 Data - */ - if self.next_epoch(spec) % spec.eth1_data_voting_period == 0 { - for eth1_data_vote in &self.eth1_data_votes { - if eth1_data_vote.vote_count * 2 > spec.eth1_data_voting_period { - self.latest_eth1_data = eth1_data_vote.eth1_data.clone(); - } - } - self.eth1_data_votes = vec![]; - } - - /* - * Justification - */ - - let mut new_justified_epoch = self.justified_epoch; - self.justification_bitfield <<= 1; - - // If > 2/3 of the total balance attested to the previous epoch boundary - // - // - Set the 2nd bit of the bitfield. - // - Set the previous epoch to be justified. - if (3 * previous_epoch_boundary_attesting_balance) >= (2 * current_total_balance) { - self.justification_bitfield |= 2; - new_justified_epoch = previous_epoch; - trace!(">= 2/3 voted for previous epoch boundary"); - } - // If > 2/3 of the total balance attested to the previous epoch boundary - // - // - Set the 1st bit of the bitfield. - // - Set the current epoch to be justified. - if (3 * current_epoch_boundary_attesting_balance) >= (2 * current_total_balance) { - self.justification_bitfield |= 1; - new_justified_epoch = current_epoch; - trace!(">= 2/3 voted for current epoch boundary"); - } - - // If: - // - // - All three epochs prior to this epoch have been justified. - // - The previous justified justified epoch was three epochs ago. - // - // Then, set the finalized epoch to be three epochs ago. - if ((self.justification_bitfield >> 1) % 8 == 0b111) - & (self.previous_justified_epoch == previous_epoch - 2) - { - self.finalized_epoch = self.previous_justified_epoch; - trace!("epoch - 3 was finalized (1st condition)."); - } - // If: - // - // - Both two epochs prior to this epoch have been justified. - // - The previous justified epoch was two epochs ago. - // - // Then, set the finalized epoch to two epochs ago. - if ((self.justification_bitfield >> 1) % 4 == 0b11) - & (self.previous_justified_epoch == previous_epoch - 1) - { - self.finalized_epoch = self.previous_justified_epoch; - trace!("epoch - 2 was finalized (2nd condition)."); - } - // If: - // - // - This epoch and the two prior have been justified. - // - The presently justified epoch was two epochs ago. - // - // Then, set the finalized epoch to two epochs ago. - if (self.justification_bitfield % 8 == 0b111) & (self.justified_epoch == previous_epoch - 1) - { - self.finalized_epoch = self.justified_epoch; - trace!("epoch - 2 was finalized (3rd condition)."); - } - // If: - // - // - This epoch and the epoch prior to it have been justified. - // - Set the previous epoch to be justified. - // - // Then, set the finalized epoch to be the previous epoch. - if (self.justification_bitfield % 4 == 0b11) & (self.justified_epoch == previous_epoch) { - self.finalized_epoch = self.justified_epoch; - trace!("epoch - 1 was finalized (4th condition)."); - } - - self.previous_justified_epoch = self.justified_epoch; - self.justified_epoch = new_justified_epoch; - - debug!( - "Finalized epoch {}, justified epoch {}.", - self.finalized_epoch, self.justified_epoch - ); - - /* - * Crosslinks - */ - - // Cached for later lookups. - let mut winning_root_for_shards: HashMap> = - HashMap::new(); - - // for slot in self.slot.saturating_sub(2 * spec.slots_per_epoch)..self.slot { - for slot in self.previous_epoch(spec).slot_iter(spec.slots_per_epoch) { - trace!( - "Finding winning root for slot: {} (epoch: {})", - slot, - slot.epoch(spec.slots_per_epoch) - ); - - // Clone is used to remove the borrow. It becomes an issue later when trying to mutate - // `self.balances`. - let crosslink_committees_at_slot = - self.get_crosslink_committees_at_slot(slot, spec)?.clone(); - - for (crosslink_committee, shard) in crosslink_committees_at_slot { - let shard = shard as u64; - - let winning_root = winning_root( - self, - shard, - ¤t_epoch_attestations, - &previous_epoch_attestations, - spec, - ); - - if let Ok(winning_root) = &winning_root { - let total_committee_balance = - self.get_total_balance(&crosslink_committee[..], spec); - - if (3 * winning_root.total_attesting_balance) >= (2 * total_committee_balance) { - self.latest_crosslinks[shard as usize] = Crosslink { - epoch: current_epoch, - shard_block_root: winning_root.shard_block_root, - } - } - } - winning_root_for_shards.insert(shard, winning_root); - } - } - - trace!( - "Found {} winning shard roots.", - winning_root_for_shards.len() - ); - - /* - * Rewards and Penalities - */ - let base_reward_quotient = - previous_total_balance.integer_sqrt() / spec.base_reward_quotient; - if base_reward_quotient == 0 { - return Err(Error::BaseRewardQuotientIsZero); - } - - /* - * Justification and finalization - */ - let epochs_since_finality = next_epoch - self.finalized_epoch; - - let previous_epoch_justified_attester_indices_hashset: HashSet = - HashSet::from_iter(previous_epoch_justified_attester_indices.iter().cloned()); - let previous_epoch_boundary_attester_indices_hashset: HashSet = - HashSet::from_iter(previous_epoch_boundary_attester_indices.iter().cloned()); - let previous_epoch_head_attester_indices_hashset: HashSet = - HashSet::from_iter(previous_epoch_head_attester_indices.iter().cloned()); - let previous_epoch_attester_indices_hashset: HashSet = - HashSet::from_iter(previous_epoch_attester_indices.iter().cloned()); - let active_validator_indices_hashset: HashSet = - HashSet::from_iter(active_validator_indices.iter().cloned()); - - debug!("previous epoch justified attesters: {}, previous epoch boundary attesters: {}, previous epoch head attesters: {}, previous epoch attesters: {}", previous_epoch_justified_attester_indices.len(), previous_epoch_boundary_attester_indices.len(), previous_epoch_head_attester_indices.len(), previous_epoch_attester_indices.len()); - - debug!("{} epochs since finality.", epochs_since_finality); - - if epochs_since_finality <= 4 { - for index in 0..self.validator_balances.len() { - let base_reward = self.base_reward(index, base_reward_quotient, spec); - - if previous_epoch_justified_attester_indices_hashset.contains(&index) { - safe_add_assign!( - self.validator_balances[index], - base_reward * previous_epoch_justified_attesting_balance - / previous_total_balance - ); - } else if active_validator_indices_hashset.contains(&index) { - safe_sub_assign!(self.validator_balances[index], base_reward); - } - - if previous_epoch_boundary_attester_indices_hashset.contains(&index) { - safe_add_assign!( - self.validator_balances[index], - base_reward * previous_epoch_boundary_attesting_balance - / previous_total_balance - ); - } else if active_validator_indices_hashset.contains(&index) { - safe_sub_assign!(self.validator_balances[index], base_reward); - } - - if previous_epoch_head_attester_indices_hashset.contains(&index) { - safe_add_assign!( - self.validator_balances[index], - base_reward * previous_epoch_head_attesting_balance - / previous_total_balance - ); - } else if active_validator_indices_hashset.contains(&index) { - safe_sub_assign!(self.validator_balances[index], base_reward); - } - } - - for index in previous_epoch_attester_indices { - let base_reward = self.base_reward(index, base_reward_quotient, spec); - let inclusion_distance = - self.inclusion_distance(&previous_epoch_attestations, index, spec)?; - - safe_add_assign!( - self.validator_balances[index], - base_reward * spec.min_attestation_inclusion_delay / inclusion_distance - ) - } - } else { - for index in 0..self.validator_balances.len() { - let inactivity_penalty = self.inactivity_penalty( - index, - epochs_since_finality, - base_reward_quotient, - spec, - ); - if active_validator_indices_hashset.contains(&index) { - if !previous_epoch_justified_attester_indices_hashset.contains(&index) { - safe_sub_assign!(self.validator_balances[index], inactivity_penalty); - } - if !previous_epoch_boundary_attester_indices_hashset.contains(&index) { - safe_sub_assign!(self.validator_balances[index], inactivity_penalty); - } - if !previous_epoch_head_attester_indices_hashset.contains(&index) { - safe_sub_assign!(self.validator_balances[index], inactivity_penalty); - } - - if self.validator_registry[index].penalized_epoch <= current_epoch { - let base_reward = self.base_reward(index, base_reward_quotient, spec); - safe_sub_assign!( - self.validator_balances[index], - 2 * inactivity_penalty + base_reward - ); - } - } - } - - for index in previous_epoch_attester_indices { - let base_reward = self.base_reward(index, base_reward_quotient, spec); - let inclusion_distance = - self.inclusion_distance(&previous_epoch_attestations, index, spec)?; - - safe_sub_assign!( - self.validator_balances[index], - base_reward - - base_reward * spec.min_attestation_inclusion_delay / inclusion_distance - ); - } - } - - trace!("Processed validator justification and finalization rewards/penalities."); - - /* - * Attestation inclusion - */ - for &index in &previous_epoch_attester_indices_hashset { - let inclusion_slot = - self.inclusion_slot(&previous_epoch_attestations[..], index, spec)?; - let proposer_index = self - .get_beacon_proposer_index(inclusion_slot, spec) - .map_err(|_| Error::UnableToDetermineProducer)?; - let base_reward = self.base_reward(proposer_index, base_reward_quotient, spec); - safe_add_assign!( - self.validator_balances[proposer_index], - base_reward / spec.includer_reward_quotient - ); - } - - trace!( - "Previous epoch attesters: {}.", - previous_epoch_attester_indices_hashset.len() - ); - - /* - * Crosslinks - */ - for slot in self.previous_epoch(spec).slot_iter(spec.slots_per_epoch) { - // Clone is used to remove the borrow. It becomes an issue later when trying to mutate - // `self.balances`. - let crosslink_committees_at_slot = - self.get_crosslink_committees_at_slot(slot, spec)?.clone(); - - for (_crosslink_committee, shard) in crosslink_committees_at_slot { - let shard = shard as u64; - - if let Some(Ok(winning_root)) = winning_root_for_shards.get(&shard) { - // TODO: remove the map. - let attesting_validator_indices: HashSet = HashSet::from_iter( - winning_root.attesting_validator_indices.iter().cloned(), - ); - - for index in 0..self.validator_balances.len() { - let base_reward = self.base_reward(index, base_reward_quotient, spec); - - if attesting_validator_indices.contains(&index) { - safe_add_assign!( - self.validator_balances[index], - base_reward * winning_root.total_attesting_balance - / winning_root.total_balance - ); - } else { - safe_sub_assign!(self.validator_balances[index], base_reward); - } - } - - for index in &winning_root.attesting_validator_indices { - let base_reward = self.base_reward(*index, base_reward_quotient, spec); - safe_add_assign!( - self.validator_balances[*index], - base_reward * winning_root.total_attesting_balance - / winning_root.total_balance - ); - } - } - } - } - - /* - * Ejections - */ - self.process_ejections(spec); - - /* - * Validator Registry - */ - self.previous_shuffling_epoch = self.current_shuffling_epoch; - self.previous_epoch_start_shard = self.current_epoch_start_shard; - - debug!( - "setting previous_shuffling_seed to : {}", - self.current_shuffling_seed - ); - - self.previous_shuffling_seed = self.current_shuffling_seed; - - let should_update_validator_registy = if self.finalized_epoch - > self.validator_registry_update_epoch - { - (0..self.get_current_epoch_committee_count(spec)).all(|i| { - let shard = (self.current_epoch_start_shard + i as u64) % spec.shard_count; - self.latest_crosslinks[shard as usize].epoch > self.validator_registry_update_epoch - }) - } else { - false - }; - - if should_update_validator_registy { - trace!("updating validator registry."); - self.update_validator_registry(spec); - - self.current_shuffling_epoch = next_epoch; - self.current_epoch_start_shard = (self.current_epoch_start_shard - + self.get_current_epoch_committee_count(spec) as u64) - % spec.shard_count; - self.current_shuffling_seed = self.generate_seed(self.current_shuffling_epoch, spec)? - } else { - trace!("not updating validator registry."); - let epochs_since_last_registry_update = - current_epoch - self.validator_registry_update_epoch; - if (epochs_since_last_registry_update > 1) - & epochs_since_last_registry_update.is_power_of_two() - { - self.current_shuffling_epoch = next_epoch; - self.current_shuffling_seed = - self.generate_seed(self.current_shuffling_epoch, spec)? - } - } - - self.process_penalties_and_exits(spec); - - self.latest_index_roots[(next_epoch.as_usize() + spec.entry_exit_delay as usize) - % spec.latest_index_roots_length] = hash_tree_root(get_active_validator_indices( - &self.validator_registry, - next_epoch + Epoch::from(spec.entry_exit_delay), - )); - self.latest_penalized_balances[next_epoch.as_usize() % spec.latest_penalized_exit_length] = - self.latest_penalized_balances - [current_epoch.as_usize() % spec.latest_penalized_exit_length]; - self.latest_randao_mixes[next_epoch.as_usize() % spec.latest_randao_mixes_length] = self - .get_randao_mix(current_epoch, spec) - .and_then(|x| Some(*x)) - .ok_or_else(|| Error::NoRandaoSeed)?; - self.latest_attestations = self - .latest_attestations - .iter() - .filter(|a| a.data.slot.epoch(spec.slots_per_epoch) >= current_epoch) - .cloned() - .collect(); - - debug!("Epoch transition complete."); - - Ok(()) - } +pub fn per_epoch_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> { + let current_epoch = state.current_epoch(spec); + let previous_epoch = state.previous_epoch(spec); + let next_epoch = state.next_epoch(spec); + + debug!( + "Starting per-epoch processing on epoch {}...", + state.current_epoch(spec) + ); + + // Ensure all of the caches are built. + state.build_epoch_cache(RelativeEpoch::Previous, spec)?; + state.build_epoch_cache(RelativeEpoch::Current, spec)?; + state.build_epoch_cache(RelativeEpoch::Next, spec)?; + + let attesters = GroupedAttesters::new(&state, spec)?; + + let active_validator_indices = get_active_validator_indices( + &state.validator_registry, + state.slot.epoch(spec.slots_per_epoch), + ); + + let current_total_balance = state.get_total_balance(&active_validator_indices[..], spec); + let previous_total_balance = state.get_total_balance( + &get_active_validator_indices(&state.validator_registry, previous_epoch)[..], + spec, + ); + + process_eth1_data(state, spec); + + process_justification( + state, + current_total_balance, + previous_total_balance, + attesters.previous_epoch_boundary.balance, + attesters.current_epoch_boundary.balance, + spec, + ); + + // Crosslinks + let winning_root_for_shards = process_crosslinks(state, spec)?; + + // Rewards and Penalities + let active_validator_indices_hashset: HashSet = + HashSet::from_iter(active_validator_indices.iter().cloned()); + process_rewards_and_penalities( + state, + active_validator_indices_hashset, + &attesters, + previous_total_balance, + &winning_root_for_shards, + spec, + )?; + + // Ejections + state.process_ejections(spec); + + // Validator Registry + process_validator_registry(state, spec)?; + + // Final updates + state.latest_active_index_roots[(next_epoch.as_usize() + + spec.activation_exit_delay as usize) + % spec.latest_active_index_roots_length] = hash_tree_root(get_active_validator_indices( + &state.validator_registry, + next_epoch + Epoch::from(spec.activation_exit_delay), + )); + + state.latest_slashed_balances[next_epoch.as_usize() % spec.latest_slashed_exit_length] = + state.latest_slashed_balances[current_epoch.as_usize() % spec.latest_slashed_exit_length]; + state.latest_randao_mixes[next_epoch.as_usize() % spec.latest_randao_mixes_length] = state + .get_randao_mix(current_epoch, spec) + .and_then(|x| Some(*x)) + .ok_or_else(|| Error::NoRandaoSeed)?; + state.latest_attestations = state + .latest_attestations + .iter() + .filter(|a| a.data.slot.epoch(spec.slots_per_epoch) >= current_epoch) + .cloned() + .collect(); + + // Rotate the epoch caches to suit the epoch transition. + state.advance_caches(); + + debug!("Epoch transition complete."); + + Ok(()) } fn hash_tree_root(input: Vec) -> Hash256 { Hash256::from(&input.hash_tree_root()[..]) } + +/// Spec v0.4.0 +fn process_eth1_data(state: &mut BeaconState, spec: &ChainSpec) { + let next_epoch = state.next_epoch(spec); + let voting_period = spec.epochs_per_eth1_voting_period; + + if next_epoch % voting_period == 0 { + for eth1_data_vote in &state.eth1_data_votes { + if eth1_data_vote.vote_count * 2 > voting_period { + state.latest_eth1_data = eth1_data_vote.eth1_data.clone(); + } + } + state.eth1_data_votes = vec![]; + } +} + +/// Spec v0.4.0 +fn process_justification( + state: &mut BeaconState, + current_total_balance: u64, + previous_total_balance: u64, + previous_epoch_boundary_attesting_balance: u64, + current_epoch_boundary_attesting_balance: u64, + spec: &ChainSpec, +) { + let previous_epoch = state.previous_epoch(spec); + let current_epoch = state.current_epoch(spec); + + let mut new_justified_epoch = state.justified_epoch; + state.justification_bitfield <<= 1; + + // If > 2/3 of the total balance attested to the previous epoch boundary + // + // - Set the 2nd bit of the bitfield. + // - Set the previous epoch to be justified. + if (3 * previous_epoch_boundary_attesting_balance) >= (2 * previous_total_balance) { + state.justification_bitfield |= 2; + new_justified_epoch = previous_epoch; + } + // If > 2/3 of the total balance attested to the previous epoch boundary + // + // - Set the 1st bit of the bitfield. + // - Set the current epoch to be justified. + if (3 * current_epoch_boundary_attesting_balance) >= (2 * current_total_balance) { + state.justification_bitfield |= 1; + new_justified_epoch = current_epoch; + } + + // If: + // + // - All three epochs prior to this epoch have been justified. + // - The previous justified justified epoch was three epochs ago. + // + // Then, set the finalized epoch to be three epochs ago. + if ((state.justification_bitfield >> 1) % 8 == 0b111) + & (state.previous_justified_epoch == previous_epoch - 2) + { + state.finalized_epoch = state.previous_justified_epoch; + } + // If: + // + // - Both two epochs prior to this epoch have been justified. + // - The previous justified epoch was two epochs ago. + // + // Then, set the finalized epoch to two epochs ago. + if ((state.justification_bitfield >> 1) % 4 == 0b11) + & (state.previous_justified_epoch == previous_epoch - 1) + { + state.finalized_epoch = state.previous_justified_epoch; + } + // If: + // + // - This epoch and the two prior have been justified. + // - The presently justified epoch was two epochs ago. + // + // Then, set the finalized epoch to two epochs ago. + if (state.justification_bitfield % 8 == 0b111) & (state.justified_epoch == previous_epoch - 1) { + state.finalized_epoch = state.justified_epoch; + } + // If: + // + // - This epoch and the epoch prior to it have been justified. + // - Set the previous epoch to be justified. + // + // Then, set the finalized epoch to be the previous epoch. + if (state.justification_bitfield % 4 == 0b11) & (state.justified_epoch == previous_epoch) { + state.finalized_epoch = state.justified_epoch; + } + + state.previous_justified_epoch = state.justified_epoch; + state.justified_epoch = new_justified_epoch; +} + +pub type WinningRootHashSet = HashMap>; + +fn process_crosslinks( + state: &mut BeaconState, + spec: &ChainSpec, +) -> Result { + let current_epoch_attestations: Vec<&PendingAttestation> = state + .latest_attestations + .par_iter() + .filter(|a| { + (a.data.slot / spec.slots_per_epoch).epoch(spec.slots_per_epoch) + == state.current_epoch(spec) + }) + .collect(); + + let previous_epoch_attestations: Vec<&PendingAttestation> = state + .latest_attestations + .par_iter() + .filter(|a| { + (a.data.slot / spec.slots_per_epoch).epoch(spec.slots_per_epoch) + == state.previous_epoch(spec) + }) + .collect(); + + let mut winning_root_for_shards: HashMap> = + HashMap::new(); + // for slot in state.slot.saturating_sub(2 * spec.slots_per_epoch)..state.slot { + for slot in state.previous_epoch(spec).slot_iter(spec.slots_per_epoch) { + trace!( + "Finding winning root for slot: {} (epoch: {})", + slot, + slot.epoch(spec.slots_per_epoch) + ); + + // Clone is used to remove the borrow. It becomes an issue later when trying to mutate + // `state.balances`. + let crosslink_committees_at_slot = + state.get_crosslink_committees_at_slot(slot, spec)?.clone(); + + for (crosslink_committee, shard) in crosslink_committees_at_slot { + let shard = shard as u64; + + let winning_root = winning_root( + state, + shard, + ¤t_epoch_attestations[..], + &previous_epoch_attestations[..], + spec, + ); + + if let Ok(winning_root) = &winning_root { + let total_committee_balance = state.get_total_balance(&crosslink_committee, spec); + + if (3 * winning_root.total_attesting_balance) >= (2 * total_committee_balance) { + state.latest_crosslinks[shard as usize] = Crosslink { + epoch: state.current_epoch(spec), + crosslink_data_root: winning_root.crosslink_data_root, + } + } + } + winning_root_for_shards.insert(shard, winning_root); + } + } + + Ok(winning_root_for_shards) +} + +fn process_rewards_and_penalities( + state: &mut BeaconState, + active_validator_indices: HashSet, + attesters: &GroupedAttesters, + previous_total_balance: u64, + winning_root_for_shards: &HashMap>, + spec: &ChainSpec, +) -> Result<(), Error> { + let next_epoch = state.next_epoch(spec); + + let previous_epoch_attestations: Vec<&PendingAttestation> = state + .latest_attestations + .par_iter() + .filter(|a| { + (a.data.slot / spec.slots_per_epoch).epoch(spec.slots_per_epoch) + == state.previous_epoch(spec) + }) + .collect(); + + let base_reward_quotient = previous_total_balance.integer_sqrt() / spec.base_reward_quotient; + if base_reward_quotient == 0 { + return Err(Error::BaseRewardQuotientIsZero); + } + + /* + * Justification and finalization + */ + let epochs_since_finality = next_epoch - state.finalized_epoch; + + let active_validator_indices_hashset: HashSet = + HashSet::from_iter(active_validator_indices.iter().cloned()); + + if epochs_since_finality <= 4 { + for index in 0..state.validator_balances.len() { + let base_reward = state.base_reward(index, base_reward_quotient, spec); + + if attesters.previous_epoch.indices.contains(&index) { + safe_add_assign!( + state.validator_balances[index], + base_reward * attesters.previous_epoch.balance / previous_total_balance + ); + } else if active_validator_indices_hashset.contains(&index) { + safe_sub_assign!(state.validator_balances[index], base_reward); + } + + if attesters.previous_epoch_boundary.indices.contains(&index) { + safe_add_assign!( + state.validator_balances[index], + base_reward * attesters.previous_epoch_boundary.balance + / previous_total_balance + ); + } else if active_validator_indices_hashset.contains(&index) { + safe_sub_assign!(state.validator_balances[index], base_reward); + } + + if attesters.previous_epoch_head.indices.contains(&index) { + safe_add_assign!( + state.validator_balances[index], + base_reward * attesters.previous_epoch_head.balance / previous_total_balance + ); + } else if active_validator_indices_hashset.contains(&index) { + safe_sub_assign!(state.validator_balances[index], base_reward); + } + } + + for &index in &attesters.previous_epoch.indices { + let base_reward = state.base_reward(index, base_reward_quotient, spec); + let inclusion_distance = + state.inclusion_distance(&previous_epoch_attestations, index, spec)?; + + safe_add_assign!( + state.validator_balances[index], + base_reward * spec.min_attestation_inclusion_delay / inclusion_distance + ) + } + } else { + for index in 0..state.validator_balances.len() { + let inactivity_penalty = + state.inactivity_penalty(index, epochs_since_finality, base_reward_quotient, spec); + if active_validator_indices_hashset.contains(&index) { + if !attesters.previous_epoch.indices.contains(&index) { + safe_sub_assign!(state.validator_balances[index], inactivity_penalty); + } + if !attesters.previous_epoch_boundary.indices.contains(&index) { + safe_sub_assign!(state.validator_balances[index], inactivity_penalty); + } + if !attesters.previous_epoch_head.indices.contains(&index) { + safe_sub_assign!(state.validator_balances[index], inactivity_penalty); + } + + if state.validator_registry[index].slashed { + let base_reward = state.base_reward(index, base_reward_quotient, spec); + safe_sub_assign!( + state.validator_balances[index], + 2 * inactivity_penalty + base_reward + ); + } + } + } + + for &index in &attesters.previous_epoch.indices { + let base_reward = state.base_reward(index, base_reward_quotient, spec); + let inclusion_distance = + state.inclusion_distance(&previous_epoch_attestations, index, spec)?; + + safe_sub_assign!( + state.validator_balances[index], + base_reward + - base_reward * spec.min_attestation_inclusion_delay / inclusion_distance + ); + } + } + + trace!("Processed validator justification and finalization rewards/penalities."); + + /* + * Attestation inclusion + */ + for &index in &attesters.previous_epoch.indices { + let inclusion_slot = state.inclusion_slot(&previous_epoch_attestations[..], index, spec)?; + let proposer_index = state + .get_beacon_proposer_index(inclusion_slot, spec) + .map_err(|_| Error::UnableToDetermineProducer)?; + let base_reward = state.base_reward(proposer_index, base_reward_quotient, spec); + safe_add_assign!( + state.validator_balances[proposer_index], + base_reward / spec.attestation_inclusion_reward_quotient + ); + } + + /* + * Crosslinks + */ + for slot in state.previous_epoch(spec).slot_iter(spec.slots_per_epoch) { + // Clone is used to remove the borrow. It becomes an issue later when trying to mutate + // `state.balances`. + let crosslink_committees_at_slot = + state.get_crosslink_committees_at_slot(slot, spec)?.clone(); + + for (_crosslink_committee, shard) in crosslink_committees_at_slot { + let shard = shard as u64; + + if let Some(Ok(winning_root)) = winning_root_for_shards.get(&shard) { + // TODO: remove the map. + let attesting_validator_indices: HashSet = + HashSet::from_iter(winning_root.attesting_validator_indices.iter().cloned()); + + for index in 0..state.validator_balances.len() { + let base_reward = state.base_reward(index, base_reward_quotient, spec); + + if attesting_validator_indices.contains(&index) { + safe_add_assign!( + state.validator_balances[index], + base_reward * winning_root.total_attesting_balance + / winning_root.total_balance + ); + } else { + safe_sub_assign!(state.validator_balances[index], base_reward); + } + } + + for index in &winning_root.attesting_validator_indices { + let base_reward = state.base_reward(*index, base_reward_quotient, spec); + safe_add_assign!( + state.validator_balances[*index], + base_reward * winning_root.total_attesting_balance + / winning_root.total_balance + ); + } + } + } + } + + Ok(()) +} + +fn process_validator_registry(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> { + let current_epoch = state.current_epoch(spec); + let next_epoch = state.next_epoch(spec); + + state.previous_shuffling_epoch = state.current_shuffling_epoch; + state.previous_shuffling_start_shard = state.current_shuffling_start_shard; + + debug!( + "setting previous_shuffling_seed to : {}", + state.current_shuffling_seed + ); + + state.previous_shuffling_seed = state.current_shuffling_seed; + + let should_update_validator_registy = if state.finalized_epoch + > state.validator_registry_update_epoch + { + (0..state.get_current_epoch_committee_count(spec)).all(|i| { + let shard = (state.current_shuffling_start_shard + i as u64) % spec.shard_count; + state.latest_crosslinks[shard as usize].epoch > state.validator_registry_update_epoch + }) + } else { + false + }; + + if should_update_validator_registy { + trace!("updating validator registry."); + state.update_validator_registry(spec); + + state.current_shuffling_epoch = next_epoch; + state.current_shuffling_start_shard = (state.current_shuffling_start_shard + + state.get_current_epoch_committee_count(spec) as u64) + % spec.shard_count; + state.current_shuffling_seed = state.generate_seed(state.current_shuffling_epoch, spec)? + } else { + trace!("not updating validator registry."); + let epochs_since_last_registry_update = + current_epoch - state.validator_registry_update_epoch; + if (epochs_since_last_registry_update > 1) + & epochs_since_last_registry_update.is_power_of_two() + { + state.current_shuffling_epoch = next_epoch; + state.current_shuffling_seed = + state.generate_seed(state.current_shuffling_epoch, spec)? + } + } + + state.process_slashings(spec); + state.process_exit_queue(spec); + + Ok(()) +} diff --git a/eth2/state_processing/src/per_epoch_processing/grouped_attesters.rs b/eth2/state_processing/src/per_epoch_processing/grouped_attesters.rs new file mode 100644 index 000000000..02dbc4050 --- /dev/null +++ b/eth2/state_processing/src/per_epoch_processing/grouped_attesters.rs @@ -0,0 +1,98 @@ +use std::collections::HashSet; +use types::*; + +#[derive(Default)] +pub struct Attesters { + pub indices: HashSet, + pub balance: u64, +} + +impl Attesters { + fn add(&mut self, additional_indices: &[usize], additional_balance: u64) { + self.indices.reserve(additional_indices.len()); + for i in additional_indices { + self.indices.insert(*i); + } + self.balance.saturating_add(additional_balance); + } +} + +pub struct GroupedAttesters { + pub current_epoch: Attesters, + pub current_epoch_boundary: Attesters, + pub previous_epoch: Attesters, + pub previous_epoch_boundary: Attesters, + pub previous_epoch_head: Attesters, +} + +impl GroupedAttesters { + pub fn new(state: &BeaconState, spec: &ChainSpec) -> Result { + let mut current_epoch = Attesters::default(); + let mut current_epoch_boundary = Attesters::default(); + let mut previous_epoch = Attesters::default(); + let mut previous_epoch_boundary = Attesters::default(); + let mut previous_epoch_head = Attesters::default(); + + for a in &state.latest_attestations { + let attesting_indices = + state.get_attestation_participants(&a.data, &a.aggregation_bitfield, spec)?; + let attesting_balance = state.get_total_balance(&attesting_indices, spec); + + if is_from_epoch(a, state.current_epoch(spec), spec) { + current_epoch.add(&attesting_indices, attesting_balance); + + if has_common_epoch_boundary_root(a, state, state.current_epoch(spec), spec)? { + current_epoch_boundary.add(&attesting_indices, attesting_balance); + } + } else if is_from_epoch(a, state.previous_epoch(spec), spec) { + previous_epoch.add(&attesting_indices, attesting_balance); + + if has_common_epoch_boundary_root(a, state, state.previous_epoch(spec), spec)? { + previous_epoch_boundary.add(&attesting_indices, attesting_balance); + } + + if has_common_beacon_block_root(a, state, spec)? { + previous_epoch_head.add(&attesting_indices, attesting_balance); + } + } + } + + Ok(Self { + current_epoch, + current_epoch_boundary, + previous_epoch, + previous_epoch_boundary, + previous_epoch_head, + }) + } +} + +fn is_from_epoch(a: &PendingAttestation, epoch: Epoch, spec: &ChainSpec) -> bool { + a.data.slot.epoch(spec.slots_per_epoch) == epoch +} + +fn has_common_epoch_boundary_root( + a: &PendingAttestation, + state: &BeaconState, + epoch: Epoch, + spec: &ChainSpec, +) -> Result { + let slot = epoch.start_slot(spec.slots_per_epoch); + let state_boundary_root = *state + .get_block_root(slot, spec) + .ok_or_else(|| BeaconStateError::InsufficientBlockRoots)?; + + Ok(a.data.epoch_boundary_root == state_boundary_root) +} + +fn has_common_beacon_block_root( + a: &PendingAttestation, + state: &BeaconState, + spec: &ChainSpec, +) -> Result { + let state_block_root = *state + .get_block_root(a.data.slot, spec) + .ok_or_else(|| BeaconStateError::InsufficientBlockRoots)?; + + Ok(a.data.beacon_block_root == state_block_root) +} diff --git a/eth2/state_processing/src/per_epoch_processing/tests.rs b/eth2/state_processing/src/per_epoch_processing/tests.rs index d683d3971..627df858b 100644 --- a/eth2/state_processing/src/per_epoch_processing/tests.rs +++ b/eth2/state_processing/src/per_epoch_processing/tests.rs @@ -1,5 +1,5 @@ #![cfg(test)] -use crate::EpochProcessable; +use crate::per_epoch_processing; use env_logger::{Builder, Env}; use types::beacon_state::BeaconStateBuilder; use types::*; @@ -17,5 +17,5 @@ fn runs_without_error() { let mut state = builder.cloned_state(); let spec = &builder.spec; - state.per_epoch_processing(spec).unwrap(); + per_epoch_processing(&mut state, spec).unwrap(); } diff --git a/eth2/state_processing/src/per_epoch_processing/winning_root.rs b/eth2/state_processing/src/per_epoch_processing/winning_root.rs index 5b1e5925f..c3b650c3d 100644 --- a/eth2/state_processing/src/per_epoch_processing/winning_root.rs +++ b/eth2/state_processing/src/per_epoch_processing/winning_root.rs @@ -1,15 +1,16 @@ -use super::WinningRootError; +use super::errors::WinningRootError; +use std::collections::HashMap; use types::*; #[derive(Clone)] pub struct WinningRoot { - pub shard_block_root: Hash256, + pub crosslink_data_root: Hash256, pub attesting_validator_indices: Vec, pub total_balance: u64, pub total_attesting_balance: u64, } -fn winning_root( +pub fn winning_root( state: &BeaconState, shard: u64, current_epoch_attestations: &[&PendingAttestation], @@ -28,16 +29,16 @@ fn winning_root( continue; } - let shard_block_root = &a.data.shard_block_root; + let crosslink_data_root = &a.data.crosslink_data_root; - if candidates.contains_key(shard_block_root) { + if candidates.contains_key(crosslink_data_root) { continue; } let attesting_validator_indices = attestations .iter() .try_fold::<_, _, Result<_, BeaconStateError>>(vec![], |mut acc, a| { - if (a.data.shard == shard) && (a.data.shard_block_root == *shard_block_root) { + if (a.data.shard == shard) && (a.data.crosslink_data_root == *crosslink_data_root) { acc.append(&mut state.get_attestation_participants( &a.data, &a.aggregation_bitfield, @@ -60,13 +61,13 @@ fn winning_root( } let candidate_root = WinningRoot { - shard_block_root: *shard_block_root, + crosslink_data_root: *crosslink_data_root, attesting_validator_indices, total_attesting_balance, total_balance, }; - candidates.insert(*shard_block_root, candidate_root); + candidates.insert(*crosslink_data_root, candidate_root); } Ok(candidates @@ -78,7 +79,7 @@ fn winning_root( None } }) - .min_by_key(|candidate| candidate.shard_block_root) + .min_by_key(|candidate| candidate.crosslink_data_root) .ok_or_else(|| WinningRootError::NoWinningRoot)? // TODO: avoid clone. .clone()) From 5a225d2983817f8fb7ff40ee11f83e272285e4f3 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 7 Mar 2019 08:37:13 +1100 Subject: [PATCH 120/132] Update per-slot processing to v0.4.0 --- eth2/state_processing/src/lib.rs | 5 +- .../src/per_slot_processing.rs | 58 +++++++++++++++ eth2/state_processing/src/slot_processable.rs | 71 ------------------- 3 files changed, 60 insertions(+), 74 deletions(-) create mode 100644 eth2/state_processing/src/per_slot_processing.rs delete mode 100644 eth2/state_processing/src/slot_processable.rs diff --git a/eth2/state_processing/src/lib.rs b/eth2/state_processing/src/lib.rs index 44c0e28a0..2b30844cb 100644 --- a/eth2/state_processing/src/lib.rs +++ b/eth2/state_processing/src/lib.rs @@ -3,12 +3,11 @@ mod macros; pub mod per_block_processing; pub mod per_epoch_processing; -// mod slot_processable; +pub mod per_slot_processing; pub use per_block_processing::{ errors::{BlockInvalid, BlockProcessingError}, per_block_processing, per_block_processing_without_verifying_block_signature, }; pub use per_epoch_processing::{errors::EpochProcessingError, per_epoch_processing}; -// pub use epoch_processable::{EpochProcessable, Error as EpochProcessingError}; -// pub use slot_processable::{Error as SlotProcessingError, SlotProcessable}; +pub use per_slot_processing::{per_slot_processing, Error as SlotProcessingError}; diff --git a/eth2/state_processing/src/per_slot_processing.rs b/eth2/state_processing/src/per_slot_processing.rs new file mode 100644 index 000000000..0bb405c98 --- /dev/null +++ b/eth2/state_processing/src/per_slot_processing.rs @@ -0,0 +1,58 @@ +use crate::*; +use types::{BeaconState, BeaconStateError, ChainSpec, Hash256}; + +#[derive(Debug, PartialEq)] +pub enum Error { + BeaconStateError(BeaconStateError), + EpochProcessingError(EpochProcessingError), +} + +/// Advances a state forward by one slot, performing per-epoch processing if required. +/// +/// Spec v0.4.0 +pub fn per_slot_processing( + state: &mut BeaconState, + previous_block_root: Hash256, + spec: &ChainSpec, +) -> Result<(), Error> { + if (state.slot + 1) % spec.slots_per_epoch == 0 { + per_epoch_processing(state, spec)?; + state.advance_caches(); + } + + state.slot += 1; + + update_block_roots(state, previous_block_root, spec); + + Ok(()) +} + +/// Updates the state's block roots as per-slot processing is performed. +/// +/// Spec v0.4.0 +pub fn update_block_roots(state: &mut BeaconState, previous_block_root: Hash256, spec: &ChainSpec) { + state.latest_block_roots[(state.slot.as_usize() - 1) % spec.latest_block_roots_length] = + previous_block_root; + + if state.slot.as_usize() % spec.latest_block_roots_length == 0 { + let root = merkle_root(&state.latest_block_roots[..]); + state.batched_block_roots.push(root); + } +} + +fn merkle_root(_input: &[Hash256]) -> Hash256 { + // TODO: implement correctly. + Hash256::zero() +} + +impl From for Error { + fn from(e: BeaconStateError) -> Error { + Error::BeaconStateError(e) + } +} + +impl From for Error { + fn from(e: EpochProcessingError) -> Error { + Error::EpochProcessingError(e) + } +} diff --git a/eth2/state_processing/src/slot_processable.rs b/eth2/state_processing/src/slot_processable.rs deleted file mode 100644 index 8d6506c36..000000000 --- a/eth2/state_processing/src/slot_processable.rs +++ /dev/null @@ -1,71 +0,0 @@ -use crate::{EpochProcessable, EpochProcessingError}; -use types::{BeaconState, BeaconStateError, ChainSpec, Hash256}; - -#[derive(Debug, PartialEq)] -pub enum Error { - BeaconStateError(BeaconStateError), - EpochProcessingError(EpochProcessingError), -} - -pub trait SlotProcessable { - fn per_slot_processing( - &mut self, - previous_block_root: Hash256, - spec: &ChainSpec, - ) -> Result<(), Error>; -} - -impl SlotProcessable for BeaconState -where - BeaconState: EpochProcessable, -{ - fn per_slot_processing( - &mut self, - previous_block_root: Hash256, - spec: &ChainSpec, - ) -> Result<(), Error> { - if (self.slot + 1) % spec.slots_per_epoch == 0 { - self.per_epoch_processing(spec)?; - self.advance_caches(); - } - - self.slot += 1; - - self.latest_randao_mixes[self.slot.as_usize() % spec.latest_randao_mixes_length] = - self.latest_randao_mixes[(self.slot.as_usize() - 1) % spec.latest_randao_mixes_length]; - - // Block roots. - self.latest_block_roots[(self.slot.as_usize() - 1) % spec.latest_block_roots_length] = - previous_block_root; - - if self.slot.as_usize() % spec.latest_block_roots_length == 0 { - let root = merkle_root(&self.latest_block_roots[..]); - self.batched_block_roots.push(root); - } - Ok(()) - } -} - -fn merkle_root(_input: &[Hash256]) -> Hash256 { - Hash256::zero() -} - -impl From for Error { - fn from(e: BeaconStateError) -> Error { - Error::BeaconStateError(e) - } -} - -impl From for Error { - fn from(e: EpochProcessingError) -> Error { - Error::EpochProcessingError(e) - } -} - -#[cfg(test)] -mod tests { - #[test] - fn it_works() { - assert_eq!(2 + 2, 4); - } -} From e6526c9895635fc800b957a126a8ef7cf520ac01 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 7 Mar 2019 11:32:53 +1100 Subject: [PATCH 121/132] Refactor winning root logic --- eth2/state_processing/src/macros.rs | 11 ++ .../src/per_epoch_processing.rs | 178 ++++++++---------- ...{grouped_attesters.rs => attester_sets.rs} | 4 +- .../src/per_epoch_processing/errors.rs | 20 +- .../inclusion_distance.rs | 61 ++++++ .../src/per_epoch_processing/winning_root.rs | 142 ++++++++------ eth2/types/src/beacon_state.rs | 74 -------- eth2/types/src/lib.rs | 4 +- 8 files changed, 253 insertions(+), 241 deletions(-) rename eth2/state_processing/src/per_epoch_processing/{grouped_attesters.rs => attester_sets.rs} (98%) create mode 100644 eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs diff --git a/eth2/state_processing/src/macros.rs b/eth2/state_processing/src/macros.rs index 5a0d0dc11..93a42764b 100644 --- a/eth2/state_processing/src/macros.rs +++ b/eth2/state_processing/src/macros.rs @@ -11,3 +11,14 @@ macro_rules! invalid { return Err(Error::Invalid($result)); }; } + +macro_rules! safe_add_assign { + ($a: expr, $b: expr) => { + $a = $a.saturating_add($b); + }; +} +macro_rules! safe_sub_assign { + ($a: expr, $b: expr) => { + $a = $a.saturating_sub($b); + }; +} diff --git a/eth2/state_processing/src/per_epoch_processing.rs b/eth2/state_processing/src/per_epoch_processing.rs index a54518364..732a6e48f 100644 --- a/eth2/state_processing/src/per_epoch_processing.rs +++ b/eth2/state_processing/src/per_epoch_processing.rs @@ -1,7 +1,8 @@ -use errors::{EpochProcessingError as Error, WinningRootError}; -use grouped_attesters::GroupedAttesters; +use attester_sets::AttesterSets; +use errors::EpochProcessingError as Error; +use inclusion_distance::{inclusion_distance, inclusion_slot}; use integer_sqrt::IntegerSquareRoot; -use log::{debug, trace}; +use log::debug; use rayon::prelude::*; use ssz::TreeHash; use std::collections::{HashMap, HashSet}; @@ -9,21 +10,11 @@ use std::iter::FromIterator; use types::{validator_registry::get_active_validator_indices, *}; use winning_root::{winning_root, WinningRoot}; +pub mod attester_sets; pub mod errors; -mod grouped_attesters; -mod tests; -mod winning_root; - -macro_rules! safe_add_assign { - ($a: expr, $b: expr) => { - $a = $a.saturating_add($b); - }; -} -macro_rules! safe_sub_assign { - ($a: expr, $b: expr) => { - $a = $a.saturating_sub($b); - }; -} +pub mod inclusion_distance; +pub mod tests; +pub mod winning_root; pub fn per_epoch_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> { let current_epoch = state.current_epoch(spec); @@ -40,7 +31,7 @@ pub fn per_epoch_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result state.build_epoch_cache(RelativeEpoch::Current, spec)?; state.build_epoch_cache(RelativeEpoch::Next, spec)?; - let attesters = GroupedAttesters::new(&state, spec)?; + let attesters = AttesterSets::new(&state, spec)?; let active_validator_indices = get_active_validator_indices( &state.validator_registry, @@ -86,12 +77,14 @@ pub fn per_epoch_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result process_validator_registry(state, spec)?; // Final updates - state.latest_active_index_roots[(next_epoch.as_usize() - + spec.activation_exit_delay as usize) - % spec.latest_active_index_roots_length] = hash_tree_root(get_active_validator_indices( + let active_tree_root = get_active_validator_indices( &state.validator_registry, next_epoch + Epoch::from(spec.activation_exit_delay), - )); + ) + .hash_tree_root(); + state.latest_active_index_roots[(next_epoch.as_usize() + + spec.activation_exit_delay as usize) + % spec.latest_active_index_roots_length] = Hash256::from(&active_tree_root[..]); state.latest_slashed_balances[next_epoch.as_usize() % spec.latest_slashed_exit_length] = state.latest_slashed_balances[current_epoch.as_usize() % spec.latest_slashed_exit_length]; @@ -114,10 +107,6 @@ pub fn per_epoch_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result Ok(()) } -fn hash_tree_root(input: Vec) -> Hash256 { - Hash256::from(&input.hash_tree_root()[..]) -} - /// Spec v0.4.0 fn process_eth1_data(state: &mut BeaconState, spec: &ChainSpec) { let next_epoch = state.next_epoch(spec); @@ -210,7 +199,7 @@ fn process_justification( state.justified_epoch = new_justified_epoch; } -pub type WinningRootHashSet = HashMap>; +pub type WinningRootHashSet = HashMap; fn process_crosslinks( state: &mut BeaconState, @@ -219,33 +208,25 @@ fn process_crosslinks( let current_epoch_attestations: Vec<&PendingAttestation> = state .latest_attestations .par_iter() - .filter(|a| { - (a.data.slot / spec.slots_per_epoch).epoch(spec.slots_per_epoch) - == state.current_epoch(spec) - }) + .filter(|a| a.data.slot.epoch(spec.slots_per_epoch) == state.current_epoch(spec)) .collect(); let previous_epoch_attestations: Vec<&PendingAttestation> = state .latest_attestations .par_iter() - .filter(|a| { - (a.data.slot / spec.slots_per_epoch).epoch(spec.slots_per_epoch) - == state.previous_epoch(spec) - }) + .filter(|a| a.data.slot.epoch(spec.slots_per_epoch) == state.previous_epoch(spec)) .collect(); - let mut winning_root_for_shards: HashMap> = - HashMap::new(); - // for slot in state.slot.saturating_sub(2 * spec.slots_per_epoch)..state.slot { - for slot in state.previous_epoch(spec).slot_iter(spec.slots_per_epoch) { - trace!( - "Finding winning root for slot: {} (epoch: {})", - slot, - slot.epoch(spec.slots_per_epoch) - ); + let mut winning_root_for_shards: WinningRootHashSet = HashMap::new(); - // Clone is used to remove the borrow. It becomes an issue later when trying to mutate - // `state.balances`. + let previous_and_current_epoch_slots: Vec = state + .previous_epoch(spec) + .slot_iter(spec.slots_per_epoch) + .chain(state.current_epoch(spec).slot_iter(spec.slots_per_epoch)) + .collect(); + + for slot in previous_and_current_epoch_slots { + // Clone removes the borrow which becomes an issue when mutating `state.balances`. let crosslink_committees_at_slot = state.get_crosslink_committees_at_slot(slot, spec)?.clone(); @@ -258,31 +239,33 @@ fn process_crosslinks( ¤t_epoch_attestations[..], &previous_epoch_attestations[..], spec, - ); + )?; - if let Ok(winning_root) = &winning_root { + if let Some(winning_root) = winning_root { let total_committee_balance = state.get_total_balance(&crosslink_committee, spec); + // TODO: I think this has a bug. if (3 * winning_root.total_attesting_balance) >= (2 * total_committee_balance) { state.latest_crosslinks[shard as usize] = Crosslink { epoch: state.current_epoch(spec), crosslink_data_root: winning_root.crosslink_data_root, } } + winning_root_for_shards.insert(shard, winning_root); } - winning_root_for_shards.insert(shard, winning_root); } } Ok(winning_root_for_shards) } +/// Spec v0.4.0 fn process_rewards_and_penalities( state: &mut BeaconState, active_validator_indices: HashSet, - attesters: &GroupedAttesters, + attesters: &AttesterSets, previous_total_balance: u64, - winning_root_for_shards: &HashMap>, + winning_root_for_shards: &WinningRootHashSet, spec: &ChainSpec, ) -> Result<(), Error> { let next_epoch = state.next_epoch(spec); @@ -290,62 +273,60 @@ fn process_rewards_and_penalities( let previous_epoch_attestations: Vec<&PendingAttestation> = state .latest_attestations .par_iter() - .filter(|a| { - (a.data.slot / spec.slots_per_epoch).epoch(spec.slots_per_epoch) - == state.previous_epoch(spec) - }) + .filter(|a| a.data.slot.epoch(spec.slots_per_epoch) == state.previous_epoch(spec)) .collect(); let base_reward_quotient = previous_total_balance.integer_sqrt() / spec.base_reward_quotient; + if base_reward_quotient == 0 { return Err(Error::BaseRewardQuotientIsZero); } - /* - * Justification and finalization - */ - let epochs_since_finality = next_epoch - state.finalized_epoch; + // Justification and finalization - let active_validator_indices_hashset: HashSet = - HashSet::from_iter(active_validator_indices.iter().cloned()); + let epochs_since_finality = next_epoch - state.finalized_epoch; if epochs_since_finality <= 4 { for index in 0..state.validator_balances.len() { let base_reward = state.base_reward(index, base_reward_quotient, spec); + // Expected FFG source if attesters.previous_epoch.indices.contains(&index) { safe_add_assign!( state.validator_balances[index], base_reward * attesters.previous_epoch.balance / previous_total_balance ); - } else if active_validator_indices_hashset.contains(&index) { + } else if active_validator_indices.contains(&index) { safe_sub_assign!(state.validator_balances[index], base_reward); } + // Expected FFG target if attesters.previous_epoch_boundary.indices.contains(&index) { safe_add_assign!( state.validator_balances[index], base_reward * attesters.previous_epoch_boundary.balance / previous_total_balance ); - } else if active_validator_indices_hashset.contains(&index) { + } else if active_validator_indices.contains(&index) { safe_sub_assign!(state.validator_balances[index], base_reward); } + // Expected beacon chain head if attesters.previous_epoch_head.indices.contains(&index) { safe_add_assign!( state.validator_balances[index], base_reward * attesters.previous_epoch_head.balance / previous_total_balance ); - } else if active_validator_indices_hashset.contains(&index) { + } else if active_validator_indices.contains(&index) { safe_sub_assign!(state.validator_balances[index], base_reward); } } + // Inclusion distance for &index in &attesters.previous_epoch.indices { let base_reward = state.base_reward(index, base_reward_quotient, spec); let inclusion_distance = - state.inclusion_distance(&previous_epoch_attestations, index, spec)?; + inclusion_distance(state, &previous_epoch_attestations, index, spec)?; safe_add_assign!( state.validator_balances[index], @@ -356,7 +337,8 @@ fn process_rewards_and_penalities( for index in 0..state.validator_balances.len() { let inactivity_penalty = state.inactivity_penalty(index, epochs_since_finality, base_reward_quotient, spec); - if active_validator_indices_hashset.contains(&index) { + + if active_validator_indices.contains(&index) { if !attesters.previous_epoch.indices.contains(&index) { safe_sub_assign!(state.validator_balances[index], inactivity_penalty); } @@ -380,7 +362,7 @@ fn process_rewards_and_penalities( for &index in &attesters.previous_epoch.indices { let base_reward = state.base_reward(index, base_reward_quotient, spec); let inclusion_distance = - state.inclusion_distance(&previous_epoch_attestations, index, spec)?; + inclusion_distance(state, &previous_epoch_attestations, index, spec)?; safe_sub_assign!( state.validator_balances[index], @@ -390,61 +372,69 @@ fn process_rewards_and_penalities( } } - trace!("Processed validator justification and finalization rewards/penalities."); + // Attestation inclusion - /* - * Attestation inclusion - */ for &index in &attesters.previous_epoch.indices { - let inclusion_slot = state.inclusion_slot(&previous_epoch_attestations[..], index, spec)?; + let inclusion_slot = inclusion_slot(state, &previous_epoch_attestations[..], index, spec)?; + let proposer_index = state .get_beacon_proposer_index(inclusion_slot, spec) .map_err(|_| Error::UnableToDetermineProducer)?; + let base_reward = state.base_reward(proposer_index, base_reward_quotient, spec); + safe_add_assign!( state.validator_balances[proposer_index], base_reward / spec.attestation_inclusion_reward_quotient ); } - /* - * Crosslinks - */ + //Crosslinks + for slot in state.previous_epoch(spec).slot_iter(spec.slots_per_epoch) { - // Clone is used to remove the borrow. It becomes an issue later when trying to mutate - // `state.balances`. + // Clone removes the borrow which becomes an issue when mutating `state.balances`. let crosslink_committees_at_slot = state.get_crosslink_committees_at_slot(slot, spec)?.clone(); - for (_crosslink_committee, shard) in crosslink_committees_at_slot { + for (crosslink_committee, shard) in crosslink_committees_at_slot { let shard = shard as u64; - if let Some(Ok(winning_root)) = winning_root_for_shards.get(&shard) { - // TODO: remove the map. + // Note: I'm a little uncertain of the logic here -- I am waiting for spec v0.5.0 to + // clear it up. + // + // What happens here is: + // + // - If there was some crosslink root elected by the super-majority of this committee, + // then we reward all who voted for that root and penalize all that did not. + // - However, if there _was not_ some super-majority-voted crosslink root, then penalize + // all the validators. + // + // I'm not quite sure that the second case (no super-majority crosslink) is correct. + if let Some(winning_root) = winning_root_for_shards.get(&shard) { + // Hash set de-dedups and (hopefully) offers a speed improvement from faster + // lookups. let attesting_validator_indices: HashSet = HashSet::from_iter(winning_root.attesting_validator_indices.iter().cloned()); - for index in 0..state.validator_balances.len() { + for &index in &crosslink_committee { let base_reward = state.base_reward(index, base_reward_quotient, spec); + let total_balance = state.get_total_balance(&crosslink_committee, spec); + if attesting_validator_indices.contains(&index) { safe_add_assign!( state.validator_balances[index], - base_reward * winning_root.total_attesting_balance - / winning_root.total_balance + base_reward * winning_root.total_attesting_balance / total_balance ); } else { safe_sub_assign!(state.validator_balances[index], base_reward); } } + } else { + for &index in &crosslink_committee { + let base_reward = state.base_reward(index, base_reward_quotient, spec); - for index in &winning_root.attesting_validator_indices { - let base_reward = state.base_reward(*index, base_reward_quotient, spec); - safe_add_assign!( - state.validator_balances[*index], - base_reward * winning_root.total_attesting_balance - / winning_root.total_balance - ); + safe_sub_assign!(state.validator_balances[index], base_reward); } } } @@ -453,6 +443,7 @@ fn process_rewards_and_penalities( Ok(()) } +// Spec v0.4.0 fn process_validator_registry(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> { let current_epoch = state.current_epoch(spec); let next_epoch = state.next_epoch(spec); @@ -460,11 +451,6 @@ fn process_validator_registry(state: &mut BeaconState, spec: &ChainSpec) -> Resu state.previous_shuffling_epoch = state.current_shuffling_epoch; state.previous_shuffling_start_shard = state.current_shuffling_start_shard; - debug!( - "setting previous_shuffling_seed to : {}", - state.current_shuffling_seed - ); - state.previous_shuffling_seed = state.current_shuffling_seed; let should_update_validator_registy = if state.finalized_epoch @@ -479,7 +465,6 @@ fn process_validator_registry(state: &mut BeaconState, spec: &ChainSpec) -> Resu }; if should_update_validator_registy { - trace!("updating validator registry."); state.update_validator_registry(spec); state.current_shuffling_epoch = next_epoch; @@ -488,7 +473,6 @@ fn process_validator_registry(state: &mut BeaconState, spec: &ChainSpec) -> Resu % spec.shard_count; state.current_shuffling_seed = state.generate_seed(state.current_shuffling_epoch, spec)? } else { - trace!("not updating validator registry."); let epochs_since_last_registry_update = current_epoch - state.validator_registry_update_epoch; if (epochs_since_last_registry_update > 1) diff --git a/eth2/state_processing/src/per_epoch_processing/grouped_attesters.rs b/eth2/state_processing/src/per_epoch_processing/attester_sets.rs similarity index 98% rename from eth2/state_processing/src/per_epoch_processing/grouped_attesters.rs rename to eth2/state_processing/src/per_epoch_processing/attester_sets.rs index 02dbc4050..2b674e1bc 100644 --- a/eth2/state_processing/src/per_epoch_processing/grouped_attesters.rs +++ b/eth2/state_processing/src/per_epoch_processing/attester_sets.rs @@ -17,7 +17,7 @@ impl Attesters { } } -pub struct GroupedAttesters { +pub struct AttesterSets { pub current_epoch: Attesters, pub current_epoch_boundary: Attesters, pub previous_epoch: Attesters, @@ -25,7 +25,7 @@ pub struct GroupedAttesters { pub previous_epoch_head: Attesters, } -impl GroupedAttesters { +impl AttesterSets { pub fn new(state: &BeaconState, spec: &ChainSpec) -> Result { let mut current_epoch = Attesters::default(); let mut current_epoch_boundary = Attesters::default(); diff --git a/eth2/state_processing/src/per_epoch_processing/errors.rs b/eth2/state_processing/src/per_epoch_processing/errors.rs index 0f7063e3b..51e9b253c 100644 --- a/eth2/state_processing/src/per_epoch_processing/errors.rs +++ b/eth2/state_processing/src/per_epoch_processing/errors.rs @@ -1,11 +1,5 @@ use types::*; -#[derive(Debug, PartialEq)] -pub enum WinningRootError { - NoWinningRoot, - BeaconStateError(BeaconStateError), -} - #[derive(Debug, PartialEq)] pub enum EpochProcessingError { UnableToDetermineProducer, @@ -14,7 +8,6 @@ pub enum EpochProcessingError { NoRandaoSeed, BeaconStateError(BeaconStateError), InclusionError(InclusionError), - WinningRootError(WinningRootError), } impl From for EpochProcessingError { @@ -29,8 +22,15 @@ impl From for EpochProcessingError { } } -impl From for WinningRootError { - fn from(e: BeaconStateError) -> WinningRootError { - WinningRootError::BeaconStateError(e) +#[derive(Debug, PartialEq)] +pub enum InclusionError { + /// The validator did not participate in an attestation in this period. + NoAttestationsForValidator, + BeaconStateError(BeaconStateError), +} + +impl From for InclusionError { + fn from(e: BeaconStateError) -> InclusionError { + InclusionError::BeaconStateError(e) } } diff --git a/eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs b/eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs new file mode 100644 index 000000000..243dc67f0 --- /dev/null +++ b/eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs @@ -0,0 +1,61 @@ +use super::errors::InclusionError; +use types::*; + +/// Returns the distance between the first included attestation for some validator and this +/// slot. +/// +/// Note: In the spec this is defined "inline", not as a helper function. +/// +/// Spec v0.4.0 +pub fn inclusion_distance( + state: &BeaconState, + attestations: &[&PendingAttestation], + validator_index: usize, + spec: &ChainSpec, +) -> Result { + let attestation = earliest_included_attestation(state, attestations, validator_index, spec)?; + Ok((attestation.inclusion_slot - attestation.data.slot).as_u64()) +} + +/// Returns the slot of the earliest included attestation for some validator. +/// +/// Note: In the spec this is defined "inline", not as a helper function. +/// +/// Spec v0.4.0 +pub fn inclusion_slot( + state: &BeaconState, + attestations: &[&PendingAttestation], + validator_index: usize, + spec: &ChainSpec, +) -> Result { + let attestation = earliest_included_attestation(state, attestations, validator_index, spec)?; + Ok(attestation.inclusion_slot) +} + +/// Finds the earliest included attestation for some validator. +/// +/// Note: In the spec this is defined "inline", not as a helper function. +/// +/// Spec v0.4.0 +fn earliest_included_attestation( + state: &BeaconState, + attestations: &[&PendingAttestation], + validator_index: usize, + spec: &ChainSpec, +) -> Result { + let mut included_attestations = vec![]; + + for (i, a) in attestations.iter().enumerate() { + let participants = + state.get_attestation_participants(&a.data, &a.aggregation_bitfield, spec)?; + if participants.iter().any(|i| *i == validator_index) { + included_attestations.push(i); + } + } + + let earliest_attestation_index = included_attestations + .iter() + .min_by_key(|i| attestations[**i].inclusion_slot) + .ok_or_else(|| InclusionError::NoAttestationsForValidator)?; + Ok(attestations[*earliest_attestation_index].clone()) +} diff --git a/eth2/state_processing/src/per_epoch_processing/winning_root.rs b/eth2/state_processing/src/per_epoch_processing/winning_root.rs index c3b650c3d..07678f93b 100644 --- a/eth2/state_processing/src/per_epoch_processing/winning_root.rs +++ b/eth2/state_processing/src/per_epoch_processing/winning_root.rs @@ -1,86 +1,118 @@ -use super::errors::WinningRootError; -use std::collections::HashMap; +use std::collections::HashSet; +use std::iter::FromIterator; use types::*; #[derive(Clone)] pub struct WinningRoot { pub crosslink_data_root: Hash256, pub attesting_validator_indices: Vec, - pub total_balance: u64, pub total_attesting_balance: u64, } +impl WinningRoot { + /// Returns `true` if `self` is a "better" candidate than `other`. + /// + /// A winning root is "better" than another if it has a higher `total_attesting_balance`. Ties + /// are broken by favouring the lower `crosslink_data_root` value. + /// + /// Spec v0.4.0 + pub fn is_better_than(&self, other: &Self) -> bool { + if self.total_attesting_balance > other.total_attesting_balance { + true + } else if self.total_attesting_balance == other.total_attesting_balance { + self.crosslink_data_root < other.crosslink_data_root + } else { + false + } + } +} + +/// Returns the `crosslink_data_root` with the highest total attesting balance for the given shard. +/// Breaks ties by favouring the smaller `crosslink_data_root` hash. +/// +/// The `WinningRoot` object also contains additional fields that are useful in later stages of +/// per-epoch processing. +/// +/// Spec v0.4.0 pub fn winning_root( state: &BeaconState, shard: u64, current_epoch_attestations: &[&PendingAttestation], previous_epoch_attestations: &[&PendingAttestation], spec: &ChainSpec, -) -> Result { - let mut attestations = current_epoch_attestations.to_vec(); - attestations.append(&mut previous_epoch_attestations.to_vec()); +) -> Result, BeaconStateError> { + let mut winning_root: Option = None; - let mut candidates: HashMap = HashMap::new(); - - let mut highest_seen_balance = 0; - - for a in &attestations { - if a.data.shard != shard { - continue; - } - - let crosslink_data_root = &a.data.crosslink_data_root; - - if candidates.contains_key(crosslink_data_root) { - continue; - } - - let attesting_validator_indices = attestations + let crosslink_data_roots: HashSet = HashSet::from_iter( + previous_epoch_attestations .iter() - .try_fold::<_, _, Result<_, BeaconStateError>>(vec![], |mut acc, a| { - if (a.data.shard == shard) && (a.data.crosslink_data_root == *crosslink_data_root) { - acc.append(&mut state.get_attestation_participants( - &a.data, - &a.aggregation_bitfield, - spec, - )?); + .chain(current_epoch_attestations.iter()) + .filter_map(|a| { + if a.data.shard == shard { + Some(a.data.crosslink_data_root) + } else { + None } - Ok(acc) - })?; + }), + ); - let total_balance: u64 = attesting_validator_indices - .iter() - .fold(0, |acc, i| acc + state.get_effective_balance(*i, spec)); + for crosslink_data_root in crosslink_data_roots { + let attesting_validator_indices = get_attesting_validator_indices( + state, + shard, + current_epoch_attestations, + previous_epoch_attestations, + &crosslink_data_root, + spec, + )?; let total_attesting_balance: u64 = attesting_validator_indices .iter() .fold(0, |acc, i| acc + state.get_effective_balance(*i, spec)); - if total_attesting_balance > highest_seen_balance { - highest_seen_balance = total_attesting_balance; - } - - let candidate_root = WinningRoot { - crosslink_data_root: *crosslink_data_root, + let candidate = WinningRoot { + crosslink_data_root, attesting_validator_indices, total_attesting_balance, - total_balance, }; - candidates.insert(*crosslink_data_root, candidate_root); + if let Some(ref winner) = winning_root { + if candidate.is_better_than(&winner) { + winning_root = Some(candidate); + } + } else { + winning_root = Some(candidate); + } } - Ok(candidates - .iter() - .filter_map(|(_hash, candidate)| { - if candidate.total_attesting_balance == highest_seen_balance { - Some(candidate) - } else { - None - } - }) - .min_by_key(|candidate| candidate.crosslink_data_root) - .ok_or_else(|| WinningRootError::NoWinningRoot)? - // TODO: avoid clone. - .clone()) + Ok(winning_root) +} + +/// Returns all indices which voted for a given crosslink. May contain duplicates. +/// +/// Spec v0.4.0 +fn get_attesting_validator_indices( + state: &BeaconState, + shard: u64, + current_epoch_attestations: &[&PendingAttestation], + previous_epoch_attestations: &[&PendingAttestation], + crosslink_data_root: &Hash256, + spec: &ChainSpec, +) -> Result, BeaconStateError> { + let mut indices = vec![]; + + for a in current_epoch_attestations + .iter() + .chain(previous_epoch_attestations.iter()) + { + if (a.data.shard == shard) && (a.data.crosslink_data_root == *crosslink_data_root) { + indices.append(&mut state.get_attestation_participants( + &a.data, + &a.aggregation_bitfield, + spec, + )?); + } + } + + Ok(indices) } diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 76b97b21d..bb77981ab 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -56,13 +56,6 @@ pub enum Error { EpochCacheUninitialized(RelativeEpoch), } -#[derive(Debug, PartialEq)] -pub enum InclusionError { - /// The validator did not participate in an attestation in this period. - NoAttestationsForValidator, - Error(Error), -} - macro_rules! safe_add_assign { ($a: expr, $b: expr) => { $a = $a.saturating_add($b); @@ -1123,67 +1116,6 @@ impl BeaconState { / 2 } - /// Returns the distance between the first included attestation for some validator and this - /// slot. - /// - /// Note: In the spec this is defined "inline", not as a helper function. - /// - /// Spec v0.4.0 - pub fn inclusion_distance( - &self, - attestations: &[&PendingAttestation], - validator_index: usize, - spec: &ChainSpec, - ) -> Result { - let attestation = - self.earliest_included_attestation(attestations, validator_index, spec)?; - Ok((attestation.inclusion_slot - attestation.data.slot).as_u64()) - } - - /// Returns the slot of the earliest included attestation for some validator. - /// - /// Note: In the spec this is defined "inline", not as a helper function. - /// - /// Spec v0.4.0 - pub fn inclusion_slot( - &self, - attestations: &[&PendingAttestation], - validator_index: usize, - spec: &ChainSpec, - ) -> Result { - let attestation = - self.earliest_included_attestation(attestations, validator_index, spec)?; - Ok(attestation.inclusion_slot) - } - - /// Finds the earliest included attestation for some validator. - /// - /// Note: In the spec this is defined "inline", not as a helper function. - /// - /// Spec v0.4.0 - fn earliest_included_attestation( - &self, - attestations: &[&PendingAttestation], - validator_index: usize, - spec: &ChainSpec, - ) -> Result { - let mut included_attestations = vec![]; - - for (i, a) in attestations.iter().enumerate() { - let participants = - self.get_attestation_participants(&a.data, &a.aggregation_bitfield, spec)?; - if participants.iter().any(|i| *i == validator_index) { - included_attestations.push(i); - } - } - - let earliest_attestation_index = included_attestations - .iter() - .min_by_key(|i| attestations[**i].inclusion_slot) - .ok_or_else(|| InclusionError::NoAttestationsForValidator)?; - Ok(attestations[*earliest_attestation_index].clone()) - } - /// Returns the base reward for some validator. /// /// Note: In the spec this is defined "inline", not as a helper function. @@ -1226,12 +1158,6 @@ fn hash_tree_root(input: Vec) -> Hash256 { Hash256::from(&input.hash_tree_root()[..]) } -impl From for InclusionError { - fn from(e: Error) -> InclusionError { - InclusionError::Error(e) - } -} - impl Encodable for BeaconState { fn ssz_append(&self, s: &mut SszStream) { s.append(&self.slot); diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index e595684bc..9bf60f2c9 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -40,9 +40,7 @@ pub use crate::attestation_data_and_custody_bit::AttestationDataAndCustodyBit; pub use crate::attester_slashing::AttesterSlashing; pub use crate::beacon_block::BeaconBlock; pub use crate::beacon_block_body::BeaconBlockBody; -pub use crate::beacon_state::{ - BeaconState, Error as BeaconStateError, InclusionError, RelativeEpoch, -}; +pub use crate::beacon_state::{BeaconState, Error as BeaconStateError, RelativeEpoch}; pub use crate::chain_spec::{ChainSpec, Domain}; pub use crate::crosslink::Crosslink; pub use crate::deposit::Deposit; From dad140a338a9f011e6f184863c1f5ce04117e5c8 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 7 Mar 2019 12:11:17 +1100 Subject: [PATCH 122/132] Fix attester and proposer compile issues - Updated to use new signed roots (`SignedRoot`, `TreeHash`) - Added a temporary domain value Note: these changes are not a fully v0.4.0 upgrade. --- eth2/attester/src/lib.rs | 15 ++++++++++----- eth2/block_proposer/src/lib.rs | 23 +++++++++++++++++------ 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/eth2/attester/src/lib.rs b/eth2/attester/src/lib.rs index 5bffd0c50..8838f022d 100644 --- a/eth2/attester/src/lib.rs +++ b/eth2/attester/src/lib.rs @@ -2,8 +2,9 @@ pub mod test_utils; mod traits; use slot_clock::SlotClock; +use ssz::TreeHash; use std::sync::Arc; -use types::{AttestationData, FreeAttestation, Signature, Slot}; +use types::{AttestationData, AttestationDataAndCustodyBit, FreeAttestation, Signature, Slot}; pub use self::traits::{ BeaconNode, BeaconNodeError, DutiesReader, DutiesReaderError, PublishOutcome, Signer, @@ -137,10 +138,14 @@ impl Attester Option { self.store_produce(attestation_data); - self.signer.sign_attestation_message( - &attestation_data.signable_message(PHASE_0_CUSTODY_BIT)[..], - DOMAIN_ATTESTATION, - ) + let message = AttestationDataAndCustodyBit { + data: attestation_data.clone(), + custody_bit: PHASE_0_CUSTODY_BIT, + } + .hash_tree_root(); + + self.signer + .sign_attestation_message(&message[..], DOMAIN_ATTESTATION) } /// Returns `true` if signing some attestation_data is safe (non-slashable). diff --git a/eth2/block_proposer/src/lib.rs b/eth2/block_proposer/src/lib.rs index cea855627..55461d986 100644 --- a/eth2/block_proposer/src/lib.rs +++ b/eth2/block_proposer/src/lib.rs @@ -3,13 +3,17 @@ mod traits; use int_to_bytes::int_to_bytes32; use slot_clock::SlotClock; +use ssz::SignedRoot; use std::sync::Arc; -use types::{BeaconBlock, ChainSpec, Slot}; +use types::{BeaconBlock, ChainSpec, Hash256, Proposal, Slot}; pub use self::traits::{ BeaconNode, BeaconNodeError, DutiesReader, DutiesReaderError, PublishOutcome, Signer, }; +//TODO: obtain the correct domain using a `Fork`. +pub const TEMPORARY_DOMAIN_VALUE: u64 = 0; + #[derive(Debug, PartialEq)] pub enum PollOutcome { /// A new block was produced. @@ -136,7 +140,7 @@ impl BlockProducer return Ok(PollOutcome::SignerRejection(slot)), Some(signature) => signature, @@ -169,10 +173,17 @@ impl BlockProducer Option { self.store_produce(&block); - match self.signer.sign_block_proposal( - &block.proposal_root(&self.spec)[..], - self.spec.domain_proposal, - ) { + let proposal = Proposal { + slot: block.slot, + shard: self.spec.beacon_chain_shard_number, + block_root: Hash256::from_slice(&block.signed_root()[..]), + signature: block.signature.clone(), + }; + + match self + .signer + .sign_block_proposal(&proposal.signed_root()[..], TEMPORARY_DOMAIN_VALUE) + { None => None, Some(signature) => { block.signature = signature; From e44888210254c22a80223f1d57ae6e760906531c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 7 Mar 2019 12:25:00 +1100 Subject: [PATCH 123/132] Re-add `canonical_root` methods to block & state Turns out they were pretty useful --- eth2/types/src/beacon_block.rs | 5 +++++ eth2/types/src/beacon_state.rs | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/eth2/types/src/beacon_block.rs b/eth2/types/src/beacon_block.rs index 111ca1b4b..2e1e24ef7 100644 --- a/eth2/types/src/beacon_block.rs +++ b/eth2/types/src/beacon_block.rs @@ -44,6 +44,11 @@ impl BeaconBlock { }, } } + + /// Returns the `hash_tree_root` of the block. + pub fn canonical_root(&self) -> Hash256 { + Hash256::from_slice(&self.hash_tree_root()[..]) + } } #[cfg(test)] diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index ac8cf164d..809408b32 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -233,6 +233,11 @@ impl BeaconState { Ok(genesis_state) } + /// Returns the `hash_tree_root` of the state. + pub fn canonical_root(&self) -> Hash256 { + Hash256::from_slice(&self.hash_tree_root()[..]) + } + /// Build an epoch cache, unless it is has already been built. pub fn build_epoch_cache( &mut self, From a4e604a41ea7d90b9eacb3fac17a112c786a9f55 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 7 Mar 2019 12:25:29 +1100 Subject: [PATCH 124/132] Update BeaconChain to spec v0.4.0 --- .../src/attestation_aggregator.rs | 24 ++++++++-------- beacon_node/beacon_chain/src/beacon_chain.rs | 28 +++++++++---------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_aggregator.rs b/beacon_node/beacon_chain/src/attestation_aggregator.rs index 54f178068..75cfd7ee5 100644 --- a/beacon_node/beacon_chain/src/attestation_aggregator.rs +++ b/beacon_node/beacon_chain/src/attestation_aggregator.rs @@ -1,10 +1,8 @@ use log::trace; -use state_processing::validate_attestation_without_signature; +use ssz::TreeHash; +use state_processing::per_block_processing::validate_attestation_without_signature; use std::collections::{HashMap, HashSet}; -use types::{ - AggregateSignature, Attestation, AttestationData, BeaconState, BeaconStateError, Bitfield, - ChainSpec, FreeAttestation, Signature, -}; +use types::*; const PHASE_0_CUSTODY_BIT: bool = false; @@ -84,11 +82,11 @@ impl AttestationAggregator { /// - The signature is verified against that of the validator at `validator_index`. pub fn process_free_attestation( &mut self, - cached_state: &BeaconState, + state: &BeaconState, free_attestation: &FreeAttestation, spec: &ChainSpec, ) -> Result { - let attestation_duties = match cached_state.attestation_slot_and_shard_for_validator( + let attestation_duties = match state.attestation_slot_and_shard_for_validator( free_attestation.validator_index as usize, spec, ) { @@ -119,9 +117,13 @@ impl AttestationAggregator { invalid_outcome!(Message::BadShard); } - let signable_message = free_attestation.data.signable_message(PHASE_0_CUSTODY_BIT); + let signable_message = AttestationDataAndCustodyBit { + data: free_attestation.data.clone(), + custody_bit: PHASE_0_CUSTODY_BIT, + } + .hash_tree_root(); - let validator_record = match cached_state + let validator_record = match state .validator_registry .get(free_attestation.validator_index as usize) { @@ -131,9 +133,7 @@ impl AttestationAggregator { if !free_attestation.signature.verify( &signable_message, - cached_state - .fork - .get_domain(cached_state.current_epoch(spec), spec.domain_attestation), + spec.get_domain(state.current_epoch(spec), Domain::Attestation, &state.fork), &validator_record.pubkey, ) { invalid_outcome!(Message::BadSignature); diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index ddb9514db..93a806864 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -10,7 +10,8 @@ use parking_lot::{RwLock, RwLockReadGuard}; use slot_clock::SlotClock; use ssz::ssz_encode; use state_processing::{ - BlockProcessable, BlockProcessingError, SlotProcessable, SlotProcessingError, + per_block_processing, per_block_processing_without_verifying_block_signature, + per_slot_processing, BlockProcessingError, SlotProcessingError, }; use std::sync::Arc; use types::{ @@ -65,7 +66,7 @@ pub struct BeaconChain { pub slot_clock: U, pub attestation_aggregator: RwLock, pub deposits_for_inclusion: RwLock>, - pub exits_for_inclusion: RwLock>, + pub exits_for_inclusion: RwLock>, pub proposer_slashings_for_inclusion: RwLock>, pub attester_slashings_for_inclusion: RwLock>, canonical_head: RwLock, @@ -214,9 +215,7 @@ where let state_slot = self.state.read().slot; let head_block_root = self.head().beacon_block_root; for _ in state_slot.as_u64()..slot.as_u64() { - self.state - .write() - .per_slot_processing(head_block_root, &self.spec)?; + per_slot_processing(&mut *self.state.write(), head_block_root, &self.spec)?; } Ok(()) } @@ -333,10 +332,10 @@ where shard, beacon_block_root: self.head().beacon_block_root, epoch_boundary_root, - shard_block_root: Hash256::zero(), + crosslink_data_root: Hash256::zero(), latest_crosslink: Crosslink { epoch: self.state.read().slot.epoch(self.spec.slots_per_epoch), - shard_block_root: Hash256::zero(), + crosslink_data_root: Hash256::zero(), }, justified_epoch, justified_block_root, @@ -411,7 +410,7 @@ where } /// Accept some exit and queue it for inclusion in an appropriate block. - pub fn receive_exit_for_inclusion(&self, exit: Exit) { + pub fn receive_exit_for_inclusion(&self, exit: VoluntaryExit) { // TODO: exits are not checked for validity; check them. // // https://github.com/sigp/lighthouse/issues/276 @@ -419,7 +418,7 @@ where } /// Return a vec of exits suitable for inclusion in some block. - pub fn get_exits_for_block(&self) -> Vec { + pub fn get_exits_for_block(&self) -> Vec { // TODO: exits are indiscriminately included; check them for validity. // // https://github.com/sigp/lighthouse/issues/275 @@ -430,7 +429,7 @@ where /// inclusion queue. /// /// This ensures that `Deposits` are not included twice in successive blocks. - pub fn set_exits_as_included(&self, included_exits: &[Exit]) { + pub fn set_exits_as_included(&self, included_exits: &[VoluntaryExit]) { // TODO: method does not take forks into account; consider this. let mut indices_to_delete = vec![]; @@ -647,7 +646,7 @@ where // Transition the parent state to the present slot. let mut state = parent_state; for _ in state.slot.as_u64()..present_slot.as_u64() { - if let Err(e) = state.per_slot_processing(parent_block_root, &self.spec) { + if let Err(e) = per_slot_processing(&mut state, parent_block_root, &self.spec) { return Ok(BlockProcessingOutcome::InvalidBlock( InvalidBlock::SlotProcessingError(e), )); @@ -656,7 +655,7 @@ where // Apply the received block to its parent state (which has been transitioned into this // slot). - if let Err(e) = state.per_block_processing(&block, &self.spec) { + if let Err(e) = per_block_processing(&mut state, &block, &self.spec) { return Ok(BlockProcessingOutcome::InvalidBlock( InvalidBlock::PerBlockProcessingError(e), )); @@ -736,14 +735,15 @@ where attester_slashings: self.get_attester_slashings_for_block(), attestations, deposits: self.get_deposits_for_block(), - exits: self.get_exits_for_block(), + voluntary_exits: self.get_exits_for_block(), + transfers: vec![], }, }; trace!("BeaconChain::produce_block: updating state for new block.",); let result = - state.per_block_processing_without_verifying_block_signature(&block, &self.spec); + per_block_processing_without_verifying_block_signature(&mut state, &block, &self.spec); debug!( "BeaconNode::produce_block: state processing result: {:?}", result From 5a21e19a31adc2cf37f59d0beb643dc4c425d87d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 7 Mar 2019 12:53:15 +1100 Subject: [PATCH 125/132] Fix all compile errors from v0.4.0 update --- beacon_node/beacon_chain/src/beacon_chain.rs | 34 ++++++------------- beacon_node/beacon_chain/src/errors.rs | 33 ++++++++++++++++++ beacon_node/beacon_chain/src/lib.rs | 6 ++-- .../test_harness/src/beacon_chain_harness.rs | 13 +++---- .../test_harness/src/test_case.rs | 19 ++++------- .../test_harness/src/test_case/state_check.rs | 4 +-- .../validator_harness/direct_beacon_node.rs | 4 +-- beacon_node/src/main.rs | 2 +- eth2/fork_choice/tests/tests.rs | 9 ++--- eth2/types/src/attester_slashing/builder.rs | 6 +--- .../beacon_block_grpc_client.rs | 3 +- validator_client/src/main.rs | 6 ++-- 12 files changed, 74 insertions(+), 65 deletions(-) create mode 100644 beacon_node/beacon_chain/src/errors.rs diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 93a806864..653b2900d 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1,5 +1,6 @@ use crate::attestation_aggregator::{AttestationAggregator, Outcome as AggregationOutcome}; use crate::checkpoint::CheckPoint; +use crate::errors::{BeaconChainError as Error, BlockProductionError}; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, ClientDB, DBError, @@ -19,18 +20,6 @@ use types::{ *, }; -#[derive(Debug, PartialEq)] -pub enum Error { - InsufficientValidators, - BadRecentBlockRoots, - BeaconStateError(BeaconStateError), - DBInconsistent(String), - DBError(String), - ForkChoiceError(ForkChoiceError), - MissingBeaconBlock(Hash256), - MissingBeaconState(Hash256), -} - #[derive(Debug, PartialEq)] pub enum ValidBlock { /// The block was successfully processed. @@ -700,7 +689,10 @@ where /// /// The produced block will not be inherently valid, it must be signed by a block producer. /// Block signing is out of the scope of this function and should be done by a separate program. - pub fn produce_block(&self, randao_reveal: Signature) -> Option<(BeaconBlock, BeaconState)> { + pub fn produce_block( + &self, + randao_reveal: Signature, + ) -> Result<(BeaconBlock, BeaconState), BlockProductionError> { debug!("Producing block at slot {}...", self.state.read().slot); let mut state = self.state.read().clone(); @@ -717,7 +709,9 @@ where attestations.len() ); - let parent_root = *state.get_block_root(state.slot.saturating_sub(1_u64), &self.spec)?; + let parent_root = *state + .get_block_root(state.slot.saturating_sub(1_u64), &self.spec) + .ok_or_else(|| BlockProductionError::UnableToGetBlockRootFromState)?; let mut block = BeaconBlock { slot: state.slot, @@ -742,21 +736,13 @@ where trace!("BeaconChain::produce_block: updating state for new block.",); - let result = - per_block_processing_without_verifying_block_signature(&mut state, &block, &self.spec); - debug!( - "BeaconNode::produce_block: state processing result: {:?}", - result - ); - result.ok()?; + per_block_processing_without_verifying_block_signature(&mut state, &block, &self.spec)?; let state_root = state.canonical_root(); block.state_root = state_root; - trace!("Block produced."); - - Some((block, state)) + Ok((block, state)) } // TODO: Left this as is, modify later diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs new file mode 100644 index 000000000..58c3f87ae --- /dev/null +++ b/beacon_node/beacon_chain/src/errors.rs @@ -0,0 +1,33 @@ +use fork_choice::ForkChoiceError; +use state_processing::BlockProcessingError; +use types::*; + +macro_rules! easy_from_to { + ($from: ident, $to: ident) => { + impl From<$from> for $to { + fn from(e: $from) -> $to { + $to::$from(e) + } + } + }; +} + +#[derive(Debug, PartialEq)] +pub enum BeaconChainError { + InsufficientValidators, + BadRecentBlockRoots, + BeaconStateError(BeaconStateError), + DBInconsistent(String), + DBError(String), + ForkChoiceError(ForkChoiceError), + MissingBeaconBlock(Hash256), + MissingBeaconState(Hash256), +} + +#[derive(Debug, PartialEq)] +pub enum BlockProductionError { + UnableToGetBlockRootFromState, + BlockProcessingError(BlockProcessingError), +} + +easy_from_to!(BlockProcessingError, BlockProductionError); diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index bd3ee0788..0e879a415 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -1,9 +1,9 @@ mod attestation_aggregator; mod beacon_chain; mod checkpoint; +mod errors; -pub use self::beacon_chain::{ - BeaconChain, BlockProcessingOutcome, Error, InvalidBlock, ValidBlock, -}; +pub use self::beacon_chain::{BeaconChain, BlockProcessingOutcome, InvalidBlock, ValidBlock}; pub use self::checkpoint::CheckPoint; +pub use self::errors::BeaconChainError; pub use fork_choice::{ForkChoice, ForkChoiceAlgorithm, ForkChoiceError}; diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index 105a6d6e1..38c41e38c 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -1,6 +1,6 @@ use super::ValidatorHarness; use beacon_chain::{BeaconChain, BlockProcessingOutcome}; -pub use beacon_chain::{CheckPoint, Error as BeaconChainError}; +pub use beacon_chain::{BeaconChainError, CheckPoint}; use bls::create_proof_of_possession; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, @@ -250,16 +250,13 @@ impl BeaconChainHarness { validator_index: usize, message: &[u8], epoch: Epoch, - domain_type: u64, + domain_type: Domain, ) -> Option { let validator = self.validators.get(validator_index)?; let domain = self - .beacon_chain - .state - .read() - .fork - .get_domain(epoch, domain_type); + .spec + .get_domain(epoch, domain_type, &self.beacon_chain.state.read().fork); Some(Signature::new(message, domain, &validator.keypair.sk)) } @@ -285,7 +282,7 @@ impl BeaconChainHarness { /// Note: the `ValidatorHarness` for this validator continues to exist. Once it is exited it /// will stop receiving duties from the beacon chain and just do nothing when prompted to /// produce/attest. - pub fn add_exit(&mut self, exit: Exit) { + pub fn add_exit(&mut self, exit: VoluntaryExit) { self.beacon_chain.receive_exit_for_inclusion(exit); } diff --git a/beacon_node/beacon_chain/test_harness/src/test_case.rs b/beacon_node/beacon_chain/test_harness/src/test_case.rs index 1a5ebfbe1..cde3f55a8 100644 --- a/beacon_node/beacon_chain/test_harness/src/test_case.rs +++ b/beacon_node/beacon_chain/test_harness/src/test_case.rs @@ -200,14 +200,14 @@ impl TestCase { } } -fn build_exit(harness: &BeaconChainHarness, validator_index: u64) -> Exit { +fn build_exit(harness: &BeaconChainHarness, validator_index: u64) -> VoluntaryExit { let epoch = harness .beacon_chain .state .read() .current_epoch(&harness.spec); - let mut exit = Exit { + let mut exit = VoluntaryExit { epoch, validator_index, signature: Signature::empty_signature(), @@ -216,13 +216,8 @@ fn build_exit(harness: &BeaconChainHarness, validator_index: u64) -> Exit { let message = exit.hash_tree_root(); exit.signature = harness - .validator_sign( - validator_index as usize, - &message[..], - epoch, - harness.spec.domain_exit, - ) - .expect("Unable to sign Exit"); + .validator_sign(validator_index as usize, &message[..], epoch, Domain::Exit) + .expect("Unable to sign VoluntaryExit"); exit } @@ -234,20 +229,20 @@ fn build_double_vote_attester_slashing( harness: &BeaconChainHarness, validator_indices: &[u64], ) -> AttesterSlashing { - let signer = |validator_index: u64, message: &[u8], epoch: Epoch, domain: u64| { + let signer = |validator_index: u64, message: &[u8], epoch: Epoch, domain: Domain| { harness .validator_sign(validator_index as usize, message, epoch, domain) .expect("Unable to sign AttesterSlashing") }; - AttesterSlashingBuilder::double_vote(validator_indices, signer, &harness.spec) + AttesterSlashingBuilder::double_vote(validator_indices, signer) } /// Builds an `ProposerSlashing` for some `validator_index`. /// /// Signs the message using a `BeaconChainHarness`. fn build_proposer_slashing(harness: &BeaconChainHarness, validator_index: u64) -> ProposerSlashing { - let signer = |validator_index: u64, message: &[u8], epoch: Epoch, domain: u64| { + let signer = |validator_index: u64, message: &[u8], epoch: Epoch, domain: Domain| { harness .validator_sign(validator_index as usize, message, epoch, domain) .expect("Unable to sign AttesterSlashing") diff --git a/beacon_node/beacon_chain/test_harness/src/test_case/state_check.rs b/beacon_node/beacon_chain/test_harness/src/test_case/state_check.rs index b52dc7d6b..6fa75364a 100644 --- a/beacon_node/beacon_chain/test_harness/src/test_case/state_check.rs +++ b/beacon_node/beacon_chain/test_harness/src/test_case/state_check.rs @@ -66,7 +66,7 @@ impl StateCheck { .iter() .enumerate() .filter_map(|(i, validator)| { - if validator.is_penalized_at(state_epoch) { + if validator.slashed { Some(i as u64) } else { None @@ -108,7 +108,7 @@ impl StateCheck { .iter() .enumerate() .filter_map(|(i, validator)| { - if validator.has_initiated_exit() { + if validator.initiated_exit { Some(i as u64) } else { None diff --git a/beacon_node/beacon_chain/test_harness/src/validator_harness/direct_beacon_node.rs b/beacon_node/beacon_chain/test_harness/src/validator_harness/direct_beacon_node.rs index 06d3e7c72..d2de354d7 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator_harness/direct_beacon_node.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator_harness/direct_beacon_node.rs @@ -80,8 +80,8 @@ impl BeaconBlockNode for DirectBeaconN let (block, _state) = self .beacon_chain .produce_block(randao_reveal.clone()) - .ok_or_else(|| { - BeaconBlockNodeError::RemoteFailure("Did not produce block.".to_string()) + .map_err(|e| { + BeaconBlockNodeError::RemoteFailure(format!("Did not produce block: {:?}", e)) })?; if block.slot == slot { diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index b9ef2c8a7..072315b6b 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -78,7 +78,7 @@ fn main() { // Slot clock let genesis_time = 1_549_935_547; // 12th Feb 2018 (arbitrary value in the past). - let slot_clock = SystemTimeSlotClock::new(genesis_time, spec.slot_duration) + let slot_clock = SystemTimeSlotClock::new(genesis_time, spec.seconds_per_slot) .expect("Unable to load SystemTimeSlotClock"); // Choose the fork choice let fork_choice = BitwiseLMDGhost::new(block_store.clone(), state_store.clone()); diff --git a/eth2/fork_choice/tests/tests.rs b/eth2/fork_choice/tests/tests.rs index 1d93cd0db..a3cab6a7c 100644 --- a/eth2/fork_choice/tests/tests.rs +++ b/eth2/fork_choice/tests/tests.rs @@ -81,7 +81,8 @@ fn test_yaml_vectors( attester_slashings: vec![], attestations: vec![], deposits: vec![], - exits: vec![], + voluntary_exits: vec![], + transfers: vec![], }; // process the tests @@ -249,9 +250,9 @@ fn setup_inital_state( withdrawal_credentials: zero_hash, activation_epoch: Epoch::from(0u64), exit_epoch: spec.far_future_epoch, - withdrawal_epoch: spec.far_future_epoch, - penalized_epoch: spec.far_future_epoch, - status_flags: None, + withdrawable_epoch: spec.far_future_epoch, + initiated_exit: false, + slashed: false, }; // activate the validators for _ in 0..no_validators { diff --git a/eth2/types/src/attester_slashing/builder.rs b/eth2/types/src/attester_slashing/builder.rs index 46dcaf174..05301f30b 100644 --- a/eth2/types/src/attester_slashing/builder.rs +++ b/eth2/types/src/attester_slashing/builder.rs @@ -15,11 +15,7 @@ impl AttesterSlashingBuilder { /// - `domain: Domain` /// /// Where domain is a domain "constant" (e.g., `spec.domain_attestation`). - pub fn double_vote( - validator_indices: &[u64], - signer: F, - spec: &ChainSpec, - ) -> AttesterSlashing + pub fn double_vote(validator_indices: &[u64], signer: F) -> AttesterSlashing where F: Fn(u64, &[u8], Epoch, Domain) -> Signature, { diff --git a/validator_client/src/block_producer_service/beacon_block_grpc_client.rs b/validator_client/src/block_producer_service/beacon_block_grpc_client.rs index 6bf3005d4..6ce5c0fa0 100644 --- a/validator_client/src/block_producer_service/beacon_block_grpc_client.rs +++ b/validator_client/src/block_producer_service/beacon_block_grpc_client.rs @@ -63,7 +63,8 @@ impl BeaconNode for BeaconBlockGrpcClient { attester_slashings: vec![], attestations: vec![], deposits: vec![], - exits: vec![], + voluntary_exits: vec![], + transfers: vec![], }, })) } else { diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index eea213a45..ebab8538c 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -111,13 +111,13 @@ fn main() { let genesis_time = 1_549_935_547; let slot_clock = { info!(log, "Genesis time"; "unix_epoch_seconds" => genesis_time); - let clock = SystemTimeSlotClock::new(genesis_time, spec.slot_duration) + let clock = SystemTimeSlotClock::new(genesis_time, spec.seconds_per_slot) .expect("Unable to instantiate SystemTimeSlotClock."); Arc::new(clock) }; - let poll_interval_millis = spec.slot_duration * 1000 / 10; // 10% epoch time precision. - info!(log, "Starting block producer service"; "polls_per_epoch" => spec.slot_duration * 1000 / poll_interval_millis); + let poll_interval_millis = spec.seconds_per_slot * 1000 / 10; // 10% epoch time precision. + info!(log, "Starting block producer service"; "polls_per_epoch" => spec.seconds_per_slot * 1000 / poll_interval_millis); /* * Start threads. From 20ac1bf1f0531e73d01a973bfa6d79c23828baba Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 7 Mar 2019 13:53:17 +1100 Subject: [PATCH 126/132] Remove unused files They were accidentally introduced by a merge --- .../state_processing/src/block_processable.rs | 455 ----------- .../state_processing/src/epoch_processable.rs | 723 ------------------ 2 files changed, 1178 deletions(-) delete mode 100644 eth2/state_processing/src/block_processable.rs delete mode 100644 eth2/state_processing/src/epoch_processable.rs diff --git a/eth2/state_processing/src/block_processable.rs b/eth2/state_processing/src/block_processable.rs deleted file mode 100644 index a54bb96d4..000000000 --- a/eth2/state_processing/src/block_processable.rs +++ /dev/null @@ -1,455 +0,0 @@ -use self::verify_slashable_attestation::verify_slashable_attestation; -use crate::SlotProcessingError; -use hashing::hash; -use int_to_bytes::int_to_bytes32; -use log::{debug, trace}; -use ssz::{ssz_encode, TreeHash}; -use types::*; - -mod verify_slashable_attestation; - -const PHASE_0_CUSTODY_BIT: bool = false; - -#[derive(Debug, PartialEq)] -pub enum Error { - DBError(String), - StateAlreadyTransitioned, - PresentSlotIsNone, - UnableToDecodeBlock, - MissingParentState(Hash256), - InvalidParentState(Hash256), - MissingBeaconBlock(Hash256), - InvalidBeaconBlock(Hash256), - MissingParentBlock(Hash256), - StateSlotMismatch, - BadBlockSignature, - BadRandaoSignature, - MaxProposerSlashingsExceeded, - BadProposerSlashing, - MaxAttesterSlashingsExceed, - MaxAttestationsExceeded, - BadAttesterSlashing, - InvalidAttestation(AttestationValidationError), - NoBlockRoot, - MaxDepositsExceeded, - BadDeposit, - MaxExitsExceeded, - BadExit, - BadCustodyReseeds, - BadCustodyChallenges, - BadCustodyResponses, - BeaconStateError(BeaconStateError), - SlotProcessingError(SlotProcessingError), -} - -#[derive(Debug, PartialEq)] -pub enum AttestationValidationError { - IncludedTooEarly, - IncludedTooLate, - WrongJustifiedSlot, - WrongJustifiedRoot, - BadLatestCrosslinkRoot, - BadSignature, - ShardBlockRootNotZero, - NoBlockRoot, - BeaconStateError(BeaconStateError), -} - -macro_rules! ensure { - ($condition: expr, $result: expr) => { - if !$condition { - return Err($result); - } - }; -} - -pub trait BlockProcessable { - fn per_block_processing(&mut self, block: &BeaconBlock, spec: &ChainSpec) -> Result<(), Error>; - fn per_block_processing_without_verifying_block_signature( - &mut self, - block: &BeaconBlock, - spec: &ChainSpec, - ) -> Result<(), Error>; -} - -impl BlockProcessable for BeaconState { - fn per_block_processing(&mut self, block: &BeaconBlock, spec: &ChainSpec) -> Result<(), Error> { - per_block_processing_signature_optional(self, block, true, spec) - } - - fn per_block_processing_without_verifying_block_signature( - &mut self, - block: &BeaconBlock, - spec: &ChainSpec, - ) -> Result<(), Error> { - per_block_processing_signature_optional(self, block, false, spec) - } -} - -fn per_block_processing_signature_optional( - mut state: &mut BeaconState, - block: &BeaconBlock, - verify_block_signature: bool, - spec: &ChainSpec, -) -> Result<(), Error> { - ensure!(block.slot == state.slot, Error::StateSlotMismatch); - - // Building the previous epoch could be delayed until an attestation from a previous epoch is - // included. This is left for future optimisation. - state.build_epoch_cache(RelativeEpoch::Previous, spec)?; - state.build_epoch_cache(RelativeEpoch::Current, spec)?; - - /* - * Proposer Signature - */ - let block_proposer_index = state.get_beacon_proposer_index(block.slot, spec)?; - let block_proposer = &state.validator_registry[block_proposer_index]; - - if verify_block_signature { - ensure!( - bls_verify( - &block_proposer.pubkey, - &block.proposal_root(spec)[..], - &block.signature, - get_domain(&state.fork, state.current_epoch(spec), spec.domain_proposal) - ), - Error::BadBlockSignature - ); - } - - /* - * RANDAO - */ - ensure!( - bls_verify( - &block_proposer.pubkey, - &int_to_bytes32(state.current_epoch(spec).as_u64()), - &block.randao_reveal, - get_domain(&state.fork, state.current_epoch(spec), spec.domain_randao) - ), - Error::BadRandaoSignature - ); - - // TODO: check this is correct. - let new_mix = { - let mut mix = state.latest_randao_mixes - [state.slot.as_usize() % spec.latest_randao_mixes_length] - .as_bytes() - .to_vec(); - mix.append(&mut ssz_encode(&block.randao_reveal)); - Hash256::from_slice(&hash(&mix)[..]) - }; - - state.latest_randao_mixes[state.slot.as_usize() % spec.latest_randao_mixes_length] = new_mix; - - /* - * Eth1 data - */ - // TODO: Eth1 data processing. - - /* - * Proposer slashings - */ - ensure!( - block.body.proposer_slashings.len() as u64 <= spec.max_proposer_slashings, - Error::MaxProposerSlashingsExceeded - ); - for proposer_slashing in &block.body.proposer_slashings { - let proposer = state - .validator_registry - .get(proposer_slashing.proposer_index as usize) - .ok_or(Error::BadProposerSlashing)?; - ensure!( - proposer_slashing.proposal_data_1.slot == proposer_slashing.proposal_data_2.slot, - Error::BadProposerSlashing - ); - ensure!( - proposer_slashing.proposal_data_1.shard == proposer_slashing.proposal_data_2.shard, - Error::BadProposerSlashing - ); - ensure!( - proposer_slashing.proposal_data_1.block_root - != proposer_slashing.proposal_data_2.block_root, - Error::BadProposerSlashing - ); - ensure!( - proposer.penalized_epoch > state.current_epoch(spec), - Error::BadProposerSlashing - ); - ensure!( - bls_verify( - &proposer.pubkey, - &proposer_slashing.proposal_data_1.hash_tree_root(), - &proposer_slashing.proposal_signature_1, - get_domain( - &state.fork, - proposer_slashing - .proposal_data_1 - .slot - .epoch(spec.epoch_length), - spec.domain_proposal - ) - ), - Error::BadProposerSlashing - ); - ensure!( - bls_verify( - &proposer.pubkey, - &proposer_slashing.proposal_data_2.hash_tree_root(), - &proposer_slashing.proposal_signature_2, - get_domain( - &state.fork, - proposer_slashing - .proposal_data_2 - .slot - .epoch(spec.epoch_length), - spec.domain_proposal - ) - ), - Error::BadProposerSlashing - ); - state.penalize_validator(proposer_slashing.proposer_index as usize, spec)?; - } - - /* - * Attester slashings - */ - ensure!( - block.body.attester_slashings.len() as u64 <= spec.max_attester_slashings, - Error::MaxAttesterSlashingsExceed - ); - for attester_slashing in &block.body.attester_slashings { - verify_slashable_attestation(&mut state, &attester_slashing, spec)?; - } - - /* - * Attestations - */ - ensure!( - block.body.attestations.len() as u64 <= spec.max_attestations, - Error::MaxAttestationsExceeded - ); - - debug!("Verifying {} attestations.", block.body.attestations.len()); - - for attestation in &block.body.attestations { - validate_attestation(&state, attestation, spec)?; - - let pending_attestation = PendingAttestation { - data: attestation.data.clone(), - aggregation_bitfield: attestation.aggregation_bitfield.clone(), - custody_bitfield: attestation.custody_bitfield.clone(), - inclusion_slot: state.slot, - }; - state.latest_attestations.push(pending_attestation); - } - - /* - * Deposits - */ - ensure!( - block.body.deposits.len() as u64 <= spec.max_deposits, - Error::MaxDepositsExceeded - ); - - // TODO: verify deposit merkle branches. - for deposit in &block.body.deposits { - debug!( - "Processing deposit for pubkey {:?}", - deposit.deposit_data.deposit_input.pubkey - ); - state - .process_deposit( - deposit.deposit_data.deposit_input.pubkey.clone(), - deposit.deposit_data.amount, - deposit - .deposit_data - .deposit_input - .proof_of_possession - .clone(), - deposit.deposit_data.deposit_input.withdrawal_credentials, - None, - spec, - ) - .map_err(|_| Error::BadDeposit)?; - } - - /* - * Exits - */ - ensure!( - block.body.exits.len() as u64 <= spec.max_exits, - Error::MaxExitsExceeded - ); - - for exit in &block.body.exits { - let validator = state - .validator_registry - .get(exit.validator_index as usize) - .ok_or(Error::BadExit)?; - ensure!( - validator.exit_epoch - > state.get_entry_exit_effect_epoch(state.current_epoch(spec), spec), - Error::BadExit - ); - ensure!(state.current_epoch(spec) >= exit.epoch, Error::BadExit); - let exit_message = { - let exit_struct = Exit { - epoch: exit.epoch, - validator_index: exit.validator_index, - signature: spec.empty_signature.clone(), - }; - exit_struct.hash_tree_root() - }; - ensure!( - bls_verify( - &validator.pubkey, - &exit_message, - &exit.signature, - get_domain(&state.fork, exit.epoch, spec.domain_exit) - ), - Error::BadProposerSlashing - ); - state.initiate_validator_exit(exit.validator_index as usize); - } - - debug!("State transition complete."); - - Ok(()) -} - -pub fn validate_attestation( - state: &BeaconState, - attestation: &Attestation, - spec: &ChainSpec, -) -> Result<(), AttestationValidationError> { - validate_attestation_signature_optional(state, attestation, spec, true) -} - -pub fn validate_attestation_without_signature( - state: &BeaconState, - attestation: &Attestation, - spec: &ChainSpec, -) -> Result<(), AttestationValidationError> { - validate_attestation_signature_optional(state, attestation, spec, false) -} - -fn validate_attestation_signature_optional( - state: &BeaconState, - attestation: &Attestation, - spec: &ChainSpec, - verify_signature: bool, -) -> Result<(), AttestationValidationError> { - trace!( - "validate_attestation_signature_optional: attestation epoch: {}", - attestation.data.slot.epoch(spec.epoch_length) - ); - ensure!( - attestation.data.slot + spec.min_attestation_inclusion_delay <= state.slot, - AttestationValidationError::IncludedTooEarly - ); - ensure!( - attestation.data.slot + spec.epoch_length >= state.slot, - AttestationValidationError::IncludedTooLate - ); - if attestation.data.slot >= state.current_epoch_start_slot(spec) { - ensure!( - attestation.data.justified_epoch == state.justified_epoch, - AttestationValidationError::WrongJustifiedSlot - ); - } else { - ensure!( - attestation.data.justified_epoch == state.previous_justified_epoch, - AttestationValidationError::WrongJustifiedSlot - ); - } - ensure!( - attestation.data.justified_block_root - == *state - .get_block_root( - attestation - .data - .justified_epoch - .start_slot(spec.epoch_length), - &spec - ) - .ok_or(AttestationValidationError::NoBlockRoot)?, - AttestationValidationError::WrongJustifiedRoot - ); - let potential_crosslink = Crosslink { - shard_block_root: attestation.data.shard_block_root, - epoch: attestation.data.slot.epoch(spec.epoch_length), - }; - ensure!( - (attestation.data.latest_crosslink - == state.latest_crosslinks[attestation.data.shard as usize]) - | (attestation.data.latest_crosslink == potential_crosslink), - AttestationValidationError::BadLatestCrosslinkRoot - ); - if verify_signature { - let participants = state.get_attestation_participants( - &attestation.data, - &attestation.aggregation_bitfield, - spec, - )?; - trace!( - "slot: {}, shard: {}, participants: {:?}", - attestation.data.slot, - attestation.data.shard, - participants - ); - let mut group_public_key = AggregatePublicKey::new(); - for participant in participants { - group_public_key.add(&state.validator_registry[participant as usize].pubkey) - } - ensure!( - attestation.verify_signature( - &group_public_key, - PHASE_0_CUSTODY_BIT, - get_domain( - &state.fork, - attestation.data.slot.epoch(spec.epoch_length), - spec.domain_attestation, - ) - ), - AttestationValidationError::BadSignature - ); - } - ensure!( - attestation.data.shard_block_root == spec.zero_hash, - AttestationValidationError::ShardBlockRootNotZero - ); - Ok(()) -} - -fn get_domain(fork: &Fork, epoch: Epoch, domain_type: u64) -> u64 { - fork.get_domain(epoch, domain_type) -} - -fn bls_verify(pubkey: &PublicKey, message: &[u8], signature: &Signature, domain: u64) -> bool { - signature.verify(message, domain, pubkey) -} - -impl From for Error { - fn from(e: AttestationValidationError) -> Error { - Error::InvalidAttestation(e) - } -} - -impl From for Error { - fn from(e: BeaconStateError) -> Error { - Error::BeaconStateError(e) - } -} - -impl From for Error { - fn from(e: SlotProcessingError) -> Error { - Error::SlotProcessingError(e) - } -} - -impl From for AttestationValidationError { - fn from(e: BeaconStateError) -> AttestationValidationError { - AttestationValidationError::BeaconStateError(e) - } -} diff --git a/eth2/state_processing/src/epoch_processable.rs b/eth2/state_processing/src/epoch_processable.rs deleted file mode 100644 index ff6f18113..000000000 --- a/eth2/state_processing/src/epoch_processable.rs +++ /dev/null @@ -1,723 +0,0 @@ -use integer_sqrt::IntegerSquareRoot; -use log::{debug, trace}; -use rayon::prelude::*; -use ssz::TreeHash; -use std::collections::{HashMap, HashSet}; -use std::iter::FromIterator; -use types::{ - validator_registry::get_active_validator_indices, BeaconState, BeaconStateError, ChainSpec, - Crosslink, Epoch, Hash256, InclusionError, PendingAttestation, RelativeEpoch, -}; - -mod tests; - -macro_rules! safe_add_assign { - ($a: expr, $b: expr) => { - $a = $a.saturating_add($b); - }; -} -macro_rules! safe_sub_assign { - ($a: expr, $b: expr) => { - $a = $a.saturating_sub($b); - }; -} - -#[derive(Debug, PartialEq)] -pub enum Error { - UnableToDetermineProducer, - NoBlockRoots, - BaseRewardQuotientIsZero, - NoRandaoSeed, - BeaconStateError(BeaconStateError), - InclusionError(InclusionError), - WinningRootError(WinningRootError), -} - -#[derive(Debug, PartialEq)] -pub enum WinningRootError { - NoWinningRoot, - BeaconStateError(BeaconStateError), -} - -#[derive(Clone)] -pub struct WinningRoot { - pub shard_block_root: Hash256, - pub attesting_validator_indices: Vec, - pub total_balance: u64, - pub total_attesting_balance: u64, -} - -pub trait EpochProcessable { - fn per_epoch_processing(&mut self, spec: &ChainSpec) -> Result<(), Error>; -} - -impl EpochProcessable for BeaconState { - // Cyclomatic complexity is ignored. It would be ideal to split this function apart, however it - // remains monolithic to allow for easier spec updates. Once the spec is more stable we can - // optimise. - #[allow(clippy::cyclomatic_complexity)] - fn per_epoch_processing(&mut self, spec: &ChainSpec) -> Result<(), Error> { - let current_epoch = self.current_epoch(spec); - let previous_epoch = self.previous_epoch(spec); - let next_epoch = self.next_epoch(spec); - - debug!( - "Starting per-epoch processing on epoch {}...", - self.current_epoch(spec) - ); - - // Ensure all of the caches are built. - self.build_epoch_cache(RelativeEpoch::Previous, spec)?; - self.build_epoch_cache(RelativeEpoch::Current, spec)?; - self.build_epoch_cache(RelativeEpoch::Next, spec)?; - - /* - * Validators attesting during the current epoch. - */ - let active_validator_indices = get_active_validator_indices( - &self.validator_registry, - self.slot.epoch(spec.epoch_length), - ); - let current_total_balance = self.get_total_balance(&active_validator_indices[..], spec); - - trace!( - "{} validators with a total balance of {} wei.", - active_validator_indices.len(), - current_total_balance - ); - - let current_epoch_attestations: Vec<&PendingAttestation> = self - .latest_attestations - .par_iter() - .filter(|a| { - (a.data.slot / spec.epoch_length).epoch(spec.epoch_length) - == self.current_epoch(spec) - }) - .collect(); - - trace!( - "Current epoch attestations: {}", - current_epoch_attestations.len() - ); - - let current_epoch_boundary_attestations: Vec<&PendingAttestation> = - current_epoch_attestations - .par_iter() - .filter( - |a| match self.get_block_root(self.current_epoch_start_slot(spec), spec) { - Some(block_root) => { - (a.data.epoch_boundary_root == *block_root) - && (a.data.justified_epoch == self.justified_epoch) - } - None => unreachable!(), - }, - ) - .cloned() - .collect(); - - let current_epoch_boundary_attester_indices = self - .get_attestation_participants_union(¤t_epoch_boundary_attestations[..], spec)?; - let current_epoch_boundary_attesting_balance = - self.get_total_balance(¤t_epoch_boundary_attester_indices[..], spec); - - trace!( - "Current epoch boundary attesters: {}", - current_epoch_boundary_attester_indices.len() - ); - - /* - * Validators attesting during the previous epoch - */ - - /* - * Validators that made an attestation during the previous epoch - */ - let previous_epoch_attestations: Vec<&PendingAttestation> = self - .latest_attestations - .par_iter() - .filter(|a| { - //TODO: ensure these saturating subs are correct. - (a.data.slot / spec.epoch_length).epoch(spec.epoch_length) - == self.previous_epoch(spec) - }) - .collect(); - - debug!( - "previous epoch attestations: {}", - previous_epoch_attestations.len() - ); - - let previous_epoch_attester_indices = - self.get_attestation_participants_union(&previous_epoch_attestations[..], spec)?; - let previous_total_balance = self.get_total_balance( - &get_active_validator_indices(&self.validator_registry, previous_epoch), - spec, - ); - - /* - * Validators targetting the previous justified slot - */ - let previous_epoch_justified_attestations: Vec<&PendingAttestation> = { - let mut a: Vec<&PendingAttestation> = current_epoch_attestations - .iter() - .filter(|a| a.data.justified_epoch == self.previous_justified_epoch) - .cloned() - .collect(); - let mut b: Vec<&PendingAttestation> = previous_epoch_attestations - .iter() - .filter(|a| a.data.justified_epoch == self.previous_justified_epoch) - .cloned() - .collect(); - a.append(&mut b); - a - }; - - let previous_epoch_justified_attester_indices = self - .get_attestation_participants_union(&previous_epoch_justified_attestations[..], spec)?; - let previous_epoch_justified_attesting_balance = - self.get_total_balance(&previous_epoch_justified_attester_indices[..], spec); - - /* - * Validators justifying the epoch boundary block at the start of the previous epoch - */ - let previous_epoch_boundary_attestations: Vec<&PendingAttestation> = - previous_epoch_justified_attestations - .iter() - .filter( - |a| match self.get_block_root(self.previous_epoch_start_slot(spec), spec) { - Some(block_root) => a.data.epoch_boundary_root == *block_root, - None => unreachable!(), - }, - ) - .cloned() - .collect(); - - let previous_epoch_boundary_attester_indices = self - .get_attestation_participants_union(&previous_epoch_boundary_attestations[..], spec)?; - let previous_epoch_boundary_attesting_balance = - self.get_total_balance(&previous_epoch_boundary_attester_indices[..], spec); - - /* - * Validators attesting to the expected beacon chain head during the previous epoch. - */ - let previous_epoch_head_attestations: Vec<&PendingAttestation> = - previous_epoch_attestations - .iter() - .filter(|a| match self.get_block_root(a.data.slot, spec) { - Some(block_root) => a.data.beacon_block_root == *block_root, - None => unreachable!(), - }) - .cloned() - .collect(); - - let previous_epoch_head_attester_indices = - self.get_attestation_participants_union(&previous_epoch_head_attestations[..], spec)?; - let previous_epoch_head_attesting_balance = - self.get_total_balance(&previous_epoch_head_attester_indices[..], spec); - - debug!( - "previous_epoch_head_attester_balance of {} wei.", - previous_epoch_head_attesting_balance - ); - - /* - * Eth1 Data - */ - if self.next_epoch(spec) % spec.eth1_data_voting_period == 0 { - for eth1_data_vote in &self.eth1_data_votes { - if eth1_data_vote.vote_count * 2 > spec.eth1_data_voting_period { - self.latest_eth1_data = eth1_data_vote.eth1_data.clone(); - } - } - self.eth1_data_votes = vec![]; - } - - /* - * Justification - */ - - let mut new_justified_epoch = self.justified_epoch; - self.justification_bitfield <<= 1; - - // If > 2/3 of the total balance attested to the previous epoch boundary - // - // - Set the 2nd bit of the bitfield. - // - Set the previous epoch to be justified. - if (3 * previous_epoch_boundary_attesting_balance) >= (2 * current_total_balance) { - self.justification_bitfield |= 2; - new_justified_epoch = previous_epoch; - trace!(">= 2/3 voted for previous epoch boundary"); - } - // If > 2/3 of the total balance attested to the previous epoch boundary - // - // - Set the 1st bit of the bitfield. - // - Set the current epoch to be justified. - if (3 * current_epoch_boundary_attesting_balance) >= (2 * current_total_balance) { - self.justification_bitfield |= 1; - new_justified_epoch = current_epoch; - trace!(">= 2/3 voted for current epoch boundary"); - } - - // If: - // - // - All three epochs prior to this epoch have been justified. - // - The previous justified justified epoch was three epochs ago. - // - // Then, set the finalized epoch to be three epochs ago. - if ((self.justification_bitfield >> 1) % 8 == 0b111) - & (self.previous_justified_epoch == previous_epoch - 2) - { - self.finalized_epoch = self.previous_justified_epoch; - trace!("epoch - 3 was finalized (1st condition)."); - } - // If: - // - // - Both two epochs prior to this epoch have been justified. - // - The previous justified epoch was two epochs ago. - // - // Then, set the finalized epoch to two epochs ago. - if ((self.justification_bitfield >> 1) % 4 == 0b11) - & (self.previous_justified_epoch == previous_epoch - 1) - { - self.finalized_epoch = self.previous_justified_epoch; - trace!("epoch - 2 was finalized (2nd condition)."); - } - // If: - // - // - This epoch and the two prior have been justified. - // - The presently justified epoch was two epochs ago. - // - // Then, set the finalized epoch to two epochs ago. - if (self.justification_bitfield % 8 == 0b111) & (self.justified_epoch == previous_epoch - 1) - { - self.finalized_epoch = self.justified_epoch; - trace!("epoch - 2 was finalized (3rd condition)."); - } - // If: - // - // - This epoch and the epoch prior to it have been justified. - // - Set the previous epoch to be justified. - // - // Then, set the finalized epoch to be the previous epoch. - if (self.justification_bitfield % 4 == 0b11) & (self.justified_epoch == previous_epoch) { - self.finalized_epoch = self.justified_epoch; - trace!("epoch - 1 was finalized (4th condition)."); - } - - self.previous_justified_epoch = self.justified_epoch; - self.justified_epoch = new_justified_epoch; - - debug!( - "Finalized epoch {}, justified epoch {}.", - self.finalized_epoch, self.justified_epoch - ); - - /* - * Crosslinks - */ - - // Cached for later lookups. - let mut winning_root_for_shards: HashMap> = - HashMap::new(); - - // for slot in self.slot.saturating_sub(2 * spec.epoch_length)..self.slot { - for slot in self.previous_epoch(spec).slot_iter(spec.epoch_length) { - trace!( - "Finding winning root for slot: {} (epoch: {})", - slot, - slot.epoch(spec.epoch_length) - ); - - // Clone is used to remove the borrow. It becomes an issue later when trying to mutate - // `self.balances`. - let crosslink_committees_at_slot = - self.get_crosslink_committees_at_slot(slot, spec)?.clone(); - - for (crosslink_committee, shard) in crosslink_committees_at_slot { - let shard = shard as u64; - - let winning_root = winning_root( - self, - shard, - ¤t_epoch_attestations, - &previous_epoch_attestations, - spec, - ); - - if let Ok(winning_root) = &winning_root { - let total_committee_balance = - self.get_total_balance(&crosslink_committee[..], spec); - - if (3 * winning_root.total_attesting_balance) >= (2 * total_committee_balance) { - self.latest_crosslinks[shard as usize] = Crosslink { - epoch: current_epoch, - shard_block_root: winning_root.shard_block_root, - } - } - } - winning_root_for_shards.insert(shard, winning_root); - } - } - - trace!( - "Found {} winning shard roots.", - winning_root_for_shards.len() - ); - - /* - * Rewards and Penalities - */ - let base_reward_quotient = - previous_total_balance.integer_sqrt() / spec.base_reward_quotient; - if base_reward_quotient == 0 { - return Err(Error::BaseRewardQuotientIsZero); - } - - /* - * Justification and finalization - */ - let epochs_since_finality = next_epoch - self.finalized_epoch; - - let previous_epoch_justified_attester_indices_hashset: HashSet = - HashSet::from_iter(previous_epoch_justified_attester_indices.iter().cloned()); - let previous_epoch_boundary_attester_indices_hashset: HashSet = - HashSet::from_iter(previous_epoch_boundary_attester_indices.iter().cloned()); - let previous_epoch_head_attester_indices_hashset: HashSet = - HashSet::from_iter(previous_epoch_head_attester_indices.iter().cloned()); - let previous_epoch_attester_indices_hashset: HashSet = - HashSet::from_iter(previous_epoch_attester_indices.iter().cloned()); - let active_validator_indices_hashset: HashSet = - HashSet::from_iter(active_validator_indices.iter().cloned()); - - debug!("previous epoch justified attesters: {}, previous epoch boundary attesters: {}, previous epoch head attesters: {}, previous epoch attesters: {}", previous_epoch_justified_attester_indices.len(), previous_epoch_boundary_attester_indices.len(), previous_epoch_head_attester_indices.len(), previous_epoch_attester_indices.len()); - - debug!("{} epochs since finality.", epochs_since_finality); - - if epochs_since_finality <= 4 { - for index in 0..self.validator_balances.len() { - let base_reward = self.base_reward(index, base_reward_quotient, spec); - - if previous_epoch_justified_attester_indices_hashset.contains(&index) { - safe_add_assign!( - self.validator_balances[index], - base_reward * previous_epoch_justified_attesting_balance - / previous_total_balance - ); - } else if active_validator_indices_hashset.contains(&index) { - safe_sub_assign!(self.validator_balances[index], base_reward); - } - - if previous_epoch_boundary_attester_indices_hashset.contains(&index) { - safe_add_assign!( - self.validator_balances[index], - base_reward * previous_epoch_boundary_attesting_balance - / previous_total_balance - ); - } else if active_validator_indices_hashset.contains(&index) { - safe_sub_assign!(self.validator_balances[index], base_reward); - } - - if previous_epoch_head_attester_indices_hashset.contains(&index) { - safe_add_assign!( - self.validator_balances[index], - base_reward * previous_epoch_head_attesting_balance - / previous_total_balance - ); - } else if active_validator_indices_hashset.contains(&index) { - safe_sub_assign!(self.validator_balances[index], base_reward); - } - } - - for index in previous_epoch_attester_indices { - let base_reward = self.base_reward(index, base_reward_quotient, spec); - let inclusion_distance = - self.inclusion_distance(&previous_epoch_attestations, index, spec)?; - - safe_add_assign!( - self.validator_balances[index], - base_reward * spec.min_attestation_inclusion_delay / inclusion_distance - ) - } - } else { - for index in 0..self.validator_balances.len() { - let inactivity_penalty = self.inactivity_penalty( - index, - epochs_since_finality, - base_reward_quotient, - spec, - ); - if active_validator_indices_hashset.contains(&index) { - if !previous_epoch_justified_attester_indices_hashset.contains(&index) { - safe_sub_assign!(self.validator_balances[index], inactivity_penalty); - } - if !previous_epoch_boundary_attester_indices_hashset.contains(&index) { - safe_sub_assign!(self.validator_balances[index], inactivity_penalty); - } - if !previous_epoch_head_attester_indices_hashset.contains(&index) { - safe_sub_assign!(self.validator_balances[index], inactivity_penalty); - } - - if self.validator_registry[index].penalized_epoch <= current_epoch { - let base_reward = self.base_reward(index, base_reward_quotient, spec); - safe_sub_assign!( - self.validator_balances[index], - 2 * inactivity_penalty + base_reward - ); - } - } - } - - for index in previous_epoch_attester_indices { - let base_reward = self.base_reward(index, base_reward_quotient, spec); - let inclusion_distance = - self.inclusion_distance(&previous_epoch_attestations, index, spec)?; - - safe_sub_assign!( - self.validator_balances[index], - base_reward - - base_reward * spec.min_attestation_inclusion_delay / inclusion_distance - ); - } - } - - trace!("Processed validator justification and finalization rewards/penalities."); - - /* - * Attestation inclusion - */ - for &index in &previous_epoch_attester_indices_hashset { - let inclusion_slot = - self.inclusion_slot(&previous_epoch_attestations[..], index, spec)?; - let proposer_index = self - .get_beacon_proposer_index(inclusion_slot, spec) - .map_err(|_| Error::UnableToDetermineProducer)?; - let base_reward = self.base_reward(proposer_index, base_reward_quotient, spec); - safe_add_assign!( - self.validator_balances[proposer_index], - base_reward / spec.includer_reward_quotient - ); - } - - trace!( - "Previous epoch attesters: {}.", - previous_epoch_attester_indices_hashset.len() - ); - - /* - * Crosslinks - */ - for slot in self.previous_epoch(spec).slot_iter(spec.epoch_length) { - // Clone is used to remove the borrow. It becomes an issue later when trying to mutate - // `self.balances`. - let crosslink_committees_at_slot = - self.get_crosslink_committees_at_slot(slot, spec)?.clone(); - - for (_crosslink_committee, shard) in crosslink_committees_at_slot { - let shard = shard as u64; - - if let Some(Ok(winning_root)) = winning_root_for_shards.get(&shard) { - // TODO: remove the map. - let attesting_validator_indices: HashSet = HashSet::from_iter( - winning_root.attesting_validator_indices.iter().cloned(), - ); - - for index in 0..self.validator_balances.len() { - let base_reward = self.base_reward(index, base_reward_quotient, spec); - - if attesting_validator_indices.contains(&index) { - safe_add_assign!( - self.validator_balances[index], - base_reward * winning_root.total_attesting_balance - / winning_root.total_balance - ); - } else { - safe_sub_assign!(self.validator_balances[index], base_reward); - } - } - - for index in &winning_root.attesting_validator_indices { - let base_reward = self.base_reward(*index, base_reward_quotient, spec); - safe_add_assign!( - self.validator_balances[*index], - base_reward * winning_root.total_attesting_balance - / winning_root.total_balance - ); - } - } - } - } - - /* - * Ejections - */ - self.process_ejections(spec); - - /* - * Validator Registry - */ - self.previous_calculation_epoch = self.current_calculation_epoch; - self.previous_epoch_start_shard = self.current_epoch_start_shard; - - debug!( - "setting previous_epoch_seed to : {}", - self.current_epoch_seed - ); - - self.previous_epoch_seed = self.current_epoch_seed; - - let should_update_validator_registy = if self.finalized_epoch - > self.validator_registry_update_epoch - { - (0..self.get_current_epoch_committee_count(spec)).all(|i| { - let shard = (self.current_epoch_start_shard + i as u64) % spec.shard_count; - self.latest_crosslinks[shard as usize].epoch > self.validator_registry_update_epoch - }) - } else { - false - }; - - if should_update_validator_registy { - trace!("updating validator registry."); - self.update_validator_registry(spec); - - self.current_calculation_epoch = next_epoch; - self.current_epoch_start_shard = (self.current_epoch_start_shard - + self.get_current_epoch_committee_count(spec) as u64) - % spec.shard_count; - self.current_epoch_seed = self.generate_seed(self.current_calculation_epoch, spec)? - } else { - trace!("not updating validator registry."); - let epochs_since_last_registry_update = - current_epoch - self.validator_registry_update_epoch; - if (epochs_since_last_registry_update > 1) - & epochs_since_last_registry_update.is_power_of_two() - { - self.current_calculation_epoch = next_epoch; - self.current_epoch_seed = - self.generate_seed(self.current_calculation_epoch, spec)? - } - } - - self.process_penalties_and_exits(spec); - - self.latest_index_roots[(next_epoch.as_usize() + spec.entry_exit_delay as usize) - % spec.latest_index_roots_length] = hash_tree_root(get_active_validator_indices( - &self.validator_registry, - next_epoch + Epoch::from(spec.entry_exit_delay), - )); - self.latest_penalized_balances[next_epoch.as_usize() % spec.latest_penalized_exit_length] = - self.latest_penalized_balances - [current_epoch.as_usize() % spec.latest_penalized_exit_length]; - self.latest_randao_mixes[next_epoch.as_usize() % spec.latest_randao_mixes_length] = self - .get_randao_mix(current_epoch, spec) - .and_then(|x| Some(*x)) - .ok_or_else(|| Error::NoRandaoSeed)?; - self.latest_attestations = self - .latest_attestations - .iter() - .filter(|a| a.data.slot.epoch(spec.epoch_length) >= current_epoch) - .cloned() - .collect(); - - debug!("Epoch transition complete."); - - Ok(()) - } -} - -fn hash_tree_root(input: Vec) -> Hash256 { - Hash256::from_slice(&input.hash_tree_root()[..]) -} - -fn winning_root( - state: &BeaconState, - shard: u64, - current_epoch_attestations: &[&PendingAttestation], - previous_epoch_attestations: &[&PendingAttestation], - spec: &ChainSpec, -) -> Result { - let mut attestations = current_epoch_attestations.to_vec(); - attestations.append(&mut previous_epoch_attestations.to_vec()); - - let mut candidates: HashMap = HashMap::new(); - - let mut highest_seen_balance = 0; - - for a in &attestations { - if a.data.shard != shard { - continue; - } - - let shard_block_root = &a.data.shard_block_root; - - if candidates.contains_key(shard_block_root) { - continue; - } - - let attesting_validator_indices = attestations - .iter() - .try_fold::<_, _, Result<_, BeaconStateError>>(vec![], |mut acc, a| { - if (a.data.shard == shard) && (a.data.shard_block_root == *shard_block_root) { - acc.append(&mut state.get_attestation_participants( - &a.data, - &a.aggregation_bitfield, - spec, - )?); - } - Ok(acc) - })?; - - let total_balance: u64 = attesting_validator_indices - .iter() - .fold(0, |acc, i| acc + state.get_effective_balance(*i, spec)); - - let total_attesting_balance: u64 = attesting_validator_indices - .iter() - .fold(0, |acc, i| acc + state.get_effective_balance(*i, spec)); - - if total_attesting_balance > highest_seen_balance { - highest_seen_balance = total_attesting_balance; - } - - let candidate_root = WinningRoot { - shard_block_root: *shard_block_root, - attesting_validator_indices, - total_attesting_balance, - total_balance, - }; - - candidates.insert(*shard_block_root, candidate_root); - } - - Ok(candidates - .iter() - .filter_map(|(_hash, candidate)| { - if candidate.total_attesting_balance == highest_seen_balance { - Some(candidate) - } else { - None - } - }) - .min_by_key(|candidate| candidate.shard_block_root) - .ok_or_else(|| WinningRootError::NoWinningRoot)? - // TODO: avoid clone. - .clone()) -} - -impl From for Error { - fn from(e: InclusionError) -> Error { - Error::InclusionError(e) - } -} - -impl From for Error { - fn from(e: BeaconStateError) -> Error { - Error::BeaconStateError(e) - } -} - -impl From for WinningRootError { - fn from(e: BeaconStateError) -> WinningRootError { - WinningRootError::BeaconStateError(e) - } -} From db3b6cba6d54c404071386c239cf0c91a49bb13c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 7 Mar 2019 13:54:56 +1100 Subject: [PATCH 127/132] Introduce `Fork` struct to `block_producer` It's a pretty crappy solution, IMO. It shouldn't really belong in "duties" but this gets the job done for now. --- .../src/validator_harness/direct_duties.rs | 6 ++- eth2/block_proposer/src/lib.rs | 37 ++++++++++++------- .../src/test_utils/epoch_map.rs | 10 ++++- eth2/block_proposer/src/traits.rs | 3 +- .../src/block_producer_service/mod.rs | 3 ++ validator_client/src/duties/epoch_duties.rs | 13 ++++++- 6 files changed, 54 insertions(+), 18 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/src/validator_harness/direct_duties.rs b/beacon_node/beacon_chain/test_harness/src/validator_harness/direct_duties.rs index 5bed59531..dec93c334 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator_harness/direct_duties.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator_harness/direct_duties.rs @@ -9,7 +9,7 @@ use db::ClientDB; use fork_choice::ForkChoice; use slot_clock::SlotClock; use std::sync::Arc; -use types::{PublicKey, Slot}; +use types::{Fork, PublicKey, Slot}; /// Connects directly to a borrowed `BeaconChain` and reads attester/proposer duties directly from /// it. @@ -40,6 +40,10 @@ impl ProducerDutiesReader for DirectDu Err(_) => Err(ProducerDutiesReaderError::UnknownEpoch), } } + + fn fork(&self) -> Result { + Ok(self.beacon_chain.state.read().fork.clone()) + } } impl AttesterDutiesReader for DirectDuties { diff --git a/eth2/block_proposer/src/lib.rs b/eth2/block_proposer/src/lib.rs index 55461d986..5cddbaedc 100644 --- a/eth2/block_proposer/src/lib.rs +++ b/eth2/block_proposer/src/lib.rs @@ -1,19 +1,15 @@ pub mod test_utils; mod traits; -use int_to_bytes::int_to_bytes32; use slot_clock::SlotClock; -use ssz::SignedRoot; +use ssz::{SignedRoot, TreeHash}; use std::sync::Arc; -use types::{BeaconBlock, ChainSpec, Hash256, Proposal, Slot}; +use types::{BeaconBlock, ChainSpec, Domain, Hash256, Proposal, Slot}; pub use self::traits::{ BeaconNode, BeaconNodeError, DutiesReader, DutiesReaderError, PublishOutcome, Signer, }; -//TODO: obtain the correct domain using a `Fork`. -pub const TEMPORARY_DOMAIN_VALUE: u64 = 0; - #[derive(Debug, PartialEq)] pub enum PollOutcome { /// A new block was produced. @@ -32,6 +28,8 @@ pub enum PollOutcome { SignerRejection(Slot), /// The public key for this validator is not an active validator. ValidatorIsUnknown(Slot), + /// Unable to determine a `Fork` for signature domain generation. + UnableToGetFork(Slot), } #[derive(Debug, PartialEq)] @@ -134,14 +132,20 @@ impl BlockProducer Result { + let fork = match self.epoch_map.fork() { + Ok(fork) => fork, + Err(_) => return Ok(PollOutcome::UnableToGetFork(slot)), + }; + let randao_reveal = { // TODO: add domain, etc to this message. Also ensure result matches `into_to_bytes32`. - let message = int_to_bytes32(slot.epoch(self.spec.slots_per_epoch).as_u64()); + let message = slot.epoch(self.spec.slots_per_epoch).hash_tree_root(); - match self - .signer - .sign_randao_reveal(&message, TEMPORARY_DOMAIN_VALUE) - { + match self.signer.sign_randao_reveal( + &message, + self.spec + .get_domain(slot.epoch(self.spec.slots_per_epoch), Domain::Randao, &fork), + ) { None => return Ok(PollOutcome::SignerRejection(slot)), Some(signature) => signature, } @@ -152,7 +156,12 @@ impl BlockProducer BlockProducer Option { + fn sign_block(&mut self, mut block: BeaconBlock, domain: u64) -> Option { self.store_produce(&block); let proposal = Proposal { @@ -182,7 +191,7 @@ impl BlockProducer None, Some(signature) => { diff --git a/eth2/block_proposer/src/test_utils/epoch_map.rs b/eth2/block_proposer/src/test_utils/epoch_map.rs index f7d8dcdcb..6658c7526 100644 --- a/eth2/block_proposer/src/test_utils/epoch_map.rs +++ b/eth2/block_proposer/src/test_utils/epoch_map.rs @@ -1,6 +1,6 @@ use crate::{DutiesReader, DutiesReaderError}; use std::collections::HashMap; -use types::{Epoch, Slot}; +use types::{Epoch, Fork, Slot}; pub struct EpochMap { slots_per_epoch: u64, @@ -25,4 +25,12 @@ impl DutiesReader for EpochMap { _ => Err(DutiesReaderError::UnknownEpoch), } } + + fn fork(&self) -> Result { + Ok(Fork { + previous_version: 0, + current_version: 0, + epoch: Epoch::new(0), + }) + } } diff --git a/eth2/block_proposer/src/traits.rs b/eth2/block_proposer/src/traits.rs index c6e57d833..1c0da9acf 100644 --- a/eth2/block_proposer/src/traits.rs +++ b/eth2/block_proposer/src/traits.rs @@ -1,4 +1,4 @@ -use types::{BeaconBlock, Signature, Slot}; +use types::{BeaconBlock, Fork, Signature, Slot}; #[derive(Debug, PartialEq, Clone)] pub enum BeaconNodeError { @@ -40,6 +40,7 @@ pub enum DutiesReaderError { /// Informs a validator of their duties (e.g., block production). pub trait DutiesReader: Send + Sync { fn is_block_production_slot(&self, slot: Slot) -> Result; + fn fork(&self) -> Result; } /// Signs message using an internally-maintained private key. diff --git a/validator_client/src/block_producer_service/mod.rs b/validator_client/src/block_producer_service/mod.rs index bd1e691cb..91e7606a7 100644 --- a/validator_client/src/block_producer_service/mod.rs +++ b/validator_client/src/block_producer_service/mod.rs @@ -50,6 +50,9 @@ impl BlockProducerServi Ok(BlockProducerPollOutcome::ValidatorIsUnknown(slot)) => { error!(self.log, "The Beacon Node does not recognise the validator"; "slot" => slot) } + Ok(BlockProducerPollOutcome::UnableToGetFork(slot)) => { + error!(self.log, "Unable to get a `Fork` struct to generate signature domains"; "slot" => slot) + } }; std::thread::sleep(Duration::from_millis(self.poll_interval_millis)); diff --git a/validator_client/src/duties/epoch_duties.rs b/validator_client/src/duties/epoch_duties.rs index d1bbfa156..35668b4a9 100644 --- a/validator_client/src/duties/epoch_duties.rs +++ b/validator_client/src/duties/epoch_duties.rs @@ -1,7 +1,7 @@ use block_proposer::{DutiesReader, DutiesReaderError}; use std::collections::HashMap; use std::sync::RwLock; -use types::{Epoch, Slot}; +use types::{Epoch, Fork, Slot}; /// The information required for a validator to propose and attest during some epoch. /// @@ -75,6 +75,17 @@ impl DutiesReader for EpochDutiesMap { .ok_or_else(|| DutiesReaderError::UnknownEpoch)?; Ok(duties.is_block_production_slot(slot)) } + + fn fork(&self) -> Result { + // TODO: this is garbage data. + // + // It will almost certainly cause signatures to fail verification. + Ok(Fork { + previous_version: 0, + current_version: 0, + epoch: Epoch::new(0), + }) + } } // TODO: add tests. From 195cb16a417c6eb621f306f3e87b95a1c1338227 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 7 Mar 2019 14:29:21 +1100 Subject: [PATCH 128/132] Update `test_harness` for spec v0.4.0 --- .../test_harness/src/test_case.rs | 51 +++++++++++++++++-- .../test_harness/src/test_case/config.rs | 25 ++------- .../per_block_processing/verify_deposit.rs | 1 - 3 files changed, 50 insertions(+), 27 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/src/test_case.rs b/beacon_node/beacon_chain/test_harness/src/test_case.rs index cde3f55a8..e3bcf740a 100644 --- a/beacon_node/beacon_chain/test_harness/src/test_case.rs +++ b/beacon_node/beacon_chain/test_harness/src/test_case.rs @@ -3,9 +3,11 @@ use crate::beacon_chain_harness::BeaconChainHarness; use beacon_chain::CheckPoint; +use bls::create_proof_of_possession; use log::{info, warn}; -use ssz::TreeHash; +use ssz::SignedRoot; use types::*; + use types::{ attester_slashing::AttesterSlashingBuilder, proposer_slashing::ProposerSlashingBuilder, }; @@ -83,12 +85,18 @@ impl TestCase { // -1 slots because genesis counts as a slot. for slot_height in 0..slots - 1 { + // Used to ensure that deposits in the same slot have incremental deposit indices. + let mut deposit_index_offset = 0; + // Feed deposits to the BeaconChain. if let Some(ref deposits) = self.config.deposits { - for (slot, deposit, keypair) in deposits { + for (slot, amount) in deposits { if *slot == slot_height { info!("Including deposit at slot height {}.", slot_height); - harness.add_deposit(deposit.clone(), Some(keypair.clone())); + let (deposit, keypair) = + build_deposit(&harness, *amount, deposit_index_offset); + harness.add_deposit(deposit, Some(keypair.clone())); + deposit_index_offset += 1; } } } @@ -200,6 +208,41 @@ impl TestCase { } } +/// Builds a `Deposit` this is valid for the given `BeaconChainHarness`. +/// +/// `index_offset` is used to ensure that `deposit.index == state.index` when adding multiple +/// deposits. +fn build_deposit( + harness: &BeaconChainHarness, + amount: u64, + index_offset: u64, +) -> (Deposit, Keypair) { + let keypair = Keypair::random(); + let proof_of_possession = create_proof_of_possession(&keypair); + let index = harness.beacon_chain.state.read().deposit_index + index_offset; + + info!("index: {}, index_offset: {}", index, index_offset); + + let deposit = Deposit { + // Note: `branch` and `index` will need to be updated once the spec defines their + // validity. + branch: vec![], + index, + deposit_data: DepositData { + amount, + timestamp: 1, + deposit_input: DepositInput { + pubkey: keypair.pk.clone(), + withdrawal_credentials: Hash256::zero(), + proof_of_possession, + }, + }, + }; + + (deposit, keypair) +} + +/// Builds a `VoluntaryExit` this is valid for the given `BeaconChainHarness`. fn build_exit(harness: &BeaconChainHarness, validator_index: u64) -> VoluntaryExit { let epoch = harness .beacon_chain @@ -213,7 +256,7 @@ fn build_exit(harness: &BeaconChainHarness, validator_index: u64) -> VoluntaryEx signature: Signature::empty_signature(), }; - let message = exit.hash_tree_root(); + let message = exit.signed_root(); exit.signature = harness .validator_sign(validator_index as usize, &message[..], epoch, Domain::Exit) diff --git a/beacon_node/beacon_chain/test_harness/src/test_case/config.rs b/beacon_node/beacon_chain/test_harness/src/test_case/config.rs index 4034c8ddc..140fc4128 100644 --- a/beacon_node/beacon_chain/test_harness/src/test_case/config.rs +++ b/beacon_node/beacon_chain/test_harness/src/test_case/config.rs @@ -1,12 +1,12 @@ use super::yaml_helpers::{as_u64, as_usize, as_vec_u64}; -use bls::create_proof_of_possession; use types::*; use yaml_rust::Yaml; pub type ValidatorIndex = u64; pub type ValidatorIndices = Vec; +pub type GweiAmount = u64; -pub type DepositTuple = (SlotHeight, Deposit, Keypair); +pub type DepositTuple = (SlotHeight, GweiAmount); pub type ExitTuple = (SlotHeight, ValidatorIndex); pub type ProposerSlashingTuple = (SlotHeight, ValidatorIndex); pub type AttesterSlashingTuple = (SlotHeight, ValidatorIndices); @@ -101,30 +101,11 @@ fn parse_deposits(yaml: &Yaml) -> Option> { let mut deposits = vec![]; for deposit in yaml["deposits"].as_vec()? { - let keypair = Keypair::random(); - let proof_of_possession = create_proof_of_possession(&keypair); - let slot = as_u64(deposit, "slot").expect("Incomplete deposit (slot)"); let amount = as_u64(deposit, "amount").expect("Incomplete deposit (amount)") * 1_000_000_000; - let deposit = Deposit { - // Note: `branch` and `index` will need to be updated once the spec defines their - // validity. - branch: vec![], - index: 0, - deposit_data: DepositData { - amount, - timestamp: 1, - deposit_input: DepositInput { - pubkey: keypair.pk.clone(), - withdrawal_credentials: Hash256::zero(), - proof_of_possession, - }, - }, - }; - - deposits.push((SlotHeight::from(slot), deposit, keypair)); + deposits.push((SlotHeight::from(slot), amount)) } Some(deposits) diff --git a/eth2/state_processing/src/per_block_processing/verify_deposit.rs b/eth2/state_processing/src/per_block_processing/verify_deposit.rs index a1731f1a1..702c6b9e0 100644 --- a/eth2/state_processing/src/per_block_processing/verify_deposit.rs +++ b/eth2/state_processing/src/per_block_processing/verify_deposit.rs @@ -16,7 +16,6 @@ pub fn verify_deposit( ) -> Result<(), Error> { // TODO: verify serialized deposit data. - // TODO: verify deposit index. verify!( deposit.index == state.deposit_index, Invalid::BadIndex(state.deposit_index, deposit.index) From 15e4aabd8a99a1c5116241810aca2902254ec342 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 7 Mar 2019 16:15:38 +1100 Subject: [PATCH 129/132] Add deposit processing, fix clippy lints --- eth2/fork_choice/src/bitwise_lmd_ghost.rs | 2 +- eth2/state_processing/Cargo.toml | 1 + .../src/per_block_processing.rs | 9 +- .../src/per_block_processing/errors.rs | 45 ++++++- .../src/per_block_processing/verify_exit.rs | 4 +- .../verify_proposer_slashing.rs | 6 +- .../per_block_processing/verify_transfer.rs | 120 +++++++++++++++++- eth2/types/src/transfer.rs | 5 +- eth2/utils/bls/src/lib.rs | 10 ++ eth2/utils/merkle_proof/src/lib.rs | 6 +- eth2/utils/ssz_derive/src/lib.rs | 2 +- 11 files changed, 184 insertions(+), 26 deletions(-) diff --git a/eth2/fork_choice/src/bitwise_lmd_ghost.rs b/eth2/fork_choice/src/bitwise_lmd_ghost.rs index d50471f56..fd1c3dea4 100644 --- a/eth2/fork_choice/src/bitwise_lmd_ghost.rs +++ b/eth2/fork_choice/src/bitwise_lmd_ghost.rs @@ -379,7 +379,7 @@ impl ForkChoice for BitwiseLMDGhost { trace!("Current Step: {}", step); if let Some(clear_winner) = self.get_clear_winner( &latest_votes, - block_height - (block_height % u64::from(step)) + u64::from(step), + block_height - (block_height % step) + step, spec, ) { current_head = clear_winner; diff --git a/eth2/state_processing/Cargo.toml b/eth2/state_processing/Cargo.toml index bdc14c1d7..b673996ae 100644 --- a/eth2/state_processing/Cargo.toml +++ b/eth2/state_processing/Cargo.toml @@ -13,6 +13,7 @@ criterion = "0.2" env_logger = "0.6.0" [dependencies] +bls = { path = "../utils/bls" } hashing = { path = "../utils/hashing" } int_to_bytes = { path = "../utils/int_to_bytes" } integer-sqrt = "0.1" diff --git a/eth2/state_processing/src/per_block_processing.rs b/eth2/state_processing/src/per_block_processing.rs index 06a5f81c7..61b12bedb 100644 --- a/eth2/state_processing/src/per_block_processing.rs +++ b/eth2/state_processing/src/per_block_processing.rs @@ -9,7 +9,7 @@ pub use self::verify_attester_slashing::verify_attester_slashing; pub use validate_attestation::{validate_attestation, validate_attestation_without_signature}; pub use verify_deposit::verify_deposit; pub use verify_exit::verify_exit; -pub use verify_transfer::verify_transfer; +pub use verify_transfer::{execute_transfer, verify_transfer}; pub mod errors; mod validate_attestation; @@ -373,12 +373,7 @@ pub fn process_transfers( ); for (i, transfer) in transfers.iter().enumerate() { verify_transfer(&state, transfer, spec).map_err(|e| e.into_with_index(i))?; - - let block_proposer = state.get_beacon_proposer_index(state.slot, spec)?; - - state.validator_balances[transfer.from as usize] -= transfer.amount + transfer.fee; - state.validator_balances[transfer.to as usize] += transfer.amount + transfer.fee; - state.validator_balances[block_proposer as usize] += transfer.fee; + execute_transfer(state, transfer, spec).map_err(|e| e.into_with_index(i))?; } Ok(()) diff --git a/eth2/state_processing/src/per_block_processing/errors.rs b/eth2/state_processing/src/per_block_processing/errors.rs index 6c9719fc2..59d3f2f80 100644 --- a/eth2/state_processing/src/per_block_processing/errors.rs +++ b/eth2/state_processing/src/per_block_processing/errors.rs @@ -332,10 +332,51 @@ impl_into_with_index_without_beacon_error!(ExitValidationError, ExitInvalid); pub enum TransferValidationError { /// Validation completed successfully and the object is invalid. Invalid(TransferInvalid), + /// Encountered a `BeaconStateError` whilst attempting to determine validity. + BeaconStateError(BeaconStateError), } /// Describes why an object is invalid. #[derive(Debug, PartialEq)] -pub enum TransferInvalid {} +pub enum TransferInvalid { + /// The validator indicated by `transfer.from` is unknown. + FromValidatorUnknown(u64), + /// The validator indicated by `transfer.to` is unknown. + ToValidatorUnknown(u64), + /// The balance of `transfer.from` is insufficient. + /// + /// (required, available) + FromBalanceInsufficient(u64, u64), + /// Adding `transfer.fee` to `transfer.amount` causes an overflow. + /// + /// (transfer_fee, transfer_amount) + FeeOverflow(u64, u64), + /// This transfer would result in the `transfer.from` account to have `0 < balance < + /// min_deposit_amount` + /// + /// (resulting_amount, min_deposit_amount) + InvalidResultingFromBalance(u64, u64), + /// The state slot does not match `transfer.slot`. + /// + /// (state_slot, transfer_slot) + StateSlotMismatch(Slot, Slot), + /// The `transfer.from` validator has been activated and is not withdrawable. + /// + /// (from_validator) + FromValidatorIneligableForTransfer(u64), + /// The validators withdrawal credentials do not match `transfer.pubkey`. + WithdrawalCredentialsMismatch, + /// The deposit was not signed by `deposit.pubkey`. + BadSignature, + /// Overflow when adding to `transfer.to` balance. + /// + /// (to_balance, transfer_amount) + ToBalanceOverflow(u64, u64), + /// Overflow when adding to beacon proposer balance. + /// + /// (proposer_balance, transfer_fee) + ProposerBalanceOverflow(u64, u64), +} -impl_into_with_index_without_beacon_error!(TransferValidationError, TransferInvalid); +impl_from_beacon_state_error!(TransferValidationError); +impl_into_with_index_with_beacon_error!(TransferValidationError, TransferInvalid); diff --git a/eth2/state_processing/src/per_block_processing/verify_exit.rs b/eth2/state_processing/src/per_block_processing/verify_exit.rs index 408b77077..8cd54fb69 100644 --- a/eth2/state_processing/src/per_block_processing/verify_exit.rs +++ b/eth2/state_processing/src/per_block_processing/verify_exit.rs @@ -16,9 +16,7 @@ pub fn verify_exit( let validator = state .validator_registry .get(exit.validator_index as usize) - .ok_or(Error::Invalid(Invalid::ValidatorUnknown( - exit.validator_index, - )))?; + .ok_or_else(|| Error::Invalid(Invalid::ValidatorUnknown(exit.validator_index)))?; verify!( validator.exit_epoch diff --git a/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs b/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs index 0350255ec..c3c0079a9 100644 --- a/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs +++ b/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs @@ -16,9 +16,9 @@ pub fn verify_proposer_slashing( let proposer = state .validator_registry .get(proposer_slashing.proposer_index as usize) - .ok_or(Error::Invalid(Invalid::ProposerUnknown( - proposer_slashing.proposer_index, - )))?; + .ok_or_else(|| { + Error::Invalid(Invalid::ProposerUnknown(proposer_slashing.proposer_index)) + })?; verify!( proposer_slashing.proposal_1.slot == proposer_slashing.proposal_2.slot, diff --git a/eth2/state_processing/src/per_block_processing/verify_transfer.rs b/eth2/state_processing/src/per_block_processing/verify_transfer.rs index cd7bcb42c..15ec17142 100644 --- a/eth2/state_processing/src/per_block_processing/verify_transfer.rs +++ b/eth2/state_processing/src/per_block_processing/verify_transfer.rs @@ -1,4 +1,6 @@ use super::errors::{TransferInvalid as Invalid, TransferValidationError as Error}; +use bls::get_withdrawal_credentials; +use ssz::SignedRoot; use types::*; /// Indicates if a `Transfer` is valid to be included in a block in the current epoch of the given @@ -10,11 +12,121 @@ use types::*; /// /// Spec v0.4.0 pub fn verify_transfer( - _state: &BeaconState, - _transfer: &Transfer, - _spec: &ChainSpec, + state: &BeaconState, + transfer: &Transfer, + spec: &ChainSpec, ) -> Result<(), Error> { - // TODO: verify transfer. + let from_balance = *state + .validator_balances + .get(transfer.from as usize) + .ok_or_else(|| Error::Invalid(Invalid::FromValidatorUnknown(transfer.from)))?; + + let total_amount = transfer + .amount + .checked_add(transfer.fee) + .ok_or_else(|| Error::Invalid(Invalid::FeeOverflow(transfer.amount, transfer.fee)))?; + + verify!( + from_balance >= transfer.amount, + Invalid::FromBalanceInsufficient(transfer.amount, from_balance) + ); + + verify!( + from_balance >= transfer.fee, + Invalid::FromBalanceInsufficient(transfer.fee, from_balance) + ); + + verify!( + (from_balance == total_amount) + || (from_balance >= (total_amount + spec.min_deposit_amount)), + Invalid::InvalidResultingFromBalance(from_balance - total_amount, spec.min_deposit_amount) + ); + + verify!( + state.slot == transfer.slot, + Invalid::StateSlotMismatch(state.slot, transfer.slot) + ); + + let from_validator = state + .validator_registry + .get(transfer.from as usize) + .ok_or_else(|| Error::Invalid(Invalid::FromValidatorUnknown(transfer.from)))?; + let epoch = state.slot.epoch(spec.slots_per_epoch); + + verify!( + from_validator.is_withdrawable_at(epoch) + || from_validator.activation_epoch == spec.far_future_epoch, + Invalid::FromValidatorIneligableForTransfer(transfer.from) + ); + + let transfer_withdrawal_credentials = Hash256::from_slice( + &get_withdrawal_credentials(&transfer.pubkey, spec.bls_withdrawal_prefix_byte)[..], + ); + verify!( + from_validator.withdrawal_credentials == transfer_withdrawal_credentials, + Invalid::WithdrawalCredentialsMismatch + ); + + let message = transfer.signed_root(); + let domain = spec.get_domain( + transfer.slot.epoch(spec.slots_per_epoch), + Domain::Transfer, + &state.fork, + ); + + verify!( + transfer + .signature + .verify(&message[..], domain, &transfer.pubkey), + Invalid::BadSignature + ); + + Ok(()) +} + +/// Executes a transfer on the state. +/// +/// Does not check that the transfer is valid, however checks for overflow in all actions. +/// +/// Spec v0.4.0 +pub fn execute_transfer( + state: &mut BeaconState, + transfer: &Transfer, + spec: &ChainSpec, +) -> Result<(), Error> { + let from_balance = *state + .validator_balances + .get(transfer.from as usize) + .ok_or_else(|| Error::Invalid(Invalid::FromValidatorUnknown(transfer.from)))?; + let to_balance = *state + .validator_balances + .get(transfer.to as usize) + .ok_or_else(|| Error::Invalid(Invalid::ToValidatorUnknown(transfer.to)))?; + + let proposer_index = state.get_beacon_proposer_index(state.slot, spec)?; + let proposer_balance = state.validator_balances[proposer_index]; + + let total_amount = transfer + .amount + .checked_add(transfer.fee) + .ok_or_else(|| Error::Invalid(Invalid::FeeOverflow(transfer.amount, transfer.fee)))?; + + state.validator_balances[transfer.from as usize] = + from_balance.checked_sub(total_amount).ok_or_else(|| { + Error::Invalid(Invalid::FromBalanceInsufficient(total_amount, from_balance)) + })?; + + state.validator_balances[transfer.to as usize] = to_balance + .checked_add(transfer.amount) + .ok_or_else(|| Error::Invalid(Invalid::ToBalanceOverflow(to_balance, transfer.amount)))?; + + state.validator_balances[proposer_index] = + proposer_balance.checked_add(transfer.fee).ok_or_else(|| { + Error::Invalid(Invalid::ProposerBalanceOverflow( + proposer_balance, + transfer.fee, + )) + })?; Ok(()) } diff --git a/eth2/types/src/transfer.rs b/eth2/types/src/transfer.rs index f23d3845e..0382dee11 100644 --- a/eth2/types/src/transfer.rs +++ b/eth2/types/src/transfer.rs @@ -3,13 +3,14 @@ use crate::test_utils::TestRandom; use bls::{PublicKey, Signature}; use rand::RngCore; use serde_derive::Serialize; -use ssz_derive::{Decode, Encode, TreeHash}; +use ssz::TreeHash; +use ssz_derive::{Decode, Encode, SignedRoot, TreeHash}; use test_random_derive::TestRandom; /// The data submitted to the deposit contract. /// /// Spec v0.4.0 -#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)] +#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)] pub struct Transfer { pub from: u64, pub to: u64, diff --git a/eth2/utils/bls/src/lib.rs b/eth2/utils/bls/src/lib.rs index 865b8d82d..bb109b0a1 100644 --- a/eth2/utils/bls/src/lib.rs +++ b/eth2/utils/bls/src/lib.rs @@ -17,6 +17,7 @@ pub use crate::signature::Signature; pub const BLS_AGG_SIG_BYTE_SIZE: usize = 96; +use hashing::hash; use ssz::ssz_encode; /// For some signature and public key, ensure that the signature message was the public key and it @@ -33,6 +34,15 @@ pub fn create_proof_of_possession(keypair: &Keypair) -> Signature { Signature::new(&ssz_encode(&keypair.pk), 0, &keypair.sk) } +/// Returns the withdrawal credentials for a given public key. +pub fn get_withdrawal_credentials(pubkey: &PublicKey, prefix_byte: u8) -> Vec { + let hashed = hash(&ssz_encode(pubkey)); + let mut prefixed = vec![prefix_byte]; + prefixed.extend_from_slice(&hashed[1..]); + + prefixed +} + pub fn bls_verify_aggregate( pubkey: &AggregatePublicKey, message: &[u8], diff --git a/eth2/utils/merkle_proof/src/lib.rs b/eth2/utils/merkle_proof/src/lib.rs index 4d0abcea3..5ff8f79e6 100644 --- a/eth2/utils/merkle_proof/src/lib.rs +++ b/eth2/utils/merkle_proof/src/lib.rs @@ -25,14 +25,14 @@ fn merkle_root_from_branch(leaf: H256, branch: &[H256], depth: usize, index: usi let mut merkle_root = leaf.as_bytes().to_vec(); - for i in 0..depth { + for (i, leaf) in branch.iter().enumerate().take(depth) { let ith_bit = (index >> i) & 0x01; if ith_bit == 1 { - let input = concat(branch[i].as_bytes().to_vec(), merkle_root); + let input = concat(leaf.as_bytes().to_vec(), merkle_root); merkle_root = hash(&input); } else { let mut input = merkle_root; - input.extend_from_slice(branch[i].as_bytes()); + input.extend_from_slice(leaf.as_bytes()); merkle_root = hash(&input); } } diff --git a/eth2/utils/ssz_derive/src/lib.rs b/eth2/utils/ssz_derive/src/lib.rs index 4c6b9dc11..0d2e17f76 100644 --- a/eth2/utils/ssz_derive/src/lib.rs +++ b/eth2/utils/ssz_derive/src/lib.rs @@ -172,7 +172,7 @@ fn type_ident_is_signature(ident: &syn::Ident) -> bool { /// the final `Ident` in that path. /// /// E.g., for `types::Signature` returns `Signature`. -fn final_type_ident<'a>(field: &'a syn::Field) -> &'a syn::Ident { +fn final_type_ident(field: &syn::Field) -> &syn::Ident { match &field.ty { syn::Type::Path(path) => &path.path.segments.last().unwrap().value().ident, _ => panic!("ssz_derive only supports Path types."), From 1ef2652cacd225403b295fc5ae322de1682969fc Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 7 Mar 2019 17:23:11 +1100 Subject: [PATCH 130/132] Add transfer processing to BeaconChain --- beacon_node/beacon_chain/src/beacon_chain.rs | 44 +++++++++++++++- .../specs/validator_registry.yaml | 20 ++++++-- .../test_harness/src/beacon_chain_harness.rs | 15 +++++- .../test_harness/src/test_case.rs | 51 ++++++++++++++++--- .../test_harness/src/test_case/config.rs | 24 ++++++++- .../test_harness/src/test_case/state_check.rs | 50 ++++++++++++++++++ .../src/per_block_processing/errors.rs | 4 +- .../per_block_processing/verify_transfer.rs | 5 +- 8 files changed, 196 insertions(+), 17 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 653b2900d..3a8a3ea14 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -56,6 +56,7 @@ pub struct BeaconChain { pub attestation_aggregator: RwLock, pub deposits_for_inclusion: RwLock>, pub exits_for_inclusion: RwLock>, + pub transfers_for_inclusion: RwLock>, pub proposer_slashings_for_inclusion: RwLock>, pub attester_slashings_for_inclusion: RwLock>, canonical_head: RwLock, @@ -126,6 +127,7 @@ where attestation_aggregator, deposits_for_inclusion: RwLock::new(vec![]), exits_for_inclusion: RwLock::new(vec![]), + transfers_for_inclusion: RwLock::new(vec![]), proposer_slashings_for_inclusion: RwLock::new(vec![]), attester_slashings_for_inclusion: RwLock::new(vec![]), state: RwLock::new(genesis_state), @@ -436,6 +438,44 @@ where } } + /// Accept some transfer and queue it for inclusion in an appropriate block. + pub fn receive_transfer_for_inclusion(&self, transfer: Transfer) { + // TODO: transfers are not checked for validity; check them. + // + // https://github.com/sigp/lighthouse/issues/276 + self.transfers_for_inclusion.write().push(transfer); + } + + /// Return a vec of transfers suitable for inclusion in some block. + pub fn get_transfers_for_block(&self) -> Vec { + // TODO: transfers are indiscriminately included; check them for validity. + // + // https://github.com/sigp/lighthouse/issues/275 + self.transfers_for_inclusion.read().clone() + } + + /// Takes a list of `Deposits` that were included in recent blocks and removes them from the + /// inclusion queue. + /// + /// This ensures that `Deposits` are not included twice in successive blocks. + pub fn set_transfers_as_included(&self, included_transfers: &[Transfer]) { + // TODO: method does not take forks into account; consider this. + let mut indices_to_delete = vec![]; + + for included in included_transfers { + for (i, for_inclusion) in self.transfers_for_inclusion.read().iter().enumerate() { + if included == for_inclusion { + indices_to_delete.push(i); + } + } + } + + let transfers_for_inclusion = &mut self.transfers_for_inclusion.write(); + for i in indices_to_delete { + transfers_for_inclusion.remove(i); + } + } + /// Accept some proposer slashing and queue it for inclusion in an appropriate block. pub fn receive_proposer_slashing_for_inclusion(&self, proposer_slashing: ProposerSlashing) { // TODO: proposer_slashings are not checked for validity; check them. @@ -664,6 +704,8 @@ where // Update the inclusion queues so they aren't re-submitted. self.set_deposits_as_included(&block.body.deposits[..]); + self.set_transfers_as_included(&block.body.transfers[..]); + self.set_exits_as_included(&block.body.voluntary_exits[..]); self.set_proposer_slashings_as_included(&block.body.proposer_slashings[..]); self.set_attester_slashings_as_included(&block.body.attester_slashings[..]); @@ -730,7 +772,7 @@ where attestations, deposits: self.get_deposits_for_block(), voluntary_exits: self.get_exits_for_block(), - transfers: vec![], + transfers: self.get_transfers_for_block(), }, }; diff --git a/beacon_node/beacon_chain/test_harness/specs/validator_registry.yaml b/beacon_node/beacon_chain/test_harness/specs/validator_registry.yaml index a6021dbe5..aea7dcf31 100644 --- a/beacon_node/beacon_chain/test_harness/specs/validator_registry.yaml +++ b/beacon_node/beacon_chain/test_harness/specs/validator_registry.yaml @@ -10,18 +10,23 @@ test_cases: num_slots: 64 skip_slots: [2, 3] deposits: - # At slot 1, create a new validator deposit of 32 ETH. + # At slot 1, create a new validator deposit of 5 ETH. - slot: 1 - amount: 32 + amount: 5000000000 # Trigger more deposits... - slot: 3 - amount: 32 + amount: 5000000000 - slot: 5 - amount: 32 + amount: 32000000000 exits: # At slot 10, submit an exit for validator #50. - slot: 10 validator_index: 50 + transfers: + - slot: 6 + from: 1000 + to: 1001 + amount: 5000000000 proposer_slashings: # At slot 2, trigger a proposer slashing for validator #42. - slot: 2 @@ -44,4 +49,11 @@ test_cases: slashed_validators: [11, 12, 13, 14, 42] exited_validators: [] exit_initiated_validators: [50] + balances: + - validator_index: 1000 + comparison: "eq" + balance: 0 + - validator_index: 1001 + comparison: "eq" + balance: 10000000000 diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index 38c41e38c..f220619ce 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -1,7 +1,7 @@ use super::ValidatorHarness; use beacon_chain::{BeaconChain, BlockProcessingOutcome}; pub use beacon_chain::{BeaconChainError, CheckPoint}; -use bls::create_proof_of_possession; +use bls::{create_proof_of_possession, get_withdrawal_credentials}; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, MemoryDB, @@ -67,7 +67,13 @@ impl BeaconChainHarness { timestamp: genesis_time - 1, deposit_input: DepositInput { pubkey: keypair.pk.clone(), - withdrawal_credentials: Hash256::zero(), // Withdrawal not possible. + // Validator can withdraw using their main keypair. + withdrawal_credentials: Hash256::from_slice( + &get_withdrawal_credentials( + &keypair.pk, + spec.bls_withdrawal_prefix_byte, + )[..], + ), proof_of_possession: create_proof_of_possession(&keypair), }, }, @@ -286,6 +292,11 @@ impl BeaconChainHarness { self.beacon_chain.receive_exit_for_inclusion(exit); } + /// Submit an transfer to the `BeaconChain` for inclusion in some block. + pub fn add_transfer(&mut self, transfer: Transfer) { + self.beacon_chain.receive_transfer_for_inclusion(transfer); + } + /// Submit a proposer slashing to the `BeaconChain` for inclusion in some block. pub fn add_proposer_slashing(&mut self, proposer_slashing: ProposerSlashing) { self.beacon_chain diff --git a/beacon_node/beacon_chain/test_harness/src/test_case.rs b/beacon_node/beacon_chain/test_harness/src/test_case.rs index e3bcf740a..97b4bb3d4 100644 --- a/beacon_node/beacon_chain/test_harness/src/test_case.rs +++ b/beacon_node/beacon_chain/test_harness/src/test_case.rs @@ -3,7 +3,7 @@ use crate::beacon_chain_harness::BeaconChainHarness; use beacon_chain::CheckPoint; -use bls::create_proof_of_possession; +use bls::{create_proof_of_possession, get_withdrawal_credentials}; use log::{info, warn}; use ssz::SignedRoot; use types::*; @@ -83,8 +83,8 @@ impl TestCase { info!("Starting simulation across {} slots...", slots); - // -1 slots because genesis counts as a slot. - for slot_height in 0..slots - 1 { + // Start at 1 because genesis counts as a slot. + for slot_height in 1..slots { // Used to ensure that deposits in the same slot have incremental deposit indices. let mut deposit_index_offset = 0; @@ -144,6 +144,20 @@ impl TestCase { } } + // Feed transfers to the BeaconChain. + if let Some(ref transfers) = self.config.transfers { + for (slot, from, to, amount) in transfers { + if *slot == slot_height { + info!( + "Including transfer at slot height {} from validator {}.", + slot_height, from + ); + let transfer = build_transfer(&harness, *from, *to, *amount); + harness.add_transfer(transfer); + } + } + } + // Build a block or skip a slot. match self.config.skip_slots { Some(ref skip_slots) if skip_slots.contains(&slot_height) => { @@ -208,6 +222,30 @@ impl TestCase { } } +/// Builds a `Deposit` this is valid for the given `BeaconChainHarness` at its next slot. +fn build_transfer(harness: &BeaconChainHarness, from: u64, to: u64, amount: u64) -> Transfer { + let slot = harness.beacon_chain.state.read().slot + 1; + + let mut transfer = Transfer { + from, + to, + amount, + fee: 0, + slot, + pubkey: harness.validators[from as usize].keypair.pk.clone(), + signature: Signature::empty_signature(), + }; + + let message = transfer.signed_root(); + let epoch = slot.epoch(harness.spec.slots_per_epoch); + + transfer.signature = harness + .validator_sign(from as usize, &message[..], epoch, Domain::Transfer) + .expect("Unable to sign Transfer"); + + transfer +} + /// Builds a `Deposit` this is valid for the given `BeaconChainHarness`. /// /// `index_offset` is used to ensure that `deposit.index == state.index` when adding multiple @@ -220,8 +258,9 @@ fn build_deposit( let keypair = Keypair::random(); let proof_of_possession = create_proof_of_possession(&keypair); let index = harness.beacon_chain.state.read().deposit_index + index_offset; - - info!("index: {}, index_offset: {}", index, index_offset); + let withdrawal_credentials = Hash256::from_slice( + &get_withdrawal_credentials(&keypair.pk, harness.spec.bls_withdrawal_prefix_byte)[..], + ); let deposit = Deposit { // Note: `branch` and `index` will need to be updated once the spec defines their @@ -233,7 +272,7 @@ fn build_deposit( timestamp: 1, deposit_input: DepositInput { pubkey: keypair.pk.clone(), - withdrawal_credentials: Hash256::zero(), + withdrawal_credentials, proof_of_possession, }, }, diff --git a/beacon_node/beacon_chain/test_harness/src/test_case/config.rs b/beacon_node/beacon_chain/test_harness/src/test_case/config.rs index 140fc4128..f336b9d53 100644 --- a/beacon_node/beacon_chain/test_harness/src/test_case/config.rs +++ b/beacon_node/beacon_chain/test_harness/src/test_case/config.rs @@ -10,6 +10,8 @@ pub type DepositTuple = (SlotHeight, GweiAmount); pub type ExitTuple = (SlotHeight, ValidatorIndex); pub type ProposerSlashingTuple = (SlotHeight, ValidatorIndex); pub type AttesterSlashingTuple = (SlotHeight, ValidatorIndices); +/// (slot_height, from, to, amount) +pub type TransferTuple = (SlotHeight, ValidatorIndex, ValidatorIndex, GweiAmount); /// Defines the execution of a `BeaconStateHarness` across a series of slots. #[derive(Debug)] @@ -30,6 +32,8 @@ pub struct Config { pub attester_slashings: Option>, /// Exits to be including during execution. pub exits: Option>, + /// Transfers to be including during execution. + pub transfers: Option>, } impl Config { @@ -47,10 +51,27 @@ impl Config { proposer_slashings: parse_proposer_slashings(&yaml), attester_slashings: parse_attester_slashings(&yaml), exits: parse_exits(&yaml), + transfers: parse_transfers(&yaml), } } } +/// Parse the `transfers` section of the YAML document. +fn parse_transfers(yaml: &Yaml) -> Option> { + let mut tuples = vec![]; + + for exit in yaml["transfers"].as_vec()? { + let slot = as_u64(exit, "slot").expect("Incomplete transfer (slot)"); + let from = as_u64(exit, "from").expect("Incomplete transfer (from)"); + let to = as_u64(exit, "to").expect("Incomplete transfer (to)"); + let amount = as_u64(exit, "amount").expect("Incomplete transfer (amount)"); + + tuples.push((SlotHeight::from(slot), from, to, amount)); + } + + Some(tuples) +} + /// Parse the `attester_slashings` section of the YAML document. fn parse_exits(yaml: &Yaml) -> Option> { let mut tuples = vec![]; @@ -102,8 +123,7 @@ fn parse_deposits(yaml: &Yaml) -> Option> { for deposit in yaml["deposits"].as_vec()? { let slot = as_u64(deposit, "slot").expect("Incomplete deposit (slot)"); - let amount = - as_u64(deposit, "amount").expect("Incomplete deposit (amount)") * 1_000_000_000; + let amount = as_u64(deposit, "amount").expect("Incomplete deposit (amount)"); deposits.push((SlotHeight::from(slot), amount)) } diff --git a/beacon_node/beacon_chain/test_harness/src/test_case/state_check.rs b/beacon_node/beacon_chain/test_harness/src/test_case/state_check.rs index 6fa75364a..4d2bfd07d 100644 --- a/beacon_node/beacon_chain/test_harness/src/test_case/state_check.rs +++ b/beacon_node/beacon_chain/test_harness/src/test_case/state_check.rs @@ -3,6 +3,11 @@ use log::info; use types::*; use yaml_rust::Yaml; +type ValidatorIndex = u64; +type BalanceGwei = u64; + +type BalanceCheckTuple = (ValidatorIndex, String, BalanceGwei); + /// Tests to be conducted upon a `BeaconState` object generated during the execution of a /// `TestCase`. #[derive(Debug)] @@ -17,6 +22,8 @@ pub struct StateCheck { pub exited_validators: Option>, /// A list of validator indices which have had an exit initiated. Must be in ascending order. pub exit_initiated_validators: Option>, + /// A list of balances to check. + pub balances: Option>, } impl StateCheck { @@ -30,6 +37,7 @@ impl StateCheck { slashed_validators: as_vec_u64(&yaml, "slashed_validators"), exited_validators: as_vec_u64(&yaml, "exited_validators"), exit_initiated_validators: as_vec_u64(&yaml, "exit_initiated_validators"), + balances: parse_balances(&yaml), } } @@ -124,5 +132,47 @@ impl StateCheck { exit_initiated_validators ); } + + // Check validator balances. + if let Some(ref balances) = self.balances { + for (index, comparison, expected) in balances { + let actual = *state + .validator_balances + .get(*index as usize) + .expect("Balance check specifies unknown validator"); + + let result = match comparison.as_ref() { + "eq" => actual == *expected, + _ => panic!("Unknown balance comparison (use `eq`)"), + }; + assert!( + result, + format!( + "Validator balance for {}: {} !{} {}.", + index, actual, comparison, expected + ) + ); + info!("OK: validator balance for {:?}.", index); + } + } } } + +/// Parse the `transfers` section of the YAML document. +fn parse_balances(yaml: &Yaml) -> Option> { + let mut tuples = vec![]; + + for exit in yaml["balances"].as_vec()? { + let from = + as_u64(exit, "validator_index").expect("Incomplete balance check (validator_index)"); + let comparison = exit["comparison"] + .clone() + .into_string() + .expect("Incomplete balance check (amount)"); + let balance = as_u64(exit, "balance").expect("Incomplete balance check (balance)"); + + tuples.push((from, comparison, balance)); + } + + Some(tuples) +} diff --git a/eth2/state_processing/src/per_block_processing/errors.rs b/eth2/state_processing/src/per_block_processing/errors.rs index 59d3f2f80..36b0d4942 100644 --- a/eth2/state_processing/src/per_block_processing/errors.rs +++ b/eth2/state_processing/src/per_block_processing/errors.rs @@ -365,7 +365,9 @@ pub enum TransferInvalid { /// (from_validator) FromValidatorIneligableForTransfer(u64), /// The validators withdrawal credentials do not match `transfer.pubkey`. - WithdrawalCredentialsMismatch, + /// + /// (state_credentials, transfer_pubkey_credentials) + WithdrawalCredentialsMismatch(Hash256, Hash256), /// The deposit was not signed by `deposit.pubkey`. BadSignature, /// Overflow when adding to `transfer.to` balance. diff --git a/eth2/state_processing/src/per_block_processing/verify_transfer.rs b/eth2/state_processing/src/per_block_processing/verify_transfer.rs index 15ec17142..4746fc75c 100644 --- a/eth2/state_processing/src/per_block_processing/verify_transfer.rs +++ b/eth2/state_processing/src/per_block_processing/verify_transfer.rs @@ -64,7 +64,10 @@ pub fn verify_transfer( ); verify!( from_validator.withdrawal_credentials == transfer_withdrawal_credentials, - Invalid::WithdrawalCredentialsMismatch + Invalid::WithdrawalCredentialsMismatch( + from_validator.withdrawal_credentials, + transfer_withdrawal_credentials + ) ); let message = transfer.signed_root(); From 60098a051d2615552621b3bd2b9c36e601d8c307 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 8 Mar 2019 09:23:57 +1100 Subject: [PATCH 131/132] Fix/silence clippy lints --- beacon_node/beacon_chain/src/beacon_chain.rs | 1 + .../test_harness/src/test_case.rs | 1 + .../test_harness/src/validator_harness/mod.rs | 28 +++++++++++-------- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 3a8a3ea14..3d2efa8ae 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -73,6 +73,7 @@ where F: ForkChoice, { /// Instantiate a new Beacon Chain, from genesis. + #[allow(clippy::too_many_arguments)] // Will be re-factored in the coming weeks. pub fn genesis( state_store: Arc>, block_store: Arc>, diff --git a/beacon_node/beacon_chain/test_harness/src/test_case.rs b/beacon_node/beacon_chain/test_harness/src/test_case.rs index 97b4bb3d4..b2709edfc 100644 --- a/beacon_node/beacon_chain/test_harness/src/test_case.rs +++ b/beacon_node/beacon_chain/test_harness/src/test_case.rs @@ -69,6 +69,7 @@ impl TestCase { } /// Executes the test case, returning an `ExecutionResult`. + #[allow(clippy::cyclomatic_complexity)] pub fn execute(&self) -> ExecutionResult { let spec = self.spec(); let validator_count = self.config.deposits_for_chain_start; diff --git a/beacon_node/beacon_chain/test_harness/src/validator_harness/mod.rs b/beacon_node/beacon_chain/test_harness/src/validator_harness/mod.rs index 60c2f8ecf..91a679463 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator_harness/mod.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator_harness/mod.rs @@ -28,24 +28,28 @@ pub enum AttestationProduceError { PollError(AttestationPollError), } +type TestingBlockProducer = BlockProducer< + TestingSlotClock, + DirectBeaconNode>, + DirectDuties>, + LocalSigner, +>; + +type TestingAttester = Attester< + TestingSlotClock, + DirectBeaconNode>, + DirectDuties>, + LocalSigner, +>; + /// A `BlockProducer` and `Attester` which sign using a common keypair. /// /// The test validator connects directly to a borrowed `BeaconChain` struct. It is useful for /// testing that the core proposer and attester logic is functioning. Also for supporting beacon /// chain tests. pub struct ValidatorHarness { - pub block_producer: BlockProducer< - TestingSlotClock, - DirectBeaconNode>, - DirectDuties>, - LocalSigner, - >, - pub attester: Attester< - TestingSlotClock, - DirectBeaconNode>, - DirectDuties>, - LocalSigner, - >, + pub block_producer: TestingBlockProducer, + pub attester: TestingAttester, pub spec: Arc, pub epoch_map: Arc>>, pub keypair: Keypair, From f479beb87e8959a09d88a281686f7717108fbd35 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 8 Mar 2019 09:26:03 +1100 Subject: [PATCH 132/132] Implement deposit merkle root verification. It is currently disabled, but it's there for later. --- eth2/state_processing/Cargo.toml | 2 + .../src/per_block_processing.rs | 8 ++- .../src/per_block_processing/errors.rs | 3 ++ .../per_block_processing/verify_deposit.rs | 54 +++++++++++++++++-- 4 files changed, 62 insertions(+), 5 deletions(-) diff --git a/eth2/state_processing/Cargo.toml b/eth2/state_processing/Cargo.toml index b673996ae..c51ce8372 100644 --- a/eth2/state_processing/Cargo.toml +++ b/eth2/state_processing/Cargo.toml @@ -18,6 +18,8 @@ hashing = { path = "../utils/hashing" } int_to_bytes = { path = "../utils/int_to_bytes" } integer-sqrt = "0.1" log = "0.4" +merkle_proof = { path = "../utils/merkle_proof" } ssz = { path = "../utils/ssz" } +ssz_derive = { path = "../utils/ssz_derive" } types = { path = "../types" } rayon = "1.0" diff --git a/eth2/state_processing/src/per_block_processing.rs b/eth2/state_processing/src/per_block_processing.rs index 61b12bedb..1ab1eed71 100644 --- a/eth2/state_processing/src/per_block_processing.rs +++ b/eth2/state_processing/src/per_block_processing.rs @@ -20,6 +20,11 @@ mod verify_proposer_slashing; mod verify_slashable_attestation; mod verify_transfer; +// Set to `true` to check the merkle proof that a deposit is in the eth1 deposit root. +// +// Presently disabled to make testing easier. +const VERIFY_DEPOSIT_MERKLE_PROOFS: bool = false; + /// Updates the state for a new block, whilst validating that the block is valid. /// /// Returns `Ok(())` if the block is valid and the state was successfully updated. Otherwise @@ -309,7 +314,8 @@ pub fn process_deposits( Invalid::MaxDepositsExceeded ); for (i, deposit) in deposits.iter().enumerate() { - verify_deposit(state, deposit, spec).map_err(|e| e.into_with_index(i))?; + verify_deposit(state, deposit, VERIFY_DEPOSIT_MERKLE_PROOFS, spec) + .map_err(|e| e.into_with_index(i))?; state .process_deposit( diff --git a/eth2/state_processing/src/per_block_processing/errors.rs b/eth2/state_processing/src/per_block_processing/errors.rs index 36b0d4942..b97d8bacc 100644 --- a/eth2/state_processing/src/per_block_processing/errors.rs +++ b/eth2/state_processing/src/per_block_processing/errors.rs @@ -292,6 +292,9 @@ pub enum DepositInvalid { /// /// (state_index, deposit_index) BadIndex(u64, u64), + /// The specified `branch` and `index` did not form a valid proof that the deposit is included + /// in the eth1 deposit root. + BadMerkleProof, } impl_into_with_index_without_beacon_error!(DepositValidationError, DepositInvalid); diff --git a/eth2/state_processing/src/per_block_processing/verify_deposit.rs b/eth2/state_processing/src/per_block_processing/verify_deposit.rs index 702c6b9e0..69dae1533 100644 --- a/eth2/state_processing/src/per_block_processing/verify_deposit.rs +++ b/eth2/state_processing/src/per_block_processing/verify_deposit.rs @@ -1,4 +1,8 @@ use super::errors::{DepositInvalid as Invalid, DepositValidationError as Error}; +use hashing::hash; +use merkle_proof::verify_merkle_proof; +use ssz::ssz_encode; +use ssz_derive::Encode; use types::*; /// Indicates if a `Deposit` is valid to be included in a block in the current epoch of the given @@ -12,16 +16,58 @@ use types::*; pub fn verify_deposit( state: &BeaconState, deposit: &Deposit, - _spec: &ChainSpec, + verify_merkle_branch: bool, + spec: &ChainSpec, ) -> Result<(), Error> { - // TODO: verify serialized deposit data. - verify!( deposit.index == state.deposit_index, Invalid::BadIndex(state.deposit_index, deposit.index) ); - // TODO: verify merkle branch. + if verify_merkle_branch { + verify!( + verify_deposit_merkle_proof(state, deposit, spec), + Invalid::BadMerkleProof + ); + } Ok(()) } + +/// Verify that a deposit is included in the state's eth1 deposit root. +/// +/// Spec v0.4.0 +fn verify_deposit_merkle_proof(state: &BeaconState, deposit: &Deposit, spec: &ChainSpec) -> bool { + let leaf = hash(&get_serialized_deposit_data(deposit)); + verify_merkle_proof( + Hash256::from_slice(&leaf), + &deposit.branch, + spec.deposit_contract_tree_depth as usize, + deposit.index as usize, + state.latest_eth1_data.deposit_root, + ) +} + +/// Helper struct for easily getting the serialized data generated by the deposit contract. +/// +/// Spec v0.4.0 +#[derive(Encode)] +struct SerializedDepositData { + amount: u64, + timestamp: u64, + input: DepositInput, +} + +/// Return the serialized data generated by the deposit contract that is used to generate the +/// merkle proof. +/// +/// Spec v0.4.0 +fn get_serialized_deposit_data(deposit: &Deposit) -> Vec { + let serialized_deposit_data = SerializedDepositData { + amount: deposit.deposit_data.amount, + timestamp: deposit.deposit_data.timestamp, + input: deposit.deposit_data.deposit_input.clone(), + }; + + ssz_encode(&serialized_deposit_data) +}